diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 91d3d44..0408e83 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -87,4 +87,4 @@ jobs: contents: read packages: write id-token: write - security-events: write \ No newline at end of file + security-events: write diff --git a/.go-version b/.go-version index 14bee92..ac1df3f 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.23.2 +1.23.3 diff --git a/Makefile b/Makefile index bf1e686..13a7514 100644 --- a/Makefile +++ b/Makefile @@ -56,16 +56,19 @@ help: ## Display this help. ##@ Development +.PHONY: generate +generate: codegen manifests fmt ## Generate code, documentation, etc + +.PHONY: codegen +codegen: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." + .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./api/..." output:crd:artifacts:config=config/crd/bases $(CONTROLLER_GEN) rbac:roleName=manager-role paths="./internal/controller/telemetry/..." output:rbac:artifacts:config=./config/rbac cp config/crd/bases/* charts/telemetry-controller/crds/ -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." - .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... diff --git a/api/telemetry/v1alpha1/otlp_config.go b/api/telemetry/v1alpha1/otlp_config.go index 9a90218..4d08996 100644 --- a/api/telemetry/v1alpha1/otlp_config.go +++ b/api/telemetry/v1alpha1/otlp_config.go @@ -29,43 +29,35 @@ type TimeoutSettings struct { // QueueSettings defines configuration for queueing batches before sending to the consumerSender. type QueueSettings struct { - // Enabled indicates whether to not enqueue batches before sending to the consumerSender. - Enabled bool `json:"enabled,omitempty"` - // NumConsumers is the number of consumers from the queue. - NumConsumers int `json:"num_consumers,omitempty"` + NumConsumers *int `json:"num_consumers,omitempty"` // QueueSize is the maximum number of batches allowed in queue at a given time. - QueueSize int `json:"queue_size,omitempty"` - - // StorageID if not empty, enables the persistent storage and uses the component specified - // as a storage extension for the persistent queue - StorageID string `json:"storage,omitempty"` //TODO this is *component.ID at Otel + // Default value is 100. + QueueSize *int `json:"queue_size,omitempty"` } // BackOffConfig defines configuration for retrying batches in case of export failure. // The current supported strategy is exponential backoff. type BackOffConfig struct { - // Enabled indicates whether to not retry sending batches in case of export failure. - Enabled bool `json:"enabled,omitempty"` - // InitialInterval the time to wait after the first failure before retrying. - InitialInterval time.Duration `json:"initial_interval,omitempty"` + InitialInterval *time.Duration `json:"initial_interval,omitempty"` // RandomizationFactor is a random factor used to calculate next backoffs // Randomized interval = RetryInterval * (1 ± RandomizationFactor) - RandomizationFactor string `json:"randomization_factor,omitempty"` + RandomizationFactor *string `json:"randomization_factor,omitempty"` // Multiplier is the value multiplied by the backoff interval bounds - Multiplier string `json:"multiplier,omitempty"` + Multiplier *string `json:"multiplier,omitempty"` // MaxInterval is the upper bound on backoff interval. Once this value is reached the delay between // consecutive retries will always be `MaxInterval`. - MaxInterval time.Duration `json:"max_interval,omitempty"` + MaxInterval *time.Duration `json:"max_interval,omitempty"` // MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. // Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. - MaxElapsedTime time.Duration `json:"max_elapsed_time,omitempty"` + // Default value is 0 to ensure that the data is never discarded. + MaxElapsedTime *time.Duration `json:"max_elapsed_time,omitempty"` } // KeepaliveClientConfig exposes the keepalive.ClientParameters to be used by the exporter. diff --git a/api/telemetry/v1alpha1/output_types.go b/api/telemetry/v1alpha1/output_types.go index 4f8a610..ccbf052 100644 --- a/api/telemetry/v1alpha1/output_types.go +++ b/api/telemetry/v1alpha1/output_types.go @@ -99,25 +99,13 @@ type OTLPHTTP struct { HTTPClientConfig `json:",inline"` } -// Configuration for the fluentforward exporter. -type Fluentforward struct { - TCPClientSettings `json:",inline"` - - // RequireAck enables the acknowledgement feature. - RequireAck *bool `json:"require_ack,omitempty"` - - // The Fluent tag parameter used for routing - Tag *string `json:"tag,omitempty"` - - // CompressGzip enables gzip compression for the payload. - CompressGzip *bool `json:"compress_gzip,omitempty"` - - // DefaultLabelsEnabled is a map of default attributes to be added to each log record. - DefaultLabelsEnabled *map[string]bool `json:"default_labels_enabled,omitempty"` - - QueueConfig *QueueSettings `json:"sending_queue,omitempty"` - RetryConfig *BackOffConfig `json:"retry_on_failure,omitempty"` - Kubernetes *KubernetesMetadata `json:"kubernetes_metadata,omitempty"` +type Endpoint struct { + // TCPAddr is the address of the server to connect to. + TCPAddr *string `json:"tcp_addr"` + // Controls whether to validate the tcp address. + // Turning this ON may result in the collector failing to start if it came up faster then the endpoint. + // default is false. + ValidateTCPResolution bool `json:"validate_tcp_resolution"` } type KubernetesMetadata struct { @@ -125,9 +113,12 @@ type KubernetesMetadata struct { IncludePodLabels bool `json:"include_pod_labels"` } +// TCPClientSettings defines common settings for a TCP client. type TCPClientSettings struct { - // The target endpoint URI to send data to (e.g.: some.url:24224). - Endpoint *string `json:"endpoint,omitempty"` + // +kubebuilder:validation:Required + + // Endpoint to send logs to. + *Endpoint `json:"endpoint"` // +kubebuilder:validation:Format=duration @@ -141,6 +132,27 @@ type TCPClientSettings struct { SharedKey *string `json:"shared_key,omitempty"` } +// Configuration for the fluentforward exporter. +type Fluentforward struct { + TCPClientSettings `json:",inline"` + + // RequireAck enables the acknowledgement feature. + RequireAck *bool `json:"require_ack,omitempty"` + + // The Fluent tag parameter used for routing + Tag *string `json:"tag,omitempty"` + + // CompressGzip enables gzip compression for the payload. + CompressGzip *bool `json:"compress_gzip,omitempty"` + + // DefaultLabelsEnabled is a map of default attributes to be added to each log record. + DefaultLabelsEnabled *map[string]bool `json:"default_labels_enabled,omitempty"` + + QueueConfig *QueueSettings `json:"sending_queue,omitempty"` + RetryConfig *BackOffConfig `json:"retry_on_failure,omitempty"` + Kubernetes *KubernetesMetadata `json:"kubernetes_metadata,omitempty"` +} + // OutputStatus defines the observed state of Output type OutputStatus struct { } diff --git a/api/telemetry/v1alpha1/tenant_types.go b/api/telemetry/v1alpha1/tenant_types.go index f84c919..77f5f89 100644 --- a/api/telemetry/v1alpha1/tenant_types.go +++ b/api/telemetry/v1alpha1/tenant_types.go @@ -28,7 +28,7 @@ type TransformStatement struct { Statements []string `json:"statements,omitempty"` } -// Transform represents the Transform processor, which modifies telemetry based on its configuration +// Transform represents the Transform processor, which modifies telemetry based on its configuration. type Transform struct { // Name of the Transform processor Name string `json:"name,omitempty"` @@ -50,7 +50,7 @@ type Transform struct { } // RouteConfig defines the routing configuration for a tenant -// it will be used to generate routing connectors +// it will be used to generate routing connectors. type RouteConfig struct { // Contains the list of pipelines to use when a record does not meet any of specified conditions. DefaultPipelines []string `json:"defaultPipelines,omitempty"` // TODO: Provide users with a guide to determine generated pipeline names @@ -67,17 +67,36 @@ type RouteConfig struct { MatchOnce bool `json:"matchOnce,omitempty"` } +// Configuration for persistence, will be used to generate +// the filestorage extension. +type PersistenceConfig struct { + // Determines whether file storage is enabled or not. + EnableFileStorage bool `json:"enableFileStorage,omitempty"` + + // The directory where logs will be persisted. + // If unset or an invalid path is given, then an OS specific + // default value will be used. + // The cluster administrator must ensure that the directory + // is unique for each tenant. + // If unset /var/lib/otelcol/file_storage/ will be used. + Directory string `json:"directory,omitempty"` +} + // TenantSpec defines the desired state of Tenant type TenantSpec struct { // Determines the namespaces from which subscriptions are collected by this tenant. SubscriptionNamespaceSelectors []metav1.LabelSelector `json:"subscriptionNamespaceSelectors,omitempty"` // Determines the namespaces from which logs are collected by this tenant. - // If initialized with an empty list, logs from all namespaces are collected. - // If uninitialized, no logs are collected. + // Cannot be used together with SelectFromAllNamespaces. LogSourceNamespaceSelectors []metav1.LabelSelector `json:"logSourceNamespaceSelectors,omitempty"` - Transform `json:"transform,omitempty"` - RouteConfig `json:"routeConfig,omitempty"` + + // If true, logs are collected from all namespaces. + // Cannot be used together with LogSourceNamespaceSelectors. + SelectFromAllNamespaces bool `json:"selectFromAllNamespaces,omitempty"` + Transform `json:"transform,omitempty"` + RouteConfig `json:"routeConfig,omitempty"` + PersistenceConfig `json:"persistenceConfig,omitempty"` } // TenantStatus defines the observed state of Tenant diff --git a/api/telemetry/v1alpha1/zz_generated.deepcopy.go b/api/telemetry/v1alpha1/zz_generated.deepcopy.go index 45972c0..08626d4 100644 --- a/api/telemetry/v1alpha1/zz_generated.deepcopy.go +++ b/api/telemetry/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,31 @@ func (in *Authentication) DeepCopy() *Authentication { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackOffConfig) DeepCopyInto(out *BackOffConfig) { *out = *in + if in.InitialInterval != nil { + in, out := &in.InitialInterval, &out.InitialInterval + *out = new(timex.Duration) + **out = **in + } + if in.RandomizationFactor != nil { + in, out := &in.RandomizationFactor, &out.RandomizationFactor + *out = new(string) + **out = **in + } + if in.Multiplier != nil { + in, out := &in.Multiplier, &out.Multiplier + *out = new(string) + **out = **in + } + if in.MaxInterval != nil { + in, out := &in.MaxInterval, &out.MaxInterval + *out = new(timex.Duration) + **out = **in + } + if in.MaxElapsedTime != nil { + in, out := &in.MaxElapsedTime, &out.MaxElapsedTime + *out = new(timex.Duration) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackOffConfig. @@ -317,6 +342,26 @@ func (in *CollectorStatus) DeepCopy() *CollectorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Endpoint) DeepCopyInto(out *Endpoint) { + *out = *in + if in.TCPAddr != nil { + in, out := &in.TCPAddr, &out.TCPAddr + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoint. +func (in *Endpoint) DeepCopy() *Endpoint { + if in == nil { + return nil + } + out := new(Endpoint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Fluentforward) DeepCopyInto(out *Fluentforward) { *out = *in @@ -350,12 +395,12 @@ func (in *Fluentforward) DeepCopyInto(out *Fluentforward) { if in.QueueConfig != nil { in, out := &in.QueueConfig, &out.QueueConfig *out = new(QueueSettings) - **out = **in + (*in).DeepCopyInto(*out) } if in.RetryConfig != nil { in, out := &in.RetryConfig, &out.RetryConfig *out = new(BackOffConfig) - **out = **in + (*in).DeepCopyInto(*out) } if in.Kubernetes != nil { in, out := &in.Kubernetes, &out.Kubernetes @@ -617,12 +662,12 @@ func (in *OTLPGRPC) DeepCopyInto(out *OTLPGRPC) { if in.QueueConfig != nil { in, out := &in.QueueConfig, &out.QueueConfig *out = new(QueueSettings) - **out = **in + (*in).DeepCopyInto(*out) } if in.RetryConfig != nil { in, out := &in.RetryConfig, &out.RetryConfig *out = new(BackOffConfig) - **out = **in + (*in).DeepCopyInto(*out) } in.TimeoutSettings.DeepCopyInto(&out.TimeoutSettings) in.GRPCClientConfig.DeepCopyInto(&out.GRPCClientConfig) @@ -644,12 +689,12 @@ func (in *OTLPHTTP) DeepCopyInto(out *OTLPHTTP) { if in.QueueConfig != nil { in, out := &in.QueueConfig, &out.QueueConfig *out = new(QueueSettings) - **out = **in + (*in).DeepCopyInto(*out) } if in.RetryConfig != nil { in, out := &in.RetryConfig, &out.RetryConfig *out = new(BackOffConfig) - **out = **in + (*in).DeepCopyInto(*out) } in.HTTPClientConfig.DeepCopyInto(&out.HTTPClientConfig) } @@ -803,9 +848,34 @@ func (in *OutputStatus) DeepCopy() *OutputStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PersistenceConfig) DeepCopyInto(out *PersistenceConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PersistenceConfig. +func (in *PersistenceConfig) DeepCopy() *PersistenceConfig { + if in == nil { + return nil + } + out := new(PersistenceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueSettings) DeepCopyInto(out *QueueSettings) { *out = *in + if in.NumConsumers != nil { + in, out := &in.NumConsumers, &out.NumConsumers + *out = new(int) + **out = **in + } + if in.QueueSize != nil { + in, out := &in.QueueSize, &out.QueueSize + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueSettings. @@ -942,8 +1012,8 @@ func (in *TCPClientSettings) DeepCopyInto(out *TCPClientSettings) { *out = *in if in.Endpoint != nil { in, out := &in.Endpoint, &out.Endpoint - *out = new(string) - **out = **in + *out = new(Endpoint) + (*in).DeepCopyInto(*out) } if in.ConnectionTimeout != nil { in, out := &in.ConnectionTimeout, &out.ConnectionTimeout @@ -1081,6 +1151,7 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { } in.Transform.DeepCopyInto(&out.Transform) in.RouteConfig.DeepCopyInto(&out.RouteConfig) + out.PersistenceConfig = in.PersistenceConfig } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantSpec. diff --git a/charts/telemetry-controller/crds/telemetry.kube-logging.dev_outputs.yaml b/charts/telemetry-controller/crds/telemetry.kube-logging.dev_outputs.yaml index 4b3d4dd..bf3c85e 100644 --- a/charts/telemetry-controller/crds/telemetry.kube-logging.dev_outputs.yaml +++ b/charts/telemetry-controller/crds/telemetry.kube-logging.dev_outputs.yaml @@ -150,8 +150,22 @@ spec: to be added to each log record. type: object endpoint: - description: 'The target endpoint URI to send data to (e.g.: some.url:24224).' - type: string + description: Endpoint to send logs to. + properties: + tcp_addr: + description: TCPAddr is the address of the server to connect + to. + type: string + validate_tcp_resolution: + description: |- + Controls whether to validate the tcp address. + Turning this ON may result in the collector failing to start if it came up faster then the endpoint. + default is false. + type: boolean + required: + - tcp_addr + - validate_tcp_resolution + type: object kubernetes_metadata: properties: include_pod_labels: @@ -170,10 +184,6 @@ spec: BackOffConfig defines configuration for retrying batches in case of export failure. The current supported strategy is exponential backoff. properties: - enabled: - description: Enabled indicates whether to not retry sending - batches in case of export failure. - type: boolean initial_interval: description: InitialInterval the time to wait after the first failure before retrying. @@ -183,6 +193,7 @@ spec: description: |- MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + Default value is 0 to ensure that the data is never discarded. format: int64 type: integer max_interval: @@ -205,23 +216,15 @@ spec: description: QueueSettings defines configuration for queueing batches before sending to the consumerSender. properties: - enabled: - description: Enabled indicates whether to not enqueue batches - before sending to the consumerSender. - type: boolean num_consumers: description: NumConsumers is the number of consumers from the queue. type: integer queue_size: - description: QueueSize is the maximum number of batches allowed - in queue at a given time. - type: integer - storage: description: |- - StorageID if not empty, enables the persistent storage and uses the component specified - as a storage extension for the persistent queue - type: string + QueueSize is the maximum number of batches allowed in queue at a given time. + Default value is 100. + type: integer type: object shared_key: description: SharedKey is used for authorization with the server @@ -294,6 +297,8 @@ spec: https://godoc.org/crypto/tls#Config for more information. (optional) type: string type: object + required: + - endpoint type: object otlp: description: |- @@ -365,10 +370,6 @@ spec: BackOffConfig defines configuration for retrying batches in case of export failure. The current supported strategy is exponential backoff. properties: - enabled: - description: Enabled indicates whether to not retry sending - batches in case of export failure. - type: boolean initial_interval: description: InitialInterval the time to wait after the first failure before retrying. @@ -378,6 +379,7 @@ spec: description: |- MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + Default value is 0 to ensure that the data is never discarded. format: int64 type: integer max_interval: @@ -400,23 +402,15 @@ spec: description: QueueSettings defines configuration for queueing batches before sending to the consumerSender. properties: - enabled: - description: Enabled indicates whether to not enqueue batches - before sending to the consumerSender. - type: boolean num_consumers: description: NumConsumers is the number of consumers from the queue. type: integer queue_size: - description: QueueSize is the maximum number of batches allowed - in queue at a given time. - type: integer - storage: description: |- - StorageID if not empty, enables the persistent storage and uses the component specified - as a storage extension for the persistent queue - type: string + QueueSize is the maximum number of batches allowed in queue at a given time. + Default value is 100. + type: integer type: object timeout: description: |- @@ -587,10 +581,6 @@ spec: BackOffConfig defines configuration for retrying batches in case of export failure. The current supported strategy is exponential backoff. properties: - enabled: - description: Enabled indicates whether to not retry sending - batches in case of export failure. - type: boolean initial_interval: description: InitialInterval the time to wait after the first failure before retrying. @@ -600,6 +590,7 @@ spec: description: |- MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + Default value is 0 to ensure that the data is never discarded. format: int64 type: integer max_interval: @@ -622,23 +613,15 @@ spec: description: QueueSettings defines configuration for queueing batches before sending to the consumerSender. properties: - enabled: - description: Enabled indicates whether to not enqueue batches - before sending to the consumerSender. - type: boolean num_consumers: description: NumConsumers is the number of consumers from the queue. type: integer queue_size: - description: QueueSize is the maximum number of batches allowed - in queue at a given time. - type: integer - storage: description: |- - StorageID if not empty, enables the persistent storage and uses the component specified - as a storage extension for the persistent queue - type: string + QueueSize is the maximum number of batches allowed in queue at a given time. + Default value is 100. + type: integer type: object timeout: description: Timeout parameter configures `http.Client.Timeout`. diff --git a/charts/telemetry-controller/crds/telemetry.kube-logging.dev_tenants.yaml b/charts/telemetry-controller/crds/telemetry.kube-logging.dev_tenants.yaml index 53a5585..2a8c059 100644 --- a/charts/telemetry-controller/crds/telemetry.kube-logging.dev_tenants.yaml +++ b/charts/telemetry-controller/crds/telemetry.kube-logging.dev_tenants.yaml @@ -57,8 +57,7 @@ spec: logSourceNamespaceSelectors: description: |- Determines the namespaces from which logs are collected by this tenant. - If initialized with an empty list, logs from all namespaces are collected. - If uninitialized, no logs are collected. + Cannot be used together with SelectFromAllNamespaces. items: description: |- A label selector is a label query over a set of resources. The result of matchLabels and @@ -109,10 +108,28 @@ spec: type: object x-kubernetes-map-type: atomic type: array + persistenceConfig: + description: |- + Configuration for persistence, will be used to generate + the filestorage extension. + properties: + directory: + description: |- + The directory where logs will be persisted. + If unset or an invalid path is given, then an OS specific + default value will be used. + The cluster administrator must ensure that the directory + is unique for each tenant. + If unset /var/lib/otelcol/file_storage/ will be used. + type: string + enableFileStorage: + description: Determines whether file storage is enabled or not. + type: boolean + type: object routeConfig: description: |- RouteConfig defines the routing configuration for a tenant - it will be used to generate routing connectors + it will be used to generate routing connectors. properties: defaultPipelines: description: Contains the list of pipelines to use when a record @@ -136,6 +153,11 @@ spec: May only be false when used with resource context. type: boolean type: object + selectFromAllNamespaces: + description: |- + If true, logs are collected from all namespaces. + Cannot be used together with LogSourceNamespaceSelectors. + type: boolean subscriptionNamespaceSelectors: description: Determines the namespaces from which subscriptions are collected by this tenant. @@ -191,7 +213,7 @@ spec: type: array transform: description: Transform represents the Transform processor, which modifies - telemetry based on its configuration + telemetry based on its configuration. properties: errorMode: description: |- diff --git a/config/crd/bases/telemetry.kube-logging.dev_outputs.yaml b/config/crd/bases/telemetry.kube-logging.dev_outputs.yaml index 4b3d4dd..bf3c85e 100644 --- a/config/crd/bases/telemetry.kube-logging.dev_outputs.yaml +++ b/config/crd/bases/telemetry.kube-logging.dev_outputs.yaml @@ -150,8 +150,22 @@ spec: to be added to each log record. type: object endpoint: - description: 'The target endpoint URI to send data to (e.g.: some.url:24224).' - type: string + description: Endpoint to send logs to. + properties: + tcp_addr: + description: TCPAddr is the address of the server to connect + to. + type: string + validate_tcp_resolution: + description: |- + Controls whether to validate the tcp address. + Turning this ON may result in the collector failing to start if it came up faster then the endpoint. + default is false. + type: boolean + required: + - tcp_addr + - validate_tcp_resolution + type: object kubernetes_metadata: properties: include_pod_labels: @@ -170,10 +184,6 @@ spec: BackOffConfig defines configuration for retrying batches in case of export failure. The current supported strategy is exponential backoff. properties: - enabled: - description: Enabled indicates whether to not retry sending - batches in case of export failure. - type: boolean initial_interval: description: InitialInterval the time to wait after the first failure before retrying. @@ -183,6 +193,7 @@ spec: description: |- MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + Default value is 0 to ensure that the data is never discarded. format: int64 type: integer max_interval: @@ -205,23 +216,15 @@ spec: description: QueueSettings defines configuration for queueing batches before sending to the consumerSender. properties: - enabled: - description: Enabled indicates whether to not enqueue batches - before sending to the consumerSender. - type: boolean num_consumers: description: NumConsumers is the number of consumers from the queue. type: integer queue_size: - description: QueueSize is the maximum number of batches allowed - in queue at a given time. - type: integer - storage: description: |- - StorageID if not empty, enables the persistent storage and uses the component specified - as a storage extension for the persistent queue - type: string + QueueSize is the maximum number of batches allowed in queue at a given time. + Default value is 100. + type: integer type: object shared_key: description: SharedKey is used for authorization with the server @@ -294,6 +297,8 @@ spec: https://godoc.org/crypto/tls#Config for more information. (optional) type: string type: object + required: + - endpoint type: object otlp: description: |- @@ -365,10 +370,6 @@ spec: BackOffConfig defines configuration for retrying batches in case of export failure. The current supported strategy is exponential backoff. properties: - enabled: - description: Enabled indicates whether to not retry sending - batches in case of export failure. - type: boolean initial_interval: description: InitialInterval the time to wait after the first failure before retrying. @@ -378,6 +379,7 @@ spec: description: |- MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + Default value is 0 to ensure that the data is never discarded. format: int64 type: integer max_interval: @@ -400,23 +402,15 @@ spec: description: QueueSettings defines configuration for queueing batches before sending to the consumerSender. properties: - enabled: - description: Enabled indicates whether to not enqueue batches - before sending to the consumerSender. - type: boolean num_consumers: description: NumConsumers is the number of consumers from the queue. type: integer queue_size: - description: QueueSize is the maximum number of batches allowed - in queue at a given time. - type: integer - storage: description: |- - StorageID if not empty, enables the persistent storage and uses the component specified - as a storage extension for the persistent queue - type: string + QueueSize is the maximum number of batches allowed in queue at a given time. + Default value is 100. + type: integer type: object timeout: description: |- @@ -587,10 +581,6 @@ spec: BackOffConfig defines configuration for retrying batches in case of export failure. The current supported strategy is exponential backoff. properties: - enabled: - description: Enabled indicates whether to not retry sending - batches in case of export failure. - type: boolean initial_interval: description: InitialInterval the time to wait after the first failure before retrying. @@ -600,6 +590,7 @@ spec: description: |- MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + Default value is 0 to ensure that the data is never discarded. format: int64 type: integer max_interval: @@ -622,23 +613,15 @@ spec: description: QueueSettings defines configuration for queueing batches before sending to the consumerSender. properties: - enabled: - description: Enabled indicates whether to not enqueue batches - before sending to the consumerSender. - type: boolean num_consumers: description: NumConsumers is the number of consumers from the queue. type: integer queue_size: - description: QueueSize is the maximum number of batches allowed - in queue at a given time. - type: integer - storage: description: |- - StorageID if not empty, enables the persistent storage and uses the component specified - as a storage extension for the persistent queue - type: string + QueueSize is the maximum number of batches allowed in queue at a given time. + Default value is 100. + type: integer type: object timeout: description: Timeout parameter configures `http.Client.Timeout`. diff --git a/config/crd/bases/telemetry.kube-logging.dev_tenants.yaml b/config/crd/bases/telemetry.kube-logging.dev_tenants.yaml index 53a5585..2a8c059 100644 --- a/config/crd/bases/telemetry.kube-logging.dev_tenants.yaml +++ b/config/crd/bases/telemetry.kube-logging.dev_tenants.yaml @@ -57,8 +57,7 @@ spec: logSourceNamespaceSelectors: description: |- Determines the namespaces from which logs are collected by this tenant. - If initialized with an empty list, logs from all namespaces are collected. - If uninitialized, no logs are collected. + Cannot be used together with SelectFromAllNamespaces. items: description: |- A label selector is a label query over a set of resources. The result of matchLabels and @@ -109,10 +108,28 @@ spec: type: object x-kubernetes-map-type: atomic type: array + persistenceConfig: + description: |- + Configuration for persistence, will be used to generate + the filestorage extension. + properties: + directory: + description: |- + The directory where logs will be persisted. + If unset or an invalid path is given, then an OS specific + default value will be used. + The cluster administrator must ensure that the directory + is unique for each tenant. + If unset /var/lib/otelcol/file_storage/ will be used. + type: string + enableFileStorage: + description: Determines whether file storage is enabled or not. + type: boolean + type: object routeConfig: description: |- RouteConfig defines the routing configuration for a tenant - it will be used to generate routing connectors + it will be used to generate routing connectors. properties: defaultPipelines: description: Contains the list of pipelines to use when a record @@ -136,6 +153,11 @@ spec: May only be false when used with resource context. type: boolean type: object + selectFromAllNamespaces: + description: |- + If true, logs are collected from all namespaces. + Cannot be used together with LogSourceNamespaceSelectors. + type: boolean subscriptionNamespaceSelectors: description: Determines the namespaces from which subscriptions are collected by this tenant. @@ -191,7 +213,7 @@ spec: type: array transform: description: Transform represents the Transform processor, which modifies - telemetry based on its configuration + telemetry based on its configuration. properties: errorMode: description: |- diff --git a/go.mod b/go.mod index f519e11..0caa2b7 100644 --- a/go.mod +++ b/go.mod @@ -7,23 +7,20 @@ require ( github.com/cisco-open/operator-tools v0.37.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/imdario/mergo v0.3.16 github.com/mitchellh/mapstructure v1.5.0 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.0 github.com/open-telemetry/opentelemetry-operator v0.114.0 github.com/prometheus/client_golang v1.20.5 - github.com/siliconbrain/go-mapseqs v0.4.0 - github.com/siliconbrain/go-seqs v0.13.0 github.com/stretchr/testify v1.10.0 - go.opentelemetry.io/collector/component v0.114.0 - go.opentelemetry.io/collector/config/configauth v0.114.0 - go.opentelemetry.io/collector/config/configcompression v1.20.0 - go.opentelemetry.io/collector/config/configopaque v1.20.0 - go.opentelemetry.io/collector/config/configtelemetry v0.114.0 - go.opentelemetry.io/collector/otelcol v0.114.0 - go.opentelemetry.io/collector/pipeline v0.114.0 - go.opentelemetry.io/collector/service v0.114.0 + go.opentelemetry.io/collector/component v0.115.0 + go.opentelemetry.io/collector/config/configauth v0.115.0 + go.opentelemetry.io/collector/config/configcompression v1.21.0 + go.opentelemetry.io/collector/config/configopaque v1.21.0 + go.opentelemetry.io/collector/config/configtelemetry v0.115.0 + go.opentelemetry.io/collector/otelcol v0.115.0 + go.opentelemetry.io/collector/pipeline v0.115.0 + go.opentelemetry.io/collector/service v0.115.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f k8s.io/api v0.31.3 @@ -68,6 +65,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -91,7 +89,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/shirou/gopsutil/v4 v4.24.10 // indirect + github.com/shirou/gopsutil/v4 v4.24.11 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -100,36 +98,36 @@ require ( github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.opentelemetry.io/collector/component/componentstatus v0.114.0 // indirect - go.opentelemetry.io/collector/component/componenttest v0.114.0 // indirect - go.opentelemetry.io/collector/confmap v1.20.0 // indirect - go.opentelemetry.io/collector/connector v0.114.0 // indirect - go.opentelemetry.io/collector/connector/connectorprofiles v0.114.0 // indirect - go.opentelemetry.io/collector/connector/connectortest v0.114.0 // indirect - go.opentelemetry.io/collector/consumer v0.114.0 // indirect - go.opentelemetry.io/collector/consumer/consumererror v0.114.0 // indirect - go.opentelemetry.io/collector/consumer/consumerprofiles v0.114.0 // indirect - go.opentelemetry.io/collector/consumer/consumertest v0.114.0 // indirect - go.opentelemetry.io/collector/exporter v0.114.0 // indirect - go.opentelemetry.io/collector/exporter/exporterprofiles v0.114.0 // indirect - go.opentelemetry.io/collector/exporter/exportertest v0.114.0 // indirect - go.opentelemetry.io/collector/extension v0.114.0 // indirect - go.opentelemetry.io/collector/extension/auth v0.114.0 // indirect - go.opentelemetry.io/collector/extension/extensioncapabilities v0.114.0 // indirect - go.opentelemetry.io/collector/extension/extensiontest v0.114.0 // indirect - go.opentelemetry.io/collector/featuregate v1.20.0 // indirect - go.opentelemetry.io/collector/internal/fanoutconsumer v0.114.0 // indirect - go.opentelemetry.io/collector/pdata v1.20.0 // indirect - go.opentelemetry.io/collector/pdata/pprofile v0.114.0 // indirect - go.opentelemetry.io/collector/pdata/testdata v0.114.0 // indirect - go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.114.0 // indirect - go.opentelemetry.io/collector/processor v0.114.0 // indirect - go.opentelemetry.io/collector/processor/processorprofiles v0.114.0 // indirect - go.opentelemetry.io/collector/processor/processortest v0.114.0 // indirect - go.opentelemetry.io/collector/receiver v0.114.0 // indirect - go.opentelemetry.io/collector/receiver/receiverprofiles v0.114.0 // indirect - go.opentelemetry.io/collector/receiver/receivertest v0.114.0 // indirect - go.opentelemetry.io/collector/semconv v0.114.0 // indirect + go.opentelemetry.io/collector/component/componentstatus v0.115.0 // indirect + go.opentelemetry.io/collector/component/componenttest v0.115.0 // indirect + go.opentelemetry.io/collector/confmap v1.21.0 // indirect + go.opentelemetry.io/collector/connector v0.115.0 // indirect + go.opentelemetry.io/collector/connector/connectorprofiles v0.115.0 // indirect + go.opentelemetry.io/collector/connector/connectortest v0.115.0 // indirect + go.opentelemetry.io/collector/consumer v1.21.0 // indirect + go.opentelemetry.io/collector/consumer/consumererror v0.115.0 // indirect + go.opentelemetry.io/collector/consumer/consumerprofiles v0.115.0 // indirect + go.opentelemetry.io/collector/consumer/consumertest v0.115.0 // indirect + go.opentelemetry.io/collector/exporter v0.115.0 // indirect + go.opentelemetry.io/collector/exporter/exporterprofiles v0.115.0 // indirect + go.opentelemetry.io/collector/exporter/exportertest v0.115.0 // indirect + go.opentelemetry.io/collector/extension v0.115.0 // indirect + go.opentelemetry.io/collector/extension/auth v0.115.0 // indirect + go.opentelemetry.io/collector/extension/extensioncapabilities v0.115.0 // indirect + go.opentelemetry.io/collector/extension/extensiontest v0.115.0 // indirect + go.opentelemetry.io/collector/featuregate v1.21.0 // indirect + go.opentelemetry.io/collector/internal/fanoutconsumer v0.115.0 // indirect + go.opentelemetry.io/collector/pdata v1.21.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.115.0 // indirect + go.opentelemetry.io/collector/pdata/testdata v0.115.0 // indirect + go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.115.0 // indirect + go.opentelemetry.io/collector/processor v0.115.0 // indirect + go.opentelemetry.io/collector/processor/processorprofiles v0.115.0 // indirect + go.opentelemetry.io/collector/processor/processortest v0.115.0 // indirect + go.opentelemetry.io/collector/receiver v0.115.0 // indirect + go.opentelemetry.io/collector/receiver/receiverprofiles v0.115.0 // indirect + go.opentelemetry.io/collector/receiver/receivertest v0.115.0 // indirect + go.opentelemetry.io/collector/semconv v0.115.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.6.0 // indirect go.opentelemetry.io/contrib/config v0.10.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.31.0 // indirect @@ -164,7 +162,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 3812fa4..60805e8 100644 --- a/go.sum +++ b/go.sum @@ -306,12 +306,8 @@ github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= -github.com/shirou/gopsutil/v4 v4.24.10 h1:7VOzPtfw/5YDU+jLEoBwXwxJbQetULywoSV4RYY7HkM= -github.com/shirou/gopsutil/v4 v4.24.10/go.mod h1:s4D/wg+ag4rG0WO7AiTj2BeYCRhym0vM7DHbZRxnIT8= -github.com/siliconbrain/go-mapseqs v0.4.0 h1:r5GL26cEOiCtGHIVu1EIDMT1iQI5zNYZMoDjeoeRf+I= -github.com/siliconbrain/go-mapseqs v0.4.0/go.mod h1:WtBy7yxv/TS3Vc/WCro59kuYdRTqUnEr8OkgC9Rkllg= -github.com/siliconbrain/go-seqs v0.13.0 h1:3xvOlPDs4FB/XZzmagMU/DTTbHhLcEVPsjauKHsT990= -github.com/siliconbrain/go-seqs v0.13.0/go.mod h1:8kptI0u8epC1dPx/nCo3fUM2xJnfAoo59/UJiCsxkR4= +github.com/shirou/gopsutil/v4 v4.24.11 h1:WaU9xqGFKvFfsUv94SXcUPD7rCkU0vr/asVdQOBZNj8= +github.com/shirou/gopsutil/v4 v4.24.11/go.mod h1:s4D/wg+ag4rG0WO7AiTj2BeYCRhym0vM7DHbZRxnIT8= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= @@ -338,98 +334,98 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/collector v0.114.0 h1:XLLLOHns06P9XjVHyp0OdEMdwXvol5MLzugqQMmXYuU= -go.opentelemetry.io/collector v0.114.0/go.mod h1:XbjD4Yw9LunLo3IJu3ZZytNZ0drEVznxw1Z14Ujlw3s= -go.opentelemetry.io/collector/client v1.20.0 h1:o60wPcj5nLtaRenF+1E5p4QXFS3TDL6vHlw+GOon3rg= -go.opentelemetry.io/collector/client v1.20.0/go.mod h1:6aqkszco9FaLWCxyJEVam6PP7cUa8mPRIXeS5eZGj0U= -go.opentelemetry.io/collector/component v0.114.0 h1:SVGbm5LvHGSTEDv7p92oPuBgK5tuiWR82I9+LL4TtBE= -go.opentelemetry.io/collector/component v0.114.0/go.mod h1:MLxtjZ6UVHjDxSdhGLuJfHBHvfl1iT/Y7IaQPD24Eww= -go.opentelemetry.io/collector/component/componentstatus v0.114.0 h1:y9my/xink8KB5lK8zFAjgB2+pEh0QYy5TM972fxZY9w= -go.opentelemetry.io/collector/component/componentstatus v0.114.0/go.mod h1:RIoeCYZpPaae7QLE/1RacqzhHuXBmzRAk9H/EwYtIIs= -go.opentelemetry.io/collector/component/componenttest v0.114.0 h1:GM4FTTlfeXoVm6sZYBHImwlRN8ayh2oAfUhvaFj7Zo8= -go.opentelemetry.io/collector/component/componenttest v0.114.0/go.mod h1:ZZEJMtbJtoVC/3/9R1HzERq+cYQRxuMFQrPCpfZ4Xos= -go.opentelemetry.io/collector/config/configauth v0.114.0 h1:R2sJ6xpsLYGH0yU0vCxotzBYDKR/Hrjv0A7y9lwMyiw= -go.opentelemetry.io/collector/config/configauth v0.114.0/go.mod h1:3Z24KcCpG+WYCeQYfs/cNp5cP2BDeOqLCtOEgs/rPqM= -go.opentelemetry.io/collector/config/configcompression v1.20.0 h1:H/mvz7J/5z+O74YsO0t2tk+REnO2tzLM8TgIQ4AZ5w0= -go.opentelemetry.io/collector/config/configcompression v1.20.0/go.mod h1:pnxkFCLUZLKWzYJvfSwZnPrnm0twX14CYj2ADth5xiU= -go.opentelemetry.io/collector/config/confighttp v0.114.0 h1:DjGsBvVm+mGK3IpJBaXianWhwcxEC1fF33cpuC1LY/I= -go.opentelemetry.io/collector/config/confighttp v0.114.0/go.mod h1:nrlNLxOZ+4JQaV9j0TiqQV7LOHhrRivPrT8nQRHED3Q= -go.opentelemetry.io/collector/config/configopaque v1.20.0 h1:2I48zKiyyyYqjm7y0B9OLp24ku2ZSX3nCHG0r5FdWOQ= -go.opentelemetry.io/collector/config/configopaque v1.20.0/go.mod h1:6zlLIyOoRpJJ+0bEKrlZOZon3rOp5Jrz9fMdR4twOS4= -go.opentelemetry.io/collector/config/configretry v1.20.0 h1:z679mrMlW2a6tOOYPGdrS/QfALxdzWLQUOpH8Uu+D5Y= -go.opentelemetry.io/collector/config/configretry v1.20.0/go.mod h1:KvQF5cfphq1rQm1dKR4eLDNQYw6iI2fY72NMZVa+0N0= -go.opentelemetry.io/collector/config/configtelemetry v0.114.0 h1:kjLeyrumge6wsX6ZIkicdNOlBXaEyW2PI2ZdVXz/rzY= -go.opentelemetry.io/collector/config/configtelemetry v0.114.0/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= -go.opentelemetry.io/collector/config/configtls v1.20.0 h1:hNlJdwfyY5Qe54RLJ41lfLqKTn9ypkR7sk7JNCcSe2U= -go.opentelemetry.io/collector/config/configtls v1.20.0/go.mod h1:sav/txSHguadTYlSSK+BJO2ljJeYEtRoBahgzWAguYg= -go.opentelemetry.io/collector/config/internal v0.114.0 h1:uWSDWTJb8T6xRjKD9/XmEARakXnxgYVYKUeId78hErc= -go.opentelemetry.io/collector/config/internal v0.114.0/go.mod h1:yC7E4h1Uj0SubxcFImh6OvBHFTjMh99+A5PuyIgDWqc= -go.opentelemetry.io/collector/confmap v1.20.0 h1:ARfOwmkKxFOud1njl03yAHQ30+uenlzqCO6LBYamDTE= -go.opentelemetry.io/collector/confmap v1.20.0/go.mod h1:DMpd9Ay/ffls3JoQBQ73vWeRsz1rNuLbwjo6WtjSQus= -go.opentelemetry.io/collector/connector v0.114.0 h1:efGAeTCtg8zp5Hyd7Am8kBUgsSxWEFlFtAu4OX4mcEA= -go.opentelemetry.io/collector/connector v0.114.0/go.mod h1:8DhGgD8RhkF0ooELl4NOPPD/wgHuXHmY7g90RwJ2eNo= -go.opentelemetry.io/collector/connector/connectorprofiles v0.114.0 h1:uVs9gy3UfQBmH0636ouIbGIoWis7zmKN+ny4XOGm36U= -go.opentelemetry.io/collector/connector/connectorprofiles v0.114.0/go.mod h1:X681qFEAsEPMDQ0pMsD/3DqePw58sfLew1QbBKvGnmw= -go.opentelemetry.io/collector/connector/connectortest v0.114.0 h1:Fpy1JHyNOLdVzNcmxUY6Jwxdz6JDcTXL1NCfw8k1AOc= -go.opentelemetry.io/collector/connector/connectortest v0.114.0/go.mod h1:1zaAlODuL9iNyfyjHQeGCpbcaUjf5b68t8LqlwHKBNA= -go.opentelemetry.io/collector/consumer v0.114.0 h1:1zVaHvfIZowGwZRitRBRo3i+RP2StlU+GClYiofSw0Q= -go.opentelemetry.io/collector/consumer v0.114.0/go.mod h1:d+Mrzt9hsH1ub3zmwSlnQVPLeTYir4Mgo7CrWfnncN4= -go.opentelemetry.io/collector/consumer/consumererror v0.114.0 h1:r2YiELfWerb40FHD23V04gNjIkLUcjEKGxI4Vtm2iO4= -go.opentelemetry.io/collector/consumer/consumererror v0.114.0/go.mod h1:MzIrLQ5jptO2egypolhlAbZsWZr29WC4FhSxQjnxcvg= -go.opentelemetry.io/collector/consumer/consumerprofiles v0.114.0 h1:5pXYy3E6UK5Huu3aQbsYL8B6E6MyWx4fvXXDn+oXZaA= -go.opentelemetry.io/collector/consumer/consumerprofiles v0.114.0/go.mod h1:PMq3f54KcJQO4v1tue0QxQScu7REFVADlXxXSAYMiN0= -go.opentelemetry.io/collector/consumer/consumertest v0.114.0 h1:isaTwJK5DOy8Bs7GuLq23ejfgj8gLIo5dOUvkRnLF4g= -go.opentelemetry.io/collector/consumer/consumertest v0.114.0/go.mod h1:GNeLPkfRPdh06n/Rv1UKa/cAtCKjN0a7ADyHjIj4HFE= -go.opentelemetry.io/collector/exporter v0.114.0 h1:5/0BBpXuCJQSQ5SQf31g7j6T4XEKkyx9mZMcA2rS5e8= -go.opentelemetry.io/collector/exporter v0.114.0/go.mod h1:atpd0wWXgh5LAZ0REU/d/Ti/q50HDfnlBIjMjJQlKFg= -go.opentelemetry.io/collector/exporter/exporterprofiles v0.114.0 h1:/wmWOSBHcvtz3Pbv7+rWCqPPQuNvYaoidKKaOqZsLKs= -go.opentelemetry.io/collector/exporter/exporterprofiles v0.114.0/go.mod h1:epRYTkyJZTSQZBJflMGHUyUo2EdLPhsaZEyo5Qm848A= -go.opentelemetry.io/collector/exporter/exportertest v0.114.0 h1:vo0idBJT+QACSM1KpjVLm9VeiXVwO7y4UnMpGxN6EyM= -go.opentelemetry.io/collector/exporter/exportertest v0.114.0/go.mod h1:420ssFrhaphneybstbMeSIiqSRoaBARPgO71O17foaM= -go.opentelemetry.io/collector/extension v0.114.0 h1:9Qb92y8hD2WDC5aMDoj4JNQN+/5BQYJWPUPzLXX+iGw= -go.opentelemetry.io/collector/extension v0.114.0/go.mod h1:Yk2/1ptVgfTr12t+22v93nYJpioP14pURv2YercSzU0= -go.opentelemetry.io/collector/extension/auth v0.114.0 h1:1K2qh4yvG8kKR/sTAobI/rw5VxzPZoKcl3FmC195vvo= -go.opentelemetry.io/collector/extension/auth v0.114.0/go.mod h1:IjtsG+jUVJB0utKF8dAK8pLutRun3aEgASshImzsw/U= -go.opentelemetry.io/collector/extension/experimental/storage v0.114.0 h1:hLyX9UvmY0t6iBnk3CqvyNck2U0QjPACekj7pDRx2hA= -go.opentelemetry.io/collector/extension/experimental/storage v0.114.0/go.mod h1:WqYRQVJjJLE1rm+y/ks1wPdPRGWePEvE1VO07xm2J2k= -go.opentelemetry.io/collector/extension/extensioncapabilities v0.114.0 h1:3OHll7gp5XIu7FVgon+koShPy797nze6EjCDokFUG7w= -go.opentelemetry.io/collector/extension/extensioncapabilities v0.114.0/go.mod h1:f0KdeLmE2jWVBmJ1U4WmfAtz1/tQUErGPfhPLKCQ49U= -go.opentelemetry.io/collector/extension/extensiontest v0.114.0 h1:ibXDms1qrswlvlR6b3d2BeyI8sXUXoFV11yOi9Sop8o= -go.opentelemetry.io/collector/extension/extensiontest v0.114.0/go.mod h1:/bOYmqu5yTDfI1bJZUxFqm8ZtmcodpquebiSxiQxtDY= -go.opentelemetry.io/collector/extension/zpagesextension v0.114.0 h1:JosqAcdWw7IGsURJNR8f17xmaGCQEtKhQt9tM0T/DEg= -go.opentelemetry.io/collector/extension/zpagesextension v0.114.0/go.mod h1:+VO4p2GZvmIMqCVyIfS/U85Xqg+HIOe+mdl/ya+jVTE= -go.opentelemetry.io/collector/featuregate v1.20.0 h1:Mi7nMy/q52eruI+6jWnMKUOeM55XvwoPnGcdB1++O8c= -go.opentelemetry.io/collector/featuregate v1.20.0/go.mod h1:47xrISO71vJ83LSMm8+yIDsUbKktUp48Ovt7RR6VbRs= -go.opentelemetry.io/collector/internal/fanoutconsumer v0.114.0 h1:JM9huYqy5LTzmuxQmbPST3l5Ez5kJNit28c6WFWls34= -go.opentelemetry.io/collector/internal/fanoutconsumer v0.114.0/go.mod h1:V28tDU4Wvf1PfW1Ty/SBL9tpKul2iYGno/HkCWGDrj0= -go.opentelemetry.io/collector/otelcol v0.114.0 h1:d/nmYc+adzZ70g4zBMtgujGHVNulF59ExCpuM/3ZKV4= -go.opentelemetry.io/collector/otelcol v0.114.0/go.mod h1:DGmFFao5jHSwD6G1HjUjs0CYcyrTau+u7GjTRUGKN+4= -go.opentelemetry.io/collector/pdata v1.20.0 h1:ePcwt4bdtISP0loHaE+C9xYoU2ZkIvWv89Fob16o9SM= -go.opentelemetry.io/collector/pdata v1.20.0/go.mod h1:Ox1YVLe87cZDB/TL30i4SUz1cA5s6AM6SpFMfY61ICs= -go.opentelemetry.io/collector/pdata/pprofile v0.114.0 h1:pUNfTzsI/JUTiE+DScDM4lsrPoxnVNLI2fbTxR/oapo= -go.opentelemetry.io/collector/pdata/pprofile v0.114.0/go.mod h1:4aNcj6WM1n1uXyFSXlhVs4ibrERgNYsTbzcYI2zGhxA= -go.opentelemetry.io/collector/pdata/testdata v0.114.0 h1:+AzszWSL1i4K6meQ8rU0JDDW55SYCXa6FVqfDixhhTo= -go.opentelemetry.io/collector/pdata/testdata v0.114.0/go.mod h1:bv8XFdCTZxG2MQB5l9dKxSxf5zBrcodwO6JOy1+AxXM= -go.opentelemetry.io/collector/pipeline v0.114.0 h1:v3YOhc5z0tD6QbO5n/pnftpIeroihM2ks9Z2yKPCcwY= -go.opentelemetry.io/collector/pipeline v0.114.0/go.mod h1:4vOvjVsoYTHVGTbfFwqfnQOSV2K3RKUHofh3jNRc2Mg= -go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.114.0 h1:LZgxMQ2zXcz8ILBefhxpZBpn/Rx+TJTncIIQy0LgtgY= -go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.114.0/go.mod h1:bmyqQCJWcA53/GtqZJ2ahwmLdFl6UelFH2nR6OJFqpw= -go.opentelemetry.io/collector/processor v0.114.0 h1:6bqQgLL7BtKrNv4YkEOGjZfkcfZv/ciJSQx1epGG9Zk= -go.opentelemetry.io/collector/processor v0.114.0/go.mod h1:DV/wa+nAmSHIDeD9NblPwkY9PbgtDQAZJ+PE5biZwPc= -go.opentelemetry.io/collector/processor/processorprofiles v0.114.0 h1:+P/1nLouEXTnN8DVQl+qWwO4BTkQyNPG9t/FrpUqrSI= -go.opentelemetry.io/collector/processor/processorprofiles v0.114.0/go.mod h1:3fuHeNIpINwx3bqFMprmDJyr6y5tWoWbJH599kltO5Y= -go.opentelemetry.io/collector/processor/processortest v0.114.0 h1:3FTaVXAp0LoVmUJn1ewBFckAby7AHa6/Kcdj0xuW14c= -go.opentelemetry.io/collector/processor/processortest v0.114.0/go.mod h1:OgsdOs1Fv5ZGTTJPF5nNIUJh2YkuV1acWd73yWgnti4= -go.opentelemetry.io/collector/receiver v0.114.0 h1:90SAnXAjNq7/k52/pFmmb06Cf1YauoPYtbio4aOXafY= -go.opentelemetry.io/collector/receiver v0.114.0/go.mod h1:KUGT0/D953LXbGH/D3lLPU8yrU3HfWnUqpt4W4hSOnE= -go.opentelemetry.io/collector/receiver/receiverprofiles v0.114.0 h1:ibhEfGpvNB3yrtpl2jYFabrunMk1hurxvMYpM0b1Ck4= -go.opentelemetry.io/collector/receiver/receiverprofiles v0.114.0/go.mod h1:UZyRfaasw+NLvN10AN8IQnmj5tQ3BOUH1uP2ctpO9f0= -go.opentelemetry.io/collector/receiver/receivertest v0.114.0 h1:D+Kh9t2n4asTnM+TiSxbrKlUemLZandWntj17BJWWb0= -go.opentelemetry.io/collector/receiver/receivertest v0.114.0/go.mod h1:mNSHQ13vFmqD+VAcRzLjStFBejbcWUn2Mp0pAd7Op+U= -go.opentelemetry.io/collector/semconv v0.114.0 h1:/eKcCJwZepQUtEuFuxa0thx2XIOvhFpaf214ZG1a11k= -go.opentelemetry.io/collector/semconv v0.114.0/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A= -go.opentelemetry.io/collector/service v0.114.0 h1:MYF/4nH1CgtiLx09503xPAAUZefCzG1kaO+oanwX590= -go.opentelemetry.io/collector/service v0.114.0/go.mod h1:xH5/RapJdf5Ohh8iU8J0ZstfFYciP1oJPesiByteZxo= +go.opentelemetry.io/collector v0.115.0 h1:qUZ0bTeNBudMxNQ7FJKS//TxTjeJ7tfU/z22mcFavWU= +go.opentelemetry.io/collector v0.115.0/go.mod h1:66qx0xKnVvdwq60e1DEfb4e+zmM9szhPsv2hxZ/Mpj4= +go.opentelemetry.io/collector/client v1.21.0 h1:3Kes8lOFMYVxoxeAmX+DTEAkuS1iTA3NkSfqzGmygJA= +go.opentelemetry.io/collector/client v1.21.0/go.mod h1:jYJGiL0UA975OOyHmjbQSokNWt1OiviI5KjPOMUMGwc= +go.opentelemetry.io/collector/component v0.115.0 h1:iLte1oCiXzjiCnaOBKdsXacfFiECecpWxW3/LeriMoo= +go.opentelemetry.io/collector/component v0.115.0/go.mod h1:oIUFiH7w1eOimdeYhFI+gAIxYSiLDocKVJ0PTvX7d6s= +go.opentelemetry.io/collector/component/componentstatus v0.115.0 h1:pbpUIL+uKDfEiSgKK+S5nuSL6MDIIQYsp4b65ZGVb9M= +go.opentelemetry.io/collector/component/componentstatus v0.115.0/go.mod h1:36A+9XSiOz0Cdhq+UwwPRlEr5CYuSkEnVO9om4BH7d0= +go.opentelemetry.io/collector/component/componenttest v0.115.0 h1:9URDJ9VyP6tuij+YHjp/kSSMecnZOd7oGvzu+rw9SJY= +go.opentelemetry.io/collector/component/componenttest v0.115.0/go.mod h1:PzXvNqKLCiSADZGZFKH+IOHMkaQ0GTHuzysfVbTPKYY= +go.opentelemetry.io/collector/config/configauth v0.115.0 h1:xa+ALdyPgva3rZnLBh1H2oS5MsHP6JxSqMtQmcELnys= +go.opentelemetry.io/collector/config/configauth v0.115.0/go.mod h1:C7anpb3Rf4KswMT+dgOzkW9UX0z/65PLORpUw3p0VYc= +go.opentelemetry.io/collector/config/configcompression v1.21.0 h1:0zbPdZAgPFMAarwJEC4gaR6f/JBP686A3TYSgb3oa+E= +go.opentelemetry.io/collector/config/configcompression v1.21.0/go.mod h1:LvYG00tbPTv0NOLoZN0wXq1F5thcxvukO8INq7xyfWU= +go.opentelemetry.io/collector/config/confighttp v0.115.0 h1:BIy394oNXnqySJwrCqgAJu4gWgAV5aQUDD6k1hy6C8o= +go.opentelemetry.io/collector/config/confighttp v0.115.0/go.mod h1:Wr50ut12NmCEAl4bWLJryw2EjUmJTtYRg89560Q51wc= +go.opentelemetry.io/collector/config/configopaque v1.21.0 h1:PcvRGkBk4Px8BQM7tX+kw4i3jBsfAHGoGQbtZg6Ox7U= +go.opentelemetry.io/collector/config/configopaque v1.21.0/go.mod h1:sW0t0iI/VfRL9VYX7Ik6XzVgPcR+Y5kejTLsYcMyDWs= +go.opentelemetry.io/collector/config/configretry v1.21.0 h1:ZHoOvAkEcv5BBeaJn8IQ6rQ4GMPZWW4S+W7R4QTEbZU= +go.opentelemetry.io/collector/config/configretry v1.21.0/go.mod h1:cleBc9I0DIWpTiiHfu9v83FUaCTqcPXmebpLxjEIqro= +go.opentelemetry.io/collector/config/configtelemetry v0.115.0 h1:U07FinCDop+r2RjWQ3aP9ZWONC7r7kQIp1GkXQi6nsI= +go.opentelemetry.io/collector/config/configtelemetry v0.115.0/go.mod h1:SlBEwQg0qly75rXZ6W1Ig8jN25KBVBkFIIAUI1GiAAE= +go.opentelemetry.io/collector/config/configtls v1.21.0 h1:ZfrlAYgBD8lzp04W0GxwiDmUbrvKsvDYJi+wkyiXlpA= +go.opentelemetry.io/collector/config/configtls v1.21.0/go.mod h1:5EsNefPfVCMOTlOrr3wyj7LrsOgY7V8iqRl8oFZEqtw= +go.opentelemetry.io/collector/config/internal v0.115.0 h1:eVk57iufZpUXyPJFKTb1Ebx5tmcCyroIlt427r5pxS8= +go.opentelemetry.io/collector/config/internal v0.115.0/go.mod h1:OVkadRWlKAoWjHslqjWtBLAne8ceQm8WYT71ZcBWLFc= +go.opentelemetry.io/collector/confmap v1.21.0 h1:1tIcx2/Suwg8VhuPmQw87ba0ludPmumpFCFRZZa6RXA= +go.opentelemetry.io/collector/confmap v1.21.0/go.mod h1:Rrhs+MWoaP6AswZp+ReQ2VO9dfOfcUjdjiSHBsG+nec= +go.opentelemetry.io/collector/connector v0.115.0 h1:4Kkm3HQFzNT1eliMOB8FbIn+PLMRJ2qQku5Vmy3V8Ko= +go.opentelemetry.io/collector/connector v0.115.0/go.mod h1:+ByuAmYLrYHoKh9B+LGqUc0N2kXcN2l8Dea8Mp6brZ8= +go.opentelemetry.io/collector/connector/connectorprofiles v0.115.0 h1:aW1f4Az0I+QJyImFccNWAXqik80bnNu27aQqi2hFfD8= +go.opentelemetry.io/collector/connector/connectorprofiles v0.115.0/go.mod h1:lmynB1CucydOsHa8RSSBh5roUZPfuiv65imXhtNzClM= +go.opentelemetry.io/collector/connector/connectortest v0.115.0 h1:GjtourFr0MJmlbtEPAZ/1BZCxkNAeJ0aMTlrxwftJ0k= +go.opentelemetry.io/collector/connector/connectortest v0.115.0/go.mod h1:f3KQXXNlh/XuV8elmnuVVyfY92dJCAovz10gD72OH0k= +go.opentelemetry.io/collector/consumer v1.21.0 h1:THKZ2Vbi6GkamjTBI2hFq5Dc4kINZTWGwQNa8d/Ty9g= +go.opentelemetry.io/collector/consumer v1.21.0/go.mod h1:FQcC4ThMtRYY41dv+IPNK8POLLhAFY3r1YR5fuP7iiY= +go.opentelemetry.io/collector/consumer/consumererror v0.115.0 h1:yli//xBCQMPZKXNgNlXemo4dvqhnFrAmCZ11DvQgmcY= +go.opentelemetry.io/collector/consumer/consumererror v0.115.0/go.mod h1:LwVzAvQ6ZVNG7mbOvurbAo+W/rKws0IcjOwriuZXqPE= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.115.0 h1:H3fDuyQW1t2HWHkz96WMBQJKUevypOCjBqnqtaAWyoA= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.115.0/go.mod h1:IzEmZ91Tp7TBxVDq8Cc9xvLsmO7H08njr6Pu9P5d9ns= +go.opentelemetry.io/collector/consumer/consumertest v0.115.0 h1:hru0I2447y0TluCdwlKYFFtgcpyCnlM+LiOK1JZyA70= +go.opentelemetry.io/collector/consumer/consumertest v0.115.0/go.mod h1:ybjALRJWR6aKNOzEMy1T1ruCULVDEjj4omtOJMrH/kU= +go.opentelemetry.io/collector/exporter v0.115.0 h1:JnxfpOnsuqhTPKJXVKJLS1Cv3BiVrVLzpHOjJEQw+xw= +go.opentelemetry.io/collector/exporter v0.115.0/go.mod h1:xof3fHQK8wADhaKLIJcQ7ChZaFLNC+haRdPN0wgl6kY= +go.opentelemetry.io/collector/exporter/exporterprofiles v0.115.0 h1:lSQEleCn/q9eFufcuK61NdFKU70ZlgI9dBjPCO/4CrE= +go.opentelemetry.io/collector/exporter/exporterprofiles v0.115.0/go.mod h1:7l5K2AecimX2kx+nZC1gKG3QkP247CO1+SodmJ4fFkQ= +go.opentelemetry.io/collector/exporter/exportertest v0.115.0 h1:P9SMTUXQOtcaq40bGtnnAe14zRmR4/yUgj/Tb2BEf/k= +go.opentelemetry.io/collector/exporter/exportertest v0.115.0/go.mod h1:1jMZ9gFGXglb8wfNrBZIgd+RvpZhSyFwdfE+Jtf9w4U= +go.opentelemetry.io/collector/extension v0.115.0 h1:/cBb8AUdD0KMWC6V3lvCC16eP9Fg0wd1Upcp5rgvuGI= +go.opentelemetry.io/collector/extension v0.115.0/go.mod h1:HI7Ak6loyi6ZrZPsQJW1OO1wbaAW8OqXLFNQlTZnreQ= +go.opentelemetry.io/collector/extension/auth v0.115.0 h1:TTMokbBsSHZRFH48PvGSJmgSS8F3Rkr9MWGHZn8eJDk= +go.opentelemetry.io/collector/extension/auth v0.115.0/go.mod h1:3w+2mzeb2OYNOO4Bi41TUo4jr32ap2y7AOq64IDpxQo= +go.opentelemetry.io/collector/extension/experimental/storage v0.115.0 h1:sZXw0+77092pq24CkUoTRoHQPLQUsDq6HFRNB0g5yR4= +go.opentelemetry.io/collector/extension/experimental/storage v0.115.0/go.mod h1:qjFH7Y3QYYs88By2ZB5GMSUN5k3ul4Brrq2J6lKACA0= +go.opentelemetry.io/collector/extension/extensioncapabilities v0.115.0 h1:/g25Hp5aoCNKdDjIb3Fc7XRglO8yaBRFLO/IUNPnqNI= +go.opentelemetry.io/collector/extension/extensioncapabilities v0.115.0/go.mod h1:EQx7ETiy330O6q05S2KRZsRNDg0aQEeJmVl7Ipx+Fcw= +go.opentelemetry.io/collector/extension/extensiontest v0.115.0 h1:GBVFxFEskR8jSdu9uaQh2qpXnN5VNXhXjpJ2UjxtE8I= +go.opentelemetry.io/collector/extension/extensiontest v0.115.0/go.mod h1:eu1ecbz5mT+cHoH2H3GmD/rOO0WsicSJD2RLrYuOmRA= +go.opentelemetry.io/collector/extension/zpagesextension v0.115.0 h1:zYrZZocc7n0ZuDyXNkIaX0P0qk2fjMQj7NegwBJZA4k= +go.opentelemetry.io/collector/extension/zpagesextension v0.115.0/go.mod h1:OaXwNHF3MAcInBzCXrhXbTNHfIi9b7YGhXjtCFZqxNY= +go.opentelemetry.io/collector/featuregate v1.21.0 h1:+EULHPJDLMipcwAGZVp9Nm8NriRvoBBMxp7MSiIZVMI= +go.opentelemetry.io/collector/featuregate v1.21.0/go.mod h1:3GaXqflNDVwWndNGBJ1+XJFy3Fv/XrFgjMN60N3z7yg= +go.opentelemetry.io/collector/internal/fanoutconsumer v0.115.0 h1:6DRiSECeApFq6Jj5ug77rG53R6FzJEZBfygkyMEXdpg= +go.opentelemetry.io/collector/internal/fanoutconsumer v0.115.0/go.mod h1:vgQf5HQdmLQqpDHpDq2S3nTRoUuKtRcZpRTsy+UiwYw= +go.opentelemetry.io/collector/otelcol v0.115.0 h1:wZhFGrSCZcTQ4qw4ePjI2PaSrOCejoQKAjprKD/xavs= +go.opentelemetry.io/collector/otelcol v0.115.0/go.mod h1:iK8DPvaizirIYKDl1zZG7DDYUj6GkkH4KHifVVM88vk= +go.opentelemetry.io/collector/pdata v1.21.0 h1:PG+UbiFMJ35X/WcAR7Rf/PWmWtRdW0aHlOidsR6c5MA= +go.opentelemetry.io/collector/pdata v1.21.0/go.mod h1:GKb1/zocKJMvxKbS+sl0W85lxhYBTFJ6h6I1tphVyDU= +go.opentelemetry.io/collector/pdata/pprofile v0.115.0 h1:NI89hy13vNDw7EOnQf7Jtitks4HJFO0SUWznTssmP94= +go.opentelemetry.io/collector/pdata/pprofile v0.115.0/go.mod h1:jGzdNfO0XTtfLjXCL/uCC1livg1LlfR+ix2WE/z3RpQ= +go.opentelemetry.io/collector/pdata/testdata v0.115.0 h1:Rblz+AKXdo3fG626jS+KSd0OSA4uMXcTQfpwed6P8LI= +go.opentelemetry.io/collector/pdata/testdata v0.115.0/go.mod h1:inNnRt6S2Nn260EfCBEcjesjlKOSsr0jPwkPqpBkt4s= +go.opentelemetry.io/collector/pipeline v0.115.0 h1:bmACBqb0e8U9ag+vGGHUP7kCfAO7HHROdtzIEg8ulus= +go.opentelemetry.io/collector/pipeline v0.115.0/go.mod h1:qE3DmoB05AW0C3lmPvdxZqd/H4po84NPzd5MrqgtL74= +go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.115.0 h1:3l9ruCAOrssTUDnyChKNzHWOdTtfThnYaoPZ1/+5sD0= +go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.115.0/go.mod h1:2Myg+law/5lcezo9PhhZ0wjCaLYdGK24s1jDWbSW9VY= +go.opentelemetry.io/collector/processor v0.115.0 h1:+fveHGRe24PZPv/F5taahGuZ9HdNW44hgNWEJhIUdyc= +go.opentelemetry.io/collector/processor v0.115.0/go.mod h1:/oLHBlLsm7tFb7zOIrA5C0j14yBtjXKAgxJJ2Bktyk4= +go.opentelemetry.io/collector/processor/processorprofiles v0.115.0 h1:cCZAs+FXaebZPppqAN3m+X3etoSBL6NvyQo8l0hOZoo= +go.opentelemetry.io/collector/processor/processorprofiles v0.115.0/go.mod h1:kMxF0gknlWX4duuAJFi2/HuIRi6C3w95tOenRa0GKOY= +go.opentelemetry.io/collector/processor/processortest v0.115.0 h1:j9HEaYFOeOB6VYl9zGhBnhQbTkqGBa2udUvu5NTh6hc= +go.opentelemetry.io/collector/processor/processortest v0.115.0/go.mod h1:Gws+VEnp/eW3qAqPpqbKsrbnnxxNfyDjqrfUXbZfZic= +go.opentelemetry.io/collector/receiver v0.115.0 h1:55Q3Jvj6zHCIA1psKqi/3kEMJO4OqUF5tNAEYNdB1U8= +go.opentelemetry.io/collector/receiver v0.115.0/go.mod h1:nBSCh2O/WUcfgpJ+Jpz+B0z0Hn5jHeRvF2WmLij5EIY= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.115.0 h1:R9JLaj2Al93smIPUkbJshAkb/cY0H5JBOxIx+Zu0NG4= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.115.0/go.mod h1:05E5hGujWeeXJmzKZwTdHyZ/+rRyrQlQB5p5Q2XY39M= +go.opentelemetry.io/collector/receiver/receivertest v0.115.0 h1:OiB684SbHQi6/Pd3ZH0cXjYvCpBS9ilQBfTQx0wVXHg= +go.opentelemetry.io/collector/receiver/receivertest v0.115.0/go.mod h1:Y8Z9U/bz9Xpyt8GI8DxZZgryw3mnnIw+AeKVLTD2cP8= +go.opentelemetry.io/collector/semconv v0.115.0 h1:SoqMvg4ZEB3mz2EdAb6XYa+TuMo5Mir5FRBr3nVFUDY= +go.opentelemetry.io/collector/semconv v0.115.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI= +go.opentelemetry.io/collector/service v0.115.0 h1:k4GAOiI5tZgB2QKgwA6c3TeAVr7QL/ft5cOQbzUr8Iw= +go.opentelemetry.io/collector/service v0.115.0/go.mod h1:DKde9LMhNebdREecDSsqiTFLI2wRc+IoV4/wGxU6goY= go.opentelemetry.io/contrib/bridges/otelzap v0.6.0 h1:j8icMXyyqNf6HGuwlYhniPnVsbJIq7n+WirDu3VAJdQ= go.opentelemetry.io/contrib/bridges/otelzap v0.6.0/go.mod h1:evIOZpl+kAlU5IsaYX2Siw+IbpacAZvXemVsgt70uvw= go.opentelemetry.io/contrib/config v0.10.0 h1:2JknAzMaYjxrHkTnZh3eOme/Y2P5eHE2SWfhfV6Xd6c= @@ -548,8 +544,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/controller/telemetry/collector_controller.go b/internal/controller/telemetry/collector_controller.go index ff1c44b..36585a6 100644 --- a/internal/controller/telemetry/collector_controller.go +++ b/internal/controller/telemetry/collector_controller.go @@ -20,11 +20,11 @@ import ( "math" "reflect" "slices" + "strings" "time" "emperror.dev/errors" "github.com/cisco-open/operator-tools/pkg/reconciler" - "github.com/imdario/mergo" otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -42,12 +42,14 @@ import ( otelcolconfgen "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/otel_conf_gen" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/otel_conf_gen/validator" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components" + "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/extension/storage" + "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/utils" ) const ( otelCollectorKind = "OpenTelemetryCollector" requeueDelayOnFailedTenant = 20 * time.Second - axoflowOtelCollectorImageRef = "ghcr.io/axoflow/axoflow-otel-collector/axoflow-otel-collector:0.112.0-dev1" + axoflowOtelCollectorImageRef = "ghcr.io/axoflow/axoflow-otel-collector/axoflow-otel-collector:0.112.0-dev10" ) var ( @@ -130,14 +132,16 @@ func (r *CollectorReconciler) buildConfigInputForCollector(ctx context.Context, } return otelcolconfgen.OtelColConfigInput{ - Tenants: tenants, - Subscriptions: subscriptions, - Bridges: bridgesReferencedByTenant, - OutputsWithSecretData: outputs, - TenantSubscriptionMap: tenantSubscriptionMap, - SubscriptionOutputMap: subscriptionOutputMap, - Debug: collector.Spec.Debug, - MemoryLimiter: *collector.Spec.MemoryLimiter, + ResourceRelations: components.ResourceRelations{ + Tenants: tenants, + Subscriptions: subscriptions, + Bridges: bridgesReferencedByTenant, + OutputsWithSecretData: outputs, + TenantSubscriptionMap: tenantSubscriptionMap, + SubscriptionOutputMap: subscriptionOutputMap, + }, + Debug: collector.Spec.Debug, + MemoryLimiter: *collector.Spec.MemoryLimiter, }, nil } @@ -231,10 +235,7 @@ func (r *CollectorReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, fmt.Errorf("%+v", err) } - otelCollector, state, err := r.otelCollector(collector, otelConfig, additionalArgs, saName.Name) - if err != nil { - return ctrl.Result{}, err - } + otelCollector, state := r.otelCollector(collector, otelConfig, additionalArgs, otelConfigInput.Tenants, saName.Name) if err := ctrl.SetControllerReference(collector, otelCollector, r.Scheme); err != nil { return ctrl.Result{}, err @@ -385,7 +386,7 @@ func (r *CollectorReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *CollectorReconciler) otelCollector(collector *v1alpha1.Collector, otelConfig otelv1beta1.Config, additionalArgs map[string]string, saName string) (*otelv1beta1.OpenTelemetryCollector, reconciler.DesiredState, error) { +func (r *CollectorReconciler) otelCollector(collector *v1alpha1.Collector, otelConfig otelv1beta1.Config, additionalArgs map[string]string, tenants []v1alpha1.Tenant, saName string) (*otelv1beta1.OpenTelemetryCollector, reconciler.DesiredState) { otelCollector := otelv1beta1.OpenTelemetryCollector{ TypeMeta: metav1.TypeMeta{ APIVersion: otelv1beta1.GroupVersion.String(), @@ -402,9 +403,8 @@ func (r *CollectorReconciler) otelCollector(collector *v1alpha1.Collector, otelC OpenTelemetryCommonFields: *collector.Spec.OtelCommonFields, }, } - if err := setOtelCommonFieldsDefaults(&otelCollector.Spec.OpenTelemetryCommonFields, additionalArgs, saName); err != nil { - return &otelCollector, nil, err - } + appendAdditionalVolumesForTenantsFileStorage(&otelCollector.Spec.OpenTelemetryCommonFields, tenants) + setOtelCommonFieldsDefaults(&otelCollector.Spec.OpenTelemetryCommonFields, additionalArgs, saName) if memoryLimit := collector.Spec.GetMemoryLimit(); memoryLimit != nil { // Calculate 80% of the specified memory limit for GOMEMLIMIT @@ -423,7 +423,7 @@ func (r *CollectorReconciler) otelCollector(collector *v1alpha1.Collector, otelC return nil }) - return &otelCollector, beforeUpdateHook, nil + return &otelCollector, beforeUpdateHook } func (r *CollectorReconciler) reconcileRBAC(ctx context.Context, collector *v1alpha1.Collector) (v1alpha1.NamespacedName, error) { @@ -563,7 +563,48 @@ func normalizeStringSlice(inputList []string) []string { return uniqueList } -func setOtelCommonFieldsDefaults(otelCommonFields *otelv1beta1.OpenTelemetryCommonFields, additionalArgs map[string]string, saName string) error { +func appendAdditionalVolumesForTenantsFileStorage(otelCommonFields *otelv1beta1.OpenTelemetryCommonFields, tenants []v1alpha1.Tenant) { + var volumeMountsForInit []corev1.VolumeMount + var chmodCommands []string + + for _, tenant := range tenants { + if tenant.Spec.PersistenceConfig.EnableFileStorage { + bufferVolumeName := fmt.Sprintf("buffervolume-%s", tenant.Name) + mountPath := storage.DetermineFileStorageDirectory(tenant.Spec.PersistenceConfig.Directory, tenant.Name) + volumeMount := corev1.VolumeMount{ + Name: bufferVolumeName, + MountPath: mountPath, + } + volumeMountsForInit = append(volumeMountsForInit, volumeMount) + chmodCommands = append(chmodCommands, fmt.Sprintf("chmod -R 777 %s", mountPath)) + + otelCommonFields.VolumeMounts = append(otelCommonFields.VolumeMounts, volumeMount) + otelCommonFields.Volumes = append(otelCommonFields.Volumes, corev1.Volume{ + Name: bufferVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: mountPath, + Type: utils.ToPtr(corev1.HostPathDirectoryOrCreate), + }, + }, + }) + } + } + + // Add a single initContainer to handle all chmod operations + if len(chmodCommands) > 0 && len(volumeMountsForInit) > 0 { + initContainer := corev1.Container{ + Name: "init-chmod", + Image: "busybox", + Command: []string{"sh", "-c"}, + Args: []string{strings.Join(chmodCommands, " && ")}, + VolumeMounts: volumeMountsForInit, + } + otelCommonFields.InitContainers = append(otelCommonFields.InitContainers, initContainer) + } +} + +func setOtelCommonFieldsDefaults(otelCommonFields *otelv1beta1.OpenTelemetryCommonFields, additionalArgs map[string]string, saName string) { if otelCommonFields == nil { otelCommonFields = &otelv1beta1.OpenTelemetryCommonFields{} } @@ -571,8 +612,11 @@ func setOtelCommonFieldsDefaults(otelCommonFields *otelv1beta1.OpenTelemetryComm otelCommonFields.Image = axoflowOtelCollectorImageRef otelCommonFields.ServiceAccount = saName - if err := mergo.Merge(&otelCommonFields.Args, additionalArgs, mergo.WithOverride); err != nil { - return err + if otelCommonFields.Args == nil { + otelCommonFields.Args = make(map[string]string) + } + for key, value := range additionalArgs { + otelCommonFields.Args[key] = value } volumeMounts := []corev1.VolumeMount{ @@ -587,9 +631,7 @@ func setOtelCommonFieldsDefaults(otelCommonFields *otelv1beta1.OpenTelemetryComm MountPath: "/var/lib/docker/containers", }, } - if err := mergo.Merge(&otelCommonFields.VolumeMounts, volumeMounts, mergo.WithOverride); err != nil { - return err - } + otelCommonFields.VolumeMounts = append(otelCommonFields.VolumeMounts, volumeMounts...) volumes := []corev1.Volume{ { @@ -609,9 +651,5 @@ func setOtelCommonFieldsDefaults(otelCommonFields *otelv1beta1.OpenTelemetryComm }, }, } - if err := mergo.Merge(&otelCommonFields.Volumes, volumes, mergo.WithOverride); err != nil { - return err - } - - return nil + otelCommonFields.Volumes = append(otelCommonFields.Volumes, volumes...) } diff --git a/internal/controller/telemetry/collector_controller_test.go b/internal/controller/telemetry/collector_controller_test.go index 5343c9e..dc2c7b6 100644 --- a/internal/controller/telemetry/collector_controller_test.go +++ b/internal/controller/telemetry/collector_controller_test.go @@ -148,11 +148,7 @@ func TestSetOtelCommonFieldsDefaults(t *testing.T) { if ttp.initialCommonFields != nil { commonFieldsCopy = ttp.initialCommonFields.DeepCopy() } - - err := setOtelCommonFieldsDefaults(commonFieldsCopy, ttp.additionalArgs, ttp.saName) - if err != nil { - assert.EqualError(t, err, ttp.expectedError.Error()) - } + setOtelCommonFieldsDefaults(commonFieldsCopy, ttp.additionalArgs, ttp.saName) assert.Equal(t, ttp.expectedResult, commonFieldsCopy) }) diff --git a/internal/controller/telemetry/otel_conf_gen/otel_col_conf_test_fixtures/complex.yaml b/internal/controller/telemetry/otel_conf_gen/otel_col_conf_test_fixtures/complex.yaml index fe0977b..03517ea 100644 --- a/internal/controller/telemetry/otel_conf_gen/otel_col_conf_test_fixtures/complex.yaml +++ b/internal/controller/telemetry/otel_conf_gen/otel_col_conf_test_fixtures/complex.yaml @@ -59,6 +59,12 @@ exporters: {} extensions: bearertokenauth/collector_otlp-test-output: token: testtoken + file_storage/example-tenant-a: + create_directory: true + directory: /var/lib/otelcol/file_storage/example-tenant-a + file_storage/example-tenant-b: + create_directory: true + directory: /var/lib/otelcol/file_storage/example-tenant-b processors: attributes/exporter_name_fluentforward-test-output: actions: @@ -159,7 +165,9 @@ processors: receivers: {} service: extensions: - - bearertokenauth/collector_otlp-test-output + - bearertokenauth/collector_otlp-test-output + - file_storage/example-tenant-a + - file_storage/example-tenant-b pipelines: logs/output_example-tenant-a-ns_subscription-example-1_collector_loki-test-output: exporters: diff --git a/internal/controller/telemetry/otel_conf_gen/otel_conf_gen.go b/internal/controller/telemetry/otel_conf_gen/otel_conf_gen.go index 7c30657..f5b4220 100644 --- a/internal/controller/telemetry/otel_conf_gen/otel_conf_gen.go +++ b/internal/controller/telemetry/otel_conf_gen/otel_conf_gen.go @@ -20,6 +20,7 @@ import ( "fmt" "reflect" "slices" + "sort" "strings" "github.com/hashicorp/go-multierror" @@ -29,6 +30,7 @@ import ( "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/connector" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/exporter" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/extension" + "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/extension/storage" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/processor" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components/receiver" otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" @@ -38,26 +40,21 @@ import ( var ErrNoResources = errors.New("there are no resources deployed that the collector(s) can use") type OtelColConfigInput struct { - // These must only include resources that are selected by the collector, tenant labelselectors, and listed outputs in the subscriptions - Tenants []v1alpha1.Tenant - Subscriptions map[v1alpha1.NamespacedName]v1alpha1.Subscription - Bridges []v1alpha1.Bridge - OutputsWithSecretData []components.OutputWithSecretData - MemoryLimiter v1alpha1.MemoryLimiter - - // Subscriptions map, where the key is the Tenants' name, value is a slice of subscriptions' namespaced name - TenantSubscriptionMap map[string][]v1alpha1.NamespacedName - SubscriptionOutputMap map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName - Debug bool + components.ResourceRelations + MemoryLimiter v1alpha1.MemoryLimiter + Debug bool } func (cfgInput *OtelColConfigInput) IsEmpty() bool { + if cfgInput == nil { + return true + } + v := reflect.ValueOf(*cfgInput) for i := range v.NumField() { field := v.Field(i) - switch field.Kind() { - case reflect.Slice, reflect.Map: - if field.Len() != 0 { + if v.Field(i).Kind() == reflect.Struct { + if !reflect.DeepEqual(field.Interface(), reflect.Zero(field.Type()).Interface()) { return false } } @@ -69,10 +66,12 @@ func (cfgInput *OtelColConfigInput) IsEmpty() bool { func (cfgInput *OtelColConfigInput) generateExporters(ctx context.Context) map[string]any { exporters := map[string]any{} maps.Copy(exporters, exporter.GenerateMetricsExporters()) - maps.Copy(exporters, exporter.GenerateOTLPGRPCExporters(ctx, cfgInput.OutputsWithSecretData)) - maps.Copy(exporters, exporter.GenerateOTLPHTTPExporters(ctx, cfgInput.OutputsWithSecretData)) - maps.Copy(exporters, exporter.GenerateFluentforwardExporters(ctx, cfgInput.OutputsWithSecretData)) - maps.Copy(exporters, exporter.GenerateDebugExporters()) + maps.Copy(exporters, exporter.GenerateOTLPGRPCExporters(ctx, cfgInput.ResourceRelations)) + maps.Copy(exporters, exporter.GenerateOTLPHTTPExporters(ctx, cfgInput.ResourceRelations)) + maps.Copy(exporters, exporter.GenerateFluentforwardExporters(ctx, cfgInput.ResourceRelations)) + if cfgInput.Debug { + maps.Copy(exporters, exporter.GenerateDebugExporters()) + } return exporters } @@ -108,7 +107,7 @@ func (cfgInput *OtelColConfigInput) generateProcessors() map[string]any { return processors } -func (cfgInput *OtelColConfigInput) generateExtensions() map[string]any { +func (cfgInput *OtelColConfigInput) generateExtensions() (map[string]any, []string) { extensions := make(map[string]any) for _, output := range cfgInput.OutputsWithSecretData { if output.Output.Spec.Authentication != nil { @@ -123,7 +122,24 @@ func (cfgInput *OtelColConfigInput) generateExtensions() map[string]any { } } - return extensions + for _, tenant := range cfgInput.Tenants { + if tenant.Spec.PersistenceConfig.EnableFileStorage { + extensions[fmt.Sprintf("file_storage/%s", tenant.Name)] = storage.GenerateFileStorageExtensionForTenant(tenant.Spec.PersistenceConfig.Directory, tenant.Name) + } + } + + var extensionNames []string + if len(extensions) > 0 { + extensionNames = make([]string, 0, len(extensions)) + for k := range extensions { + extensionNames = append(extensionNames, k) + } + } else { + extensionNames = nil + } + sort.Strings(extensionNames) + + return extensions, extensionNames } func (cfgInput *OtelColConfigInput) generateReceivers() map[string]any { @@ -133,11 +149,8 @@ func (cfgInput *OtelColConfigInput) generateReceivers() map[string]any { return tenantName == t.Name }); tenantIdx != -1 { namespaces := cfgInput.Tenants[tenantIdx].Status.LogSourceNamespaces - - // Generate filelog receiver for the tenant if it has any logsource namespaces. - // Or Handle "all namespaces" case: selectors are initialized but empty - if len(namespaces) > 0 || (cfgInput.Tenants[tenantIdx].Spec.LogSourceNamespaceSelectors != nil && len(namespaces) == 0) { - receivers[fmt.Sprintf("filelog/%s", tenantName)] = receiver.GenerateDefaultKubernetesReceiver(namespaces) + if len(namespaces) > 0 || cfgInput.Tenants[tenantIdx].Spec.SelectFromAllNamespaces { + receivers[fmt.Sprintf("filelog/%s", tenantName)] = receiver.GenerateDefaultKubernetesReceiver(namespaces, cfgInput.Tenants[tenantIdx]) } } } @@ -278,7 +291,7 @@ func (cfgInput *OtelColConfigInput) AssembleConfig(ctx context.Context) (otelv1b processors := cfgInput.generateProcessors() - extensions := cfgInput.generateExtensions() + extensions, extensionNames := cfgInput.generateExtensions() receivers := cfgInput.generateReceivers() @@ -299,16 +312,6 @@ func (cfgInput *OtelColConfigInput) AssembleConfig(ctx context.Context) (otelv1b } } - var extensionNames []string - if len(extensions) > 0 { - extensionNames = make([]string, 0, len(extensions)) - for k := range extensions { - extensionNames = append(extensionNames, k) - } - } else { - extensionNames = nil - } - otelConfig := otelv1beta1.Config{ Receivers: otelv1beta1.AnyConfig{Object: receivers}, Exporters: otelv1beta1.AnyConfig{Object: exporters}, @@ -352,6 +355,12 @@ func validateTenants(tenants *[]v1alpha1.Tenant) error { return errors.New("no tenants provided, at least one tenant must be provided") } + for _, tenant := range *tenants { + if tenant.Status.LogSourceNamespaces != nil && tenant.Spec.SelectFromAllNamespaces { + result = multierror.Append(result, fmt.Errorf("tenant %s has both log source namespace selectors and select from all namespaces enabled", tenant.Name)) + } + } + return result.ErrorOrNil() } diff --git a/internal/controller/telemetry/otel_conf_gen/otel_conf_gen_test.go b/internal/controller/telemetry/otel_conf_gen/otel_conf_gen_test.go index 940ca7b..8617287 100644 --- a/internal/controller/telemetry/otel_conf_gen/otel_conf_gen_test.go +++ b/internal/controller/telemetry/otel_conf_gen/otel_conf_gen_test.go @@ -17,15 +17,12 @@ package otel_conf_gen import ( "context" _ "embed" - "encoding/json" "reflect" "testing" "time" "github.com/google/go-cmp/cmp" otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" - "github.com/siliconbrain/go-mapseqs/mapseqs" - "github.com/siliconbrain/go-seqs/seqs" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,6 +59,9 @@ func TestOtelColConfComplex(t *testing.T) { }, }, }, + Status: v1alpha1.SubscriptionStatus{ + Tenant: "example-tenant-a", + }, }, {Name: "subscription-example-2", Namespace: "example-tenant-a-ns"}: { ObjectMeta: metav1.ObjectMeta{ @@ -77,6 +77,9 @@ func TestOtelColConfComplex(t *testing.T) { }, }, }, + Status: v1alpha1.SubscriptionStatus{ + Tenant: "example-tenant-a", + }, }, {Name: "subscription-example-3", Namespace: "example-tenant-b-ns"}: { ObjectMeta: metav1.ObjectMeta{ @@ -96,177 +99,185 @@ func TestOtelColConfComplex(t *testing.T) { }, }, }, + Status: v1alpha1.SubscriptionStatus{ + Tenant: "example-tenant-b", + }, }, } inputCfg := OtelColConfigInput{ - Subscriptions: subscriptions, - Tenants: []v1alpha1.Tenant{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "example-tenant-a", - }, - Spec: v1alpha1.TenantSpec{ - SubscriptionNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "example-tenant-a", - }, - }, + ResourceRelations: components.ResourceRelations{ + Subscriptions: subscriptions, + Tenants: []v1alpha1.Tenant{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "example-tenant-a", }, - LogSourceNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "example-tenant-a", + Spec: v1alpha1.TenantSpec{ + SubscriptionNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "example-tenant-a", + }, }, }, - }, - }, - Status: v1alpha1.TenantStatus{ - LogSourceNamespaces: []string{ - "example-tenant-a", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "example-tenant-b", - }, - Spec: v1alpha1.TenantSpec{ - SubscriptionNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "example-tenant-b", + LogSourceNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "example-tenant-a", + }, }, }, + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, }, - LogSourceNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "example-tenant-b", + Status: v1alpha1.TenantStatus{ + LogSourceNamespaces: []string{ + "example-tenant-a", + }, + Subscriptions: []v1alpha1.NamespacedName{ + { + Namespace: "example-tenant-a-ns", + Name: "subscription-example-1", + }, + { + Namespace: "example-tenant-a-ns", + Name: "subscription-example-2", }, }, }, }, - Status: v1alpha1.TenantStatus{ - LogSourceNamespaces: []string{ - "example-tenant-b", + { + ObjectMeta: metav1.ObjectMeta{ + Name: "example-tenant-b", }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "example-tenant-a", - }, - Spec: v1alpha1.TenantSpec{ - SubscriptionNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "example-tenant-a", + Spec: v1alpha1.TenantSpec{ + SubscriptionNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "example-tenant-b", + }, + }, + }, + LogSourceNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "example-tenant-b", + }, }, }, + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, }, - LogSourceNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "example-tenant-a", + Status: v1alpha1.TenantStatus{ + LogSourceNamespaces: []string{ + "example-tenant-b", + }, + Subscriptions: []v1alpha1.NamespacedName{ + { + Namespace: "example-tenant-b-ns", + Name: "subscription-example-3", }, }, }, }, }, - }, - OutputsWithSecretData: []components.OutputWithSecretData{ - { - Secret: corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "bearer-test-secret", - Namespace: "collector", - }, - Data: map[string][]byte{ - "token": []byte("testtoken"), - }, - Type: "Opaque", - }, - Output: v1alpha1.Output{ - ObjectMeta: metav1.ObjectMeta{ - Name: "otlp-test-output", - Namespace: "collector", + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Secret: corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bearer-test-secret", + Namespace: "collector", + }, + Data: map[string][]byte{ + "token": []byte("testtoken"), + }, + Type: "Opaque", }, - Spec: v1alpha1.OutputSpec{ - OTLPGRPC: &v1alpha1.OTLPGRPC{ - GRPCClientConfig: v1alpha1.GRPCClientConfig{ - Endpoint: utils.ToPtr("receiver-collector.example-tenant-a-ns.svc.cluster.local:4317"), - TLSSetting: &v1alpha1.TLSClientSetting{ - Insecure: true, + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "otlp-test-output", + Namespace: "collector", + }, + Spec: v1alpha1.OutputSpec{ + OTLPGRPC: &v1alpha1.OTLPGRPC{ + GRPCClientConfig: v1alpha1.GRPCClientConfig{ + Endpoint: utils.ToPtr("receiver-collector.example-tenant-a-ns.svc.cluster.local:4317"), + TLSSetting: &v1alpha1.TLSClientSetting{ + Insecure: true, + }, }, }, - }, - Authentication: &v1alpha1.OutputAuth{ - BearerAuth: &v1alpha1.BearerAuthConfig{ - SecretRef: &corev1.SecretReference{ - Name: "bearer-test-secret", - Namespace: "collector", + Authentication: &v1alpha1.OutputAuth{ + BearerAuth: &v1alpha1.BearerAuthConfig{ + SecretRef: &corev1.SecretReference{ + Name: "bearer-test-secret", + Namespace: "collector", + }, }, }, - }, - Batch: &v1alpha1.Batch{ - Timeout: "5s", - SendBatchSize: 512, - SendBatchMaxSize: 4096, - MetadataKeys: []string{"key1", "key2"}, - MetadataCardinalityLimit: 100, + Batch: &v1alpha1.Batch{ + Timeout: "5s", + SendBatchSize: 512, + SendBatchMaxSize: 4096, + MetadataKeys: []string{"key1", "key2"}, + MetadataCardinalityLimit: 100, + }, }, }, }, - }, - { - Output: v1alpha1.Output{ - ObjectMeta: metav1.ObjectMeta{ - Name: "otlp-test-output-2", - Namespace: "collector", - }, - Spec: v1alpha1.OutputSpec{ - OTLPGRPC: &v1alpha1.OTLPGRPC{ - GRPCClientConfig: v1alpha1.GRPCClientConfig{ - Endpoint: utils.ToPtr("receiver-collector.example-tenant-a-ns.svc.cluster.local:4317"), - TLSSetting: &v1alpha1.TLSClientSetting{ - Insecure: true, + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "otlp-test-output-2", + Namespace: "collector", + }, + Spec: v1alpha1.OutputSpec{ + OTLPGRPC: &v1alpha1.OTLPGRPC{ + GRPCClientConfig: v1alpha1.GRPCClientConfig{ + Endpoint: utils.ToPtr("receiver-collector.example-tenant-a-ns.svc.cluster.local:4317"), + TLSSetting: &v1alpha1.TLSClientSetting{ + Insecure: true, + }, }, }, }, }, }, - }, - { - Output: v1alpha1.Output{ - ObjectMeta: metav1.ObjectMeta{ - Name: "loki-test-output", - Namespace: "collector", - }, - Spec: v1alpha1.OutputSpec{ - OTLPHTTP: &v1alpha1.OTLPHTTP{ - HTTPClientConfig: v1alpha1.HTTPClientConfig{ - Endpoint: utils.ToPtr[string]("loki.example-tenant-a-ns.svc.cluster.local:4317"), - TLSSetting: &v1alpha1.TLSClientSetting{ - Insecure: true, + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "loki-test-output", + Namespace: "collector", + }, + Spec: v1alpha1.OutputSpec{ + OTLPHTTP: &v1alpha1.OTLPHTTP{ + HTTPClientConfig: v1alpha1.HTTPClientConfig{ + Endpoint: utils.ToPtr("loki.example-tenant-a-ns.svc.cluster.local:4317"), + TLSSetting: &v1alpha1.TLSClientSetting{ + Insecure: true, + }, }, }, }, }, }, - }, - { - Output: v1alpha1.Output{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fluentforward-test-output", - Namespace: "collector", - }, - Spec: v1alpha1.OutputSpec{ - Fluentforward: &v1alpha1.Fluentforward{ - TCPClientSettings: v1alpha1.TCPClientSettings{ - Endpoint: utils.ToPtr("fluentd.example-tenant-b-ns.svc.cluster.local:24224"), - TLSSetting: &v1alpha1.TLSClientSetting{ - Insecure: true, + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fluentforward-test-output", + Namespace: "collector", + }, + Spec: v1alpha1.OutputSpec{ + Fluentforward: &v1alpha1.Fluentforward{ + TCPClientSettings: v1alpha1.TCPClientSettings{ + Endpoint: &v1alpha1.Endpoint{ + TCPAddr: utils.ToPtr("fluentd.example-tenant-b-ns.svc.cluster.local:24224"), + }, + TLSSetting: &v1alpha1.TLSClientSetting{ + Insecure: true, + }, }, }, }, @@ -281,42 +292,24 @@ func TestOtelColConfComplex(t *testing.T) { }, } - // TODO extract this logic - - subscriptionOutputMap := map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{} + inputCfg.SubscriptionOutputMap = make(map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName) for _, subscription := range inputCfg.Subscriptions { - subscriptionOutputMap[subscription.NamespacedName()] = subscription.Spec.Outputs + inputCfg.SubscriptionOutputMap[subscription.NamespacedName()] = subscription.Spec.Outputs } - inputCfg.SubscriptionOutputMap = subscriptionOutputMap - inputCfg.TenantSubscriptionMap = map[string][]v1alpha1.NamespacedName{} - tenantA := inputCfg.Tenants[0] - tenantASubscriptions := make(map[v1alpha1.NamespacedName]v1alpha1.Subscription) - for subName, sub := range inputCfg.Subscriptions { - if subName.Namespace != "example-tenant-b-ns" { - tenantASubscriptions[subName] = sub + inputCfg.TenantSubscriptionMap = make(map[string][]v1alpha1.NamespacedName) + for _, tenant := range inputCfg.Tenants { + for _, subscription := range inputCfg.Subscriptions { + if subscription.Status.Tenant == tenant.Name { + inputCfg.TenantSubscriptionMap[tenant.Name] = append(inputCfg.TenantSubscriptionMap[tenant.Name], subscription.NamespacedName()) + } } } - inputCfg.TenantSubscriptionMap[tenantA.Name] = seqs.ToSlice(mapseqs.KeysOf(tenantASubscriptions)) - - tenantB := inputCfg.Tenants[1] - inputCfg.TenantSubscriptionMap[tenantB.Name] = []v1alpha1.NamespacedName{ - { - Name: "subscription-example-3", - Namespace: "example-tenant-b-ns", - }, - } // Config // The receiver and exporter entries are not serialized because of tags on the underlying data structure. The tests won't contain them, this is a known issue. generatedConfig, _ := inputCfg.AssembleConfig(context.TODO()) - actualJSONBytes, err1 := json.Marshal(generatedConfig) - if err1 != nil { - t.Fatalf("error %v", err1) - - } - print(actualJSONBytes) actualYAMLBytes, err := yaml.Marshal(generatedConfig) if err != nil { t.Fatalf("error %v", err) @@ -438,7 +431,9 @@ func TestOtelColConfigInput_generateRoutingConnectorForTenantsSubscription(t *te for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfgInput := &OtelColConfigInput{ - Subscriptions: tt.fields.Subscriptions, + ResourceRelations: components.ResourceRelations{ + Subscriptions: tt.fields.Subscriptions, + }, } got := connector.GenerateRoutingConnectorForTenantsSubscriptions(tt.args.tenantName, v1alpha1.RouteConfig{}, tt.args.subscriptionNames, cfgInput.Subscriptions) assert.Equal(t, got, tt.want) @@ -455,23 +450,24 @@ func TestOtelColConfigInput_generateNamedPipelines(t *testing.T) { { name: "Single tenant with no subscriptions", cfgInput: OtelColConfigInput{ - TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ - "tenant1": { + ResourceRelations: components.ResourceRelations{ + Bridges: nil, + OutputsWithSecretData: nil, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + "tenant1": { + { + Namespace: "ns1", + Name: "sub1", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ { Namespace: "ns1", Name: "sub1", - }, + }: {}, }, }, - SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ - { - Namespace: "ns1", - Name: "sub1", - }: {}, - }, - Bridges: nil, - OutputsWithSecretData: nil, - Debug: false, }, expectedPipelines: map[string]*otelv1beta1.Pipeline{ "logs/tenant_tenant1": pipeline.GenerateRootPipeline([]v1alpha1.Tenant{}, "tenant1"), @@ -495,174 +491,175 @@ func TestOtelColConfigInput_generateNamedPipelines(t *testing.T) { { name: "Three tenants two bridges", cfgInput: OtelColConfigInput{ - Tenants: []v1alpha1.Tenant{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant1", - }, - Spec: v1alpha1.TenantSpec{ - LogSourceNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "ns1", + ResourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant1", + }, + Spec: v1alpha1.TenantSpec{ + LogSourceNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "ns1", + }, }, }, }, + Status: v1alpha1.TenantStatus{ + LogSourceNamespaces: []string{"ns1"}, + }, }, - Status: v1alpha1.TenantStatus{ - LogSourceNamespaces: []string{"ns1"}, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant2", - }, - Spec: v1alpha1.TenantSpec{ - SubscriptionNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "ns2", + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant2", + }, + Spec: v1alpha1.TenantSpec{ + SubscriptionNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "ns2", + }, }, }, }, - }, - Status: v1alpha1.TenantStatus{ - Subscriptions: []v1alpha1.NamespacedName{ - { - Namespace: "ns2", - Name: "sub2", + Status: v1alpha1.TenantStatus{ + Subscriptions: []v1alpha1.NamespacedName{ + { + Namespace: "ns2", + Name: "sub2", + }, }, }, }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant3", - }, - Spec: v1alpha1.TenantSpec{ - SubscriptionNamespaceSelectors: []metav1.LabelSelector{ - { - MatchLabels: map[string]string{ - "nsSelector": "ns3", + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant3", + }, + Spec: v1alpha1.TenantSpec{ + SubscriptionNamespaceSelectors: []metav1.LabelSelector{ + { + MatchLabels: map[string]string{ + "nsSelector": "ns3", + }, }, }, }, - }, - Status: v1alpha1.TenantStatus{ - Subscriptions: []v1alpha1.NamespacedName{ - { - Namespace: "ns3", - Name: "sub3", + Status: v1alpha1.TenantStatus{ + Subscriptions: []v1alpha1.NamespacedName{ + { + Namespace: "ns3", + Name: "sub3", + }, }, }, }, }, - }, - Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ - { - Namespace: "ns2", - Name: "sub2", - }: { - ObjectMeta: metav1.ObjectMeta{ - Name: "sub2", + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { Namespace: "ns2", - }, - Spec: v1alpha1.SubscriptionSpec{ - Condition: "true", - Outputs: []v1alpha1.NamespacedName{ - { - Namespace: "xy", - Name: "zq", + Name: "sub2", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub2", + Namespace: "ns2", + }, + Spec: v1alpha1.SubscriptionSpec{ + Condition: "true", + Outputs: []v1alpha1.NamespacedName{ + { + Namespace: "xy", + Name: "zq", + }, }, }, + Status: v1alpha1.SubscriptionStatus{ + Tenant: "tenant2", + Outputs: []v1alpha1.NamespacedName{{Namespace: "xy", Name: "zq"}}, + }, }, - Status: v1alpha1.SubscriptionStatus{ - Tenant: "tenant2", - Outputs: []v1alpha1.NamespacedName{{Namespace: "xy", Name: "zq"}}, - }, - }, - { - Namespace: "ns3", - Name: "sub3", - }: { - ObjectMeta: metav1.ObjectMeta{ - Name: "sub3", + { Namespace: "ns3", - }, - Spec: v1alpha1.SubscriptionSpec{ - Condition: "true", - Outputs: []v1alpha1.NamespacedName{ - { - Namespace: "xy", - Name: "zq", + Name: "sub3", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub3", + Namespace: "ns3", + }, + Spec: v1alpha1.SubscriptionSpec{ + Condition: "true", + Outputs: []v1alpha1.NamespacedName{ + { + Namespace: "xy", + Name: "zq", + }, }, }, - }, - Status: v1alpha1.SubscriptionStatus{ - Tenant: "tenant3", - Outputs: []v1alpha1.NamespacedName{{Namespace: "xy", Name: "zq"}}, + Status: v1alpha1.SubscriptionStatus{ + Tenant: "tenant3", + Outputs: []v1alpha1.NamespacedName{{Namespace: "xy", Name: "zq"}}, + }, }, }, - }, - Bridges: []v1alpha1.Bridge{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "bridge1", + Bridges: []v1alpha1.Bridge{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "bridge1", + }, + Spec: v1alpha1.BridgeSpec{ + SourceTenant: "tenant1", + TargetTenant: "tenant2", + Condition: `attributes["parsed"]["method"] == "GET"`, + }, }, - Spec: v1alpha1.BridgeSpec{ - SourceTenant: "tenant1", - TargetTenant: "tenant2", - Condition: `attributes["parsed"]["method"] == "GET"`, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "bridge2", + }, + Spec: v1alpha1.BridgeSpec{ + SourceTenant: "tenant1", + TargetTenant: "tenant3", + Condition: `attributes["parsed"]["method"] == "PUT"`, + }, }, }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "bridge2", + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + "tenant1": {}, + "tenant2": { + { + Namespace: "ns2", + Name: "sub2", + }, }, - Spec: v1alpha1.BridgeSpec{ - SourceTenant: "tenant1", - TargetTenant: "tenant3", - Condition: `attributes["parsed"]["method"] == "PUT"`, + "tenant3": { + { + Namespace: "ns3", + Name: "sub3", + }, }, }, - }, - TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ - "tenant1": {}, - "tenant2": { + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ { Namespace: "ns2", Name: "sub2", + }: { + { + Namespace: "xy", + Name: "zq", + }, }, - }, - "tenant3": { { Namespace: "ns3", Name: "sub3", + }: { + { + Namespace: "xy", + Name: "zq", + }, }, }, }, - SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ - { - Namespace: "ns2", - Name: "sub2", - }: { - { - Namespace: "xy", - Name: "zq", - }, - }, - { - Namespace: "ns3", - Name: "sub3", - }: { - { - Namespace: "xy", - Name: "zq", - }, - }, - }, - OutputsWithSecretData: nil, - Debug: false, + Debug: false, }, expectedPipelines: map[string]*otelv1beta1.Pipeline{ "logs/tenant_tenant1": { diff --git a/internal/controller/telemetry/otel_conf_gen/validator/validator.go b/internal/controller/telemetry/otel_conf_gen/validator/validator.go index 6ab87ac..eb5f49b 100644 --- a/internal/controller/telemetry/otel_conf_gen/validator/validator.go +++ b/internal/controller/telemetry/otel_conf_gen/validator/validator.go @@ -16,6 +16,7 @@ package validator import ( "fmt" + "strings" "emperror.dev/errors" "github.com/mitchellh/mapstructure" @@ -122,7 +123,12 @@ func decodeTelemetryConfig(anyConfig otelv1beta1.AnyConfig) (telemetry.Config, e func decodeExtensionsConfig(extensionsConfig []string) extensions.Config { var result extensions.Config for _, extName := range extensionsConfig { - result = append(result, component.NewID(component.MustNewType(extName))) + parts := strings.Split(extName, "/") + if len(parts) != 2 { + result = append(result, component.MustNewID(extName)) + } else { + result = append(result, component.MustNewIDWithName(parts[0], parts[1])) + } } return result diff --git a/internal/controller/telemetry/pipeline/components/common.go b/internal/controller/telemetry/pipeline/components/common.go index 4d6ba18..d40df5a 100644 --- a/internal/controller/telemetry/pipeline/components/common.go +++ b/internal/controller/telemetry/pipeline/components/common.go @@ -16,6 +16,8 @@ package components import ( "fmt" + "slices" + "strings" "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -46,3 +48,65 @@ func GetExporterNameForOutput(output v1alpha1.Output) string { return exporterName } + +type ResourceRelations struct { + // These must only include resources that are selected by the collector, tenant labelselectors, and listed outputs in the subscriptions + Tenants []v1alpha1.Tenant + Subscriptions map[v1alpha1.NamespacedName]v1alpha1.Subscription + Bridges []v1alpha1.Bridge + OutputsWithSecretData []OutputWithSecretData + // Subscriptions map, where the key is the Tenants' name, value is a slice of subscriptions' namespaced name + TenantSubscriptionMap map[string][]v1alpha1.NamespacedName + SubscriptionOutputMap map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName +} + +// FindOutputsForSubscription retrieves all outputs for a given subscription +func (r *ResourceRelations) FindOutputsForSubscription(subscription v1alpha1.NamespacedName) []v1alpha1.NamespacedName { + outputs, exists := r.SubscriptionOutputMap[subscription] + if !exists { + return nil + } + + return outputs +} + +// IsOutputInSubscription checks if a specific output belongs to a subscription +func (r *ResourceRelations) IsOutputInSubscription(subscription, output v1alpha1.NamespacedName) bool { + return slices.Contains(r.FindOutputsForSubscription(subscription), output) +} + +// FindTenantForOutput determines which tenant owns a specific output +func (r *ResourceRelations) FindTenantForOutput(targetOutput v1alpha1.NamespacedName) (*v1alpha1.Tenant, error) { + for tenantName, tenantSubscriptions := range r.TenantSubscriptionMap { + // Check each of the tenant's subscriptions + for _, subscription := range tenantSubscriptions { + // Check if this output belongs to the current subscription + if r.IsOutputInSubscription(subscription, targetOutput) { + tenant, err := r.GetTenantByName(tenantName) + if err != nil { + return tenant, err + } + + return tenant, nil + } + } + } + + return nil, fmt.Errorf("tenant for output %s not found", targetOutput) +} + +func (r *ResourceRelations) GetTenantByName(tenantName string) (*v1alpha1.Tenant, error) { + for _, tenant := range r.Tenants { + if tenant.Name == tenantName { + return &tenant, nil + } + } + + return nil, fmt.Errorf("tenant %s not found", tenantName) +} + +func SortNamespacedNames(names []v1alpha1.NamespacedName) { + slices.SortFunc(names, func(a, b v1alpha1.NamespacedName) int { + return strings.Compare(a.String(), b.String()) + }) +} diff --git a/internal/controller/telemetry/pipeline/components/connector/routing_connector.go b/internal/controller/telemetry/pipeline/components/connector/routing_connector.go index 29ea3df..ffd92e4 100644 --- a/internal/controller/telemetry/pipeline/components/connector/routing_connector.go +++ b/internal/controller/telemetry/pipeline/components/connector/routing_connector.go @@ -19,7 +19,6 @@ import ( "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components" - "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/utils" otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" ) @@ -57,7 +56,7 @@ func newRoutingConnector(name string, tenantRouteConfig v1alpha1.RouteConfig) Ro func GenerateRoutingConnectorForTenantsSubscriptions(tenantName string, tenantRouteConfig v1alpha1.RouteConfig, subscriptionNames []v1alpha1.NamespacedName, subscriptions map[v1alpha1.NamespacedName]v1alpha1.Subscription) RoutingConnector { rc := newRoutingConnector(fmt.Sprintf("routing/tenant_%s_subscriptions", tenantName), tenantRouteConfig) - utils.SortNamespacedNames(subscriptionNames) + components.SortNamespacedNames(subscriptionNames) seenConditionsPipelineMap := make(map[string][]string) for _, subscriptionRef := range subscriptionNames { subscription, ok := subscriptions[subscriptionRef] @@ -77,7 +76,7 @@ func GenerateRoutingConnectorForTenantsSubscriptions(tenantName string, tenantRo func GenerateRoutingConnectorForSubscriptionsOutputs(subscriptionRef v1alpha1.NamespacedName, outputNames []v1alpha1.NamespacedName) RoutingConnector { rc := newRoutingConnector(fmt.Sprintf("routing/subscription_%s_%s_outputs", subscriptionRef.Namespace, subscriptionRef.Name), v1alpha1.RouteConfig{}) - utils.SortNamespacedNames(outputNames) + components.SortNamespacedNames(outputNames) pipelines := []string{} for _, outputRef := range outputNames { pipelines = append(pipelines, fmt.Sprintf("logs/output_%s_%s_%s_%s", subscriptionRef.Namespace, subscriptionRef.Name, outputRef.Namespace, outputRef.Name)) diff --git a/internal/controller/telemetry/pipeline/components/exporter/common.go b/internal/controller/telemetry/pipeline/components/exporter/common.go new file mode 100644 index 0000000..9a0a222 --- /dev/null +++ b/internal/controller/telemetry/pipeline/components/exporter/common.go @@ -0,0 +1,99 @@ +// Copyright © 2024 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exporter + +import ( + "time" + + "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" + "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/utils" +) + +type queueWrapper struct { + // Enabled indicates whether to not enqueue batches before sending to the consumerSender. + Enabled *bool `json:"enabled,omitempty"` + + // NumConsumers is the number of consumers from the queue. + NumConsumers *int `json:"num_consumers,omitempty"` + + // QueueSize is the maximum number of batches allowed in queue at a given time. + // Default value is 100. + QueueSize *int `json:"queue_size,omitempty"` + + // If Storage is not empty, enables the persistent storage and uses the component specified + // as a storage extension for the persistent queue. + Storage *string `json:"storage,omitempty"` +} + +func (q *queueWrapper) setDefaultQueueSettings(apiQueueSettings *v1alpha1.QueueSettings) { + q.Enabled = utils.ToPtr(true) + q.QueueSize = utils.ToPtr(100) + + if apiQueueSettings != nil { + if apiQueueSettings.NumConsumers != nil { + q.NumConsumers = apiQueueSettings.NumConsumers + } + if apiQueueSettings.QueueSize != nil { + q.QueueSize = apiQueueSettings.QueueSize + } + } +} + +type backOffWrapper struct { + // Enabled indicates whether to retry sending the batch to the consumerSender in case of a failure. + Enabled *bool `json:"enabled,omitempty"` + + // InitialInterval the time to wait after the first failure before retrying. + InitialInterval *time.Duration `json:"initial_interval,omitempty"` + + // RandomizationFactor is a random factor used to calculate next backoffs + // Randomized interval = RetryInterval * (1 ± RandomizationFactor) + RandomizationFactor *string `json:"randomization_factor,omitempty"` + + // Multiplier is the value multiplied by the backoff interval bounds + Multiplier *string `json:"multiplier,omitempty"` + + // MaxInterval is the upper bound on backoff interval. Once this value is reached the delay between + // consecutive retries will always be `MaxInterval`. + MaxInterval *time.Duration `json:"max_interval,omitempty"` + + // MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. + // Once this value is reached, the data is discarded. If set to 0, the retries are never stopped. + // Default value is 0 to ensure that the data is never discarded. + MaxElapsedTime *time.Duration `json:"max_elapsed_time,omitempty"` +} + +func (b *backOffWrapper) setDefaultBackOffConfig(apiBackOffConfig *v1alpha1.BackOffConfig) { + b.Enabled = utils.ToPtr(true) + b.MaxElapsedTime = utils.ToPtr(0 * time.Second) + + if apiBackOffConfig != nil { + if apiBackOffConfig.InitialInterval != nil { + b.InitialInterval = apiBackOffConfig.InitialInterval + } + if apiBackOffConfig.RandomizationFactor != nil { + b.RandomizationFactor = apiBackOffConfig.RandomizationFactor + } + if apiBackOffConfig.Multiplier != nil { + b.Multiplier = apiBackOffConfig.Multiplier + } + if apiBackOffConfig.MaxInterval != nil { + b.MaxInterval = apiBackOffConfig.MaxInterval + } + if apiBackOffConfig.MaxElapsedTime != nil { + b.MaxElapsedTime = apiBackOffConfig.MaxElapsedTime + } + } +} diff --git a/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter.go b/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter.go index 2bb6c25..836dbc4 100644 --- a/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter.go +++ b/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter.go @@ -20,17 +20,74 @@ import ( "errors" "fmt" + "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components" + "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/utils" "sigs.k8s.io/controller-runtime/pkg/log" ) -func GenerateFluentforwardExporters(ctx context.Context, outputsWithSecretData []components.OutputWithSecretData) map[string]any { +type FluentForwardWrapper struct { + v1alpha1.TCPClientSettings `json:",inline"` + + // RequireAck enables the acknowledgement feature. + RequireAck *bool `json:"require_ack,omitempty"` + + // The Fluent tag parameter used for routing + Tag *string `json:"tag,omitempty"` + + // CompressGzip enables gzip compression for the payload. + CompressGzip *bool `json:"compress_gzip,omitempty"` + + // DefaultLabelsEnabled is a map of default attributes to be added to each log record. + DefaultLabelsEnabled *map[string]bool `json:"default_labels_enabled,omitempty"` + + QueueConfig *queueWrapper `json:"sending_queue,omitempty"` + RetryConfig *backOffWrapper `json:"retry_on_failure,omitempty"` + Kubernetes *v1alpha1.KubernetesMetadata `json:"kubernetes_metadata,omitempty"` +} + +func (w *FluentForwardWrapper) mapToFluentForwardWrapper(userConfig *v1alpha1.Fluentforward) { + w.QueueConfig = &queueWrapper{} + w.RetryConfig = &backOffWrapper{} + w.QueueConfig.setDefaultQueueSettings(userConfig.QueueConfig) + w.RetryConfig.setDefaultBackOffConfig(userConfig.RetryConfig) + + if userConfig.RequireAck != nil { + w.RequireAck = userConfig.RequireAck + } + if userConfig.Tag != nil { + w.Tag = userConfig.Tag + } + if userConfig.CompressGzip != nil { + w.CompressGzip = userConfig.CompressGzip + } + if userConfig.DefaultLabelsEnabled != nil { + w.DefaultLabelsEnabled = userConfig.DefaultLabelsEnabled + } + if userConfig.Kubernetes != nil { + w.Kubernetes = userConfig.Kubernetes + } + w.TCPClientSettings = userConfig.TCPClientSettings +} + +func GenerateFluentforwardExporters(ctx context.Context, resourceRelations components.ResourceRelations) map[string]any { logger := log.FromContext(ctx) result := make(map[string]any) - for _, output := range outputsWithSecretData { + for _, output := range resourceRelations.OutputsWithSecretData { if output.Output.Spec.Fluentforward != nil { - fluentForwardMarshaled, err := json.Marshal(output.Output.Spec.Fluentforward) + internalConfig := FluentForwardWrapper{} + internalConfig.mapToFluentForwardWrapper(output.Output.Spec.Fluentforward) + tenant, err := resourceRelations.FindTenantForOutput(output.Output.NamespacedName()) + if err != nil { + logger.Error(err, "failed to find tenant for output, skipping", "output", output.Output.NamespacedName().String()) + continue + } + if tenant.Spec.PersistenceConfig.EnableFileStorage { + internalConfig.QueueConfig.Storage = utils.ToPtr(fmt.Sprintf("file_storage/%s", tenant.Name)) + } + + fluentForwardMarshaled, err := json.Marshal(internalConfig) if err != nil { logger.Error(errors.New("failed to compile config for output"), "failed to compile config for output %q", output.Output.NamespacedName().String()) } diff --git a/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter_test.go b/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter_test.go index 37bbae0..d1a385f 100644 --- a/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter_test.go +++ b/internal/controller/telemetry/pipeline/components/exporter/fluent_forward_exporter_test.go @@ -16,6 +16,7 @@ package exporter import ( "context" + "fmt" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,76 +27,208 @@ import ( "github.com/stretchr/testify/assert" ) +const testTenantName = "tenant1" + func TestGenerateFluentforwardExporters(t *testing.T) { tests := []struct { - name string - outputsWithSecretData []components.OutputWithSecretData - expectedResult map[string]any + name string + resourceRelations components.ResourceRelations + expectedResult map[string]any }{ { name: "Valid config", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output1", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub1", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - Fluentforward: &v1alpha1.Fluentforward{ - TCPClientSettings: v1alpha1.TCPClientSettings{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output1", + Namespace: "default", }, }, }, }, }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output1", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + Fluentforward: &v1alpha1.Fluentforward{ + TCPClientSettings: v1alpha1.TCPClientSettings{ + Endpoint: &v1alpha1.Endpoint{ + TCPAddr: utils.ToPtr("http://example.com"), + }, + }, + }, + }, + }, + }, + }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub1", + Namespace: "default", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub1", + Namespace: "default", + }: { + { + Name: "output1", + Namespace: "default", + }, + }, + }, }, expectedResult: map[string]any{ "fluentforwardexporter/default_output1": map[string]any{ - "endpoint": "http://example.com", + "endpoint": map[string]any{ + "tcp_addr": "http://example.com", + "validate_tcp_resolution": false, + }, + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "All fields set, tls settings omitted", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { ObjectMeta: metav1.ObjectMeta{ - Name: "output3", + Name: "sub1", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - Fluentforward: &v1alpha1.Fluentforward{ - TCPClientSettings: v1alpha1.TCPClientSettings{ - Endpoint: utils.ToPtr("http://example.com"), - ConnectionTimeout: utils.ToPtr("30s"), - SharedKey: utils.ToPtr("shared-key"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output2", + Namespace: "default", + }, + }, + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output2", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + Fluentforward: &v1alpha1.Fluentforward{ + TCPClientSettings: v1alpha1.TCPClientSettings{ + Endpoint: &v1alpha1.Endpoint{ + TCPAddr: utils.ToPtr("http://example.com"), + }, + ConnectionTimeout: utils.ToPtr("30s"), + SharedKey: utils.ToPtr("shared-key"), + }, + RequireAck: utils.ToPtr(true), + Tag: utils.ToPtr("tag"), + CompressGzip: utils.ToPtr(true), + DefaultLabelsEnabled: &map[string]bool{"label1": true}, + QueueConfig: &v1alpha1.QueueSettings{}, + RetryConfig: &v1alpha1.BackOffConfig{}, + Kubernetes: &v1alpha1.KubernetesMetadata{Key: "key", IncludePodLabels: true}, }, - RequireAck: utils.ToPtr(true), - Tag: utils.ToPtr("tag"), - CompressGzip: utils.ToPtr(true), - DefaultLabelsEnabled: &map[string]bool{"label1": true}, - QueueConfig: &v1alpha1.QueueSettings{}, - RetryConfig: &v1alpha1.BackOffConfig{}, - Kubernetes: &v1alpha1.KubernetesMetadata{Key: "key", IncludePodLabels: true}, }, }, }, }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub1", + Namespace: "default", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub1", + Namespace: "default", + }: { + { + Name: "output2", + Namespace: "default", + }, + }, + }, }, expectedResult: map[string]any{ - "fluentforwardexporter/default_output3": map[string]any{ - "endpoint": "http://example.com", + "fluentforwardexporter/default_output2": map[string]any{ + "endpoint": map[string]any{ + "tcp_addr": "http://example.com", + "validate_tcp_resolution": false, + }, "connection_timeout": "30s", "shared_key": "shared-key", "require_ack": true, "tag": "tag", "compress_gzip": true, "default_labels_enabled": map[string]any{"label1": true}, - "sending_queue": map[string]any{}, - "retry_on_failure": map[string]any{}, + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, "kubernetes_metadata": map[string]any{ "key": "key", "include_pod_labels": true, @@ -108,7 +241,7 @@ func TestGenerateFluentforwardExporters(t *testing.T) { for _, tt := range tests { ttp := tt t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, ttp.expectedResult, GenerateFluentforwardExporters(context.TODO(), ttp.outputsWithSecretData)) + assert.Equal(t, ttp.expectedResult, GenerateFluentforwardExporters(context.TODO(), ttp.resourceRelations)) }) } } diff --git a/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter.go b/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter.go index f6c9184..ba06de7 100644 --- a/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter.go +++ b/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter.go @@ -26,22 +26,52 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -func GenerateOTLPGRPCExporters(ctx context.Context, outputsWithSecretData []components.OutputWithSecretData) map[string]any { +type OTLPGRPCWrapper struct { + QueueConfig *queueWrapper `json:"sending_queue,omitempty"` + RetryConfig *backOffWrapper `json:"retry_on_failure,omitempty"` + v1alpha1.TimeoutSettings `json:",inline"` + v1alpha1.GRPCClientConfig `json:",inline"` +} + +func (w *OTLPGRPCWrapper) mapToOTLPGRPCWrapper(apiConfig *v1alpha1.OTLPGRPC) { + w.QueueConfig = &queueWrapper{} + w.RetryConfig = &backOffWrapper{} + w.QueueConfig.setDefaultQueueSettings(apiConfig.QueueConfig) + w.RetryConfig.setDefaultBackOffConfig(apiConfig.RetryConfig) + + if apiConfig.TimeoutSettings.Timeout != nil { + w.TimeoutSettings.Timeout = apiConfig.TimeoutSettings.Timeout + } + w.GRPCClientConfig = apiConfig.GRPCClientConfig +} + +func GenerateOTLPGRPCExporters(ctx context.Context, resourceRelations components.ResourceRelations) map[string]any { logger := log.FromContext(ctx) result := make(map[string]any) - for _, output := range outputsWithSecretData { + for _, output := range resourceRelations.OutputsWithSecretData { if output.Output.Spec.OTLPGRPC != nil { + internalConfig := OTLPGRPCWrapper{} + internalConfig.mapToOTLPGRPCWrapper(output.Output.Spec.OTLPGRPC) + tenant, err := resourceRelations.FindTenantForOutput(output.Output.NamespacedName()) + if err != nil { + logger.Error(err, "failed to find tenant for output, skipping", "output", output.Output.NamespacedName().String()) + continue + } + if tenant.Spec.PersistenceConfig.EnableFileStorage { + internalConfig.QueueConfig.Storage = utils.ToPtr(fmt.Sprintf("file_storage/%s", tenant.Name)) + } + if output.Output.Spec.Authentication != nil { if output.Output.Spec.Authentication.BasicAuth != nil { - output.Output.Spec.OTLPGRPC.Auth = &v1alpha1.Authentication{ + internalConfig.Auth = &v1alpha1.Authentication{ AuthenticatorID: utils.ToPtr(fmt.Sprintf("basicauth/%s_%s", output.Output.Namespace, output.Output.Name))} } else if output.Output.Spec.Authentication.BearerAuth != nil { - output.Output.Spec.OTLPGRPC.Auth = &v1alpha1.Authentication{ + internalConfig.Auth = &v1alpha1.Authentication{ AuthenticatorID: utils.ToPtr(fmt.Sprintf("bearertokenauth/%s_%s", output.Output.Namespace, output.Output.Name))} } } - otlpGrpcValuesMarshaled, err := json.Marshal(output.Output.Spec.OTLPGRPC) + otlpGrpcValuesMarshaled, err := json.Marshal(internalConfig) if err != nil { logger.Error(errors.New("failed to compile config for output"), "failed to compile config for output %q", output.Output.NamespacedName().String()) } diff --git a/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter_test.go b/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter_test.go index 53b896e..1d6b099 100644 --- a/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter_test.go +++ b/internal/controller/telemetry/pipeline/components/exporter/otlp_exporter_test.go @@ -16,6 +16,7 @@ package exporter import ( "context" + "fmt" "testing" "time" @@ -31,45 +32,97 @@ import ( func TestGenerateOTLPGRPCExporters(t *testing.T) { tests := []struct { - name string - outputsWithSecretData []components.OutputWithSecretData - expectedResult map[string]any + name string + resourceRelations components.ResourceRelations + expectedResult map[string]any }{ { name: "Basic auth", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output1", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub1", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPGRPC: &v1alpha1.OTLPGRPC{ - GRPCClientConfig: v1alpha1.GRPCClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output1", + Namespace: "default", }, }, - Authentication: &v1alpha1.OutputAuth{ - BasicAuth: &v1alpha1.BasicAuthConfig{ - SecretRef: &corev1.SecretReference{ - Name: "secret-name", - Namespace: "secret-ns", + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output1", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPGRPC: &v1alpha1.OTLPGRPC{ + GRPCClientConfig: v1alpha1.GRPCClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + }, + }, + Authentication: &v1alpha1.OutputAuth{ + BasicAuth: &v1alpha1.BasicAuthConfig{ + SecretRef: &corev1.SecretReference{ + Name: "secret-name", + Namespace: "secret-ns", + }, + UsernameField: "username", + PasswordField: "password", }, - UsernameField: "username", - PasswordField: "password", }, }, }, + Secret: corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: "secret-ns", + }, + Data: map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + }, + }, }, - Secret: corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret-name", - Namespace: "secret-ns", + }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub1", + Namespace: "default", }, - Data: map[string][]byte{ - "username": []byte("user"), - "password": []byte("pass"), + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub1", + Namespace: "default", + }: { + { + Name: "output1", + Namespace: "default", }, }, }, @@ -80,42 +133,103 @@ func TestGenerateOTLPGRPCExporters(t *testing.T) { "auth": map[string]any{ "authenticator": "basicauth/default_output1", }, + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "Bearer auth", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output2", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub2", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPGRPC: &v1alpha1.OTLPGRPC{ - GRPCClientConfig: v1alpha1.GRPCClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output2", + Namespace: "default", }, }, - Authentication: &v1alpha1.OutputAuth{ - BearerAuth: &v1alpha1.BearerAuthConfig{ - SecretRef: &corev1.SecretReference{ - Name: "secret-name", - Namespace: "secret-ns", + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output2", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPGRPC: &v1alpha1.OTLPGRPC{ + GRPCClientConfig: v1alpha1.GRPCClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + }, + }, + Authentication: &v1alpha1.OutputAuth{ + BearerAuth: &v1alpha1.BearerAuthConfig{ + SecretRef: &corev1.SecretReference{ + Name: "secret-name", + Namespace: "secret-ns", + }, + TokenField: "token", }, - TokenField: "token", }, }, }, + Secret: corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: "secret-ns", + }, + Data: map[string][]byte{ + "token": []byte("token-value"), + }, + }, }, - Secret: corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret-name", - Namespace: "secret-ns", + }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub2", + Namespace: "default", }, - Data: map[string][]byte{ - "token": []byte("token-value"), + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub2", + Namespace: "default", + }: { + { + Name: "output2", + Namespace: "default", }, }, }, @@ -126,88 +240,207 @@ func TestGenerateOTLPGRPCExporters(t *testing.T) { "auth": map[string]any{ "authenticator": "bearertokenauth/default_output2", }, + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "No auth", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output3", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub3", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPGRPC: &v1alpha1.OTLPGRPC{ - GRPCClientConfig: v1alpha1.GRPCClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output3", + Namespace: "default", + }, + }, + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output3", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPGRPC: &v1alpha1.OTLPGRPC{ + GRPCClientConfig: v1alpha1.GRPCClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + }, }, }, }, }, }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub3", + Namespace: "default", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub3", + Namespace: "default", + }: { + { + Name: "output3", + Namespace: "default", + }, + }, + }, }, expectedResult: map[string]any{ "otlp/default_output3": map[string]any{ "endpoint": "http://example.com", + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "All fields set", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output4", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub4", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPGRPC: &v1alpha1.OTLPGRPC{ - QueueConfig: &v1alpha1.QueueSettings{ - Enabled: true, - NumConsumers: 10, - QueueSize: 100, - StorageID: "storage-id", - }, - RetryConfig: &v1alpha1.BackOffConfig{ - Enabled: true, - InitialInterval: 5 * time.Second, - RandomizationFactor: "0.1", - Multiplier: "2.0", - MaxInterval: 10 * time.Second, - MaxElapsedTime: 60 * time.Second, + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output4", + Namespace: "default", }, - TimeoutSettings: v1alpha1.TimeoutSettings{ - Timeout: utils.ToPtr(5 * time.Second), - }, - GRPCClientConfig: v1alpha1.GRPCClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), - Compression: utils.ToPtr(configcompression.Type("gzip")), - TLSSetting: &v1alpha1.TLSClientSetting{ - Insecure: true, - InsecureSkipVerify: true, - ServerName: "server-name", + }, + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output4", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPGRPC: &v1alpha1.OTLPGRPC{ + QueueConfig: &v1alpha1.QueueSettings{ + NumConsumers: utils.ToPtr(10), + QueueSize: utils.ToPtr(100), + }, + RetryConfig: &v1alpha1.BackOffConfig{ + InitialInterval: utils.ToPtr(5 * time.Second), + RandomizationFactor: utils.ToPtr("0.1"), + Multiplier: utils.ToPtr("2.0"), + MaxInterval: utils.ToPtr(10 * time.Second), + MaxElapsedTime: utils.ToPtr(60 * time.Second), }, - Keepalive: &v1alpha1.KeepaliveClientConfig{ - Time: 5 * time.Second, - Timeout: 5 * time.Second, - PermitWithoutStream: true, + TimeoutSettings: v1alpha1.TimeoutSettings{ + Timeout: utils.ToPtr(5 * time.Second), }, - ReadBufferSize: utils.ToPtr(1024), - WriteBufferSize: utils.ToPtr(1024), - WaitForReady: utils.ToPtr(true), - Headers: &map[string]string{ - "header1": "value1", + GRPCClientConfig: v1alpha1.GRPCClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + Compression: utils.ToPtr(configcompression.Type("gzip")), + TLSSetting: &v1alpha1.TLSClientSetting{ + Insecure: true, + InsecureSkipVerify: true, + ServerName: "server-name", + }, + Keepalive: &v1alpha1.KeepaliveClientConfig{ + Time: 5 * time.Second, + Timeout: 5 * time.Second, + PermitWithoutStream: true, + }, + ReadBufferSize: utils.ToPtr(1024), + WriteBufferSize: utils.ToPtr(1024), + WaitForReady: utils.ToPtr(true), + Headers: &map[string]string{ + "header1": "value1", + }, + BalancerName: utils.ToPtr("round_robin"), + Authority: utils.ToPtr("authority"), }, - BalancerName: utils.ToPtr("round_robin"), - Authority: utils.ToPtr("authority"), }, }, }, }, }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub4", + Namespace: "default", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub4", + Namespace: "default", + }: { + { + Name: "output4", + Namespace: "default", + }, + }, + }, }, expectedResult: map[string]any{ "otlp/default_output4": map[string]any{ @@ -235,7 +468,7 @@ func TestGenerateOTLPGRPCExporters(t *testing.T) { "enabled": true, "num_consumers": float64(10), "queue_size": float64(100), - "storage": "storage-id", + "storage": fmt.Sprintf("file_storage/%s", testTenantName), }, "retry_on_failure": map[string]any{ "enabled": true, @@ -254,7 +487,7 @@ func TestGenerateOTLPGRPCExporters(t *testing.T) { for _, tt := range tests { ttp := tt t.Run(ttp.name, func(t *testing.T) { - assert.Equal(t, ttp.expectedResult, GenerateOTLPGRPCExporters(context.TODO(), ttp.outputsWithSecretData)) + assert.Equal(t, ttp.expectedResult, GenerateOTLPGRPCExporters(context.TODO(), ttp.resourceRelations)) }) } } diff --git a/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter.go b/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter.go index c03f1ae..ac38274 100644 --- a/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter.go +++ b/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter.go @@ -26,22 +26,47 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -func GenerateOTLPHTTPExporters(ctx context.Context, outputsWithSecretData []components.OutputWithSecretData) map[string]any { +type OTLPHTTPWrapper struct { + QueueConfig *queueWrapper `json:"sending_queue,omitempty"` + RetryConfig *backOffWrapper `json:"retry_on_failure,omitempty"` + v1alpha1.HTTPClientConfig `json:",inline"` +} + +func (w *OTLPHTTPWrapper) mapToOTLPHTTPWrapper(apiConfig *v1alpha1.OTLPHTTP) { + w.QueueConfig = &queueWrapper{} + w.RetryConfig = &backOffWrapper{} + w.QueueConfig.setDefaultQueueSettings(apiConfig.QueueConfig) + w.RetryConfig.setDefaultBackOffConfig(apiConfig.RetryConfig) + w.HTTPClientConfig = apiConfig.HTTPClientConfig +} + +func GenerateOTLPHTTPExporters(ctx context.Context, resourceRelations components.ResourceRelations) map[string]any { logger := log.FromContext(ctx) result := make(map[string]any) - for _, output := range outputsWithSecretData { + for _, output := range resourceRelations.OutputsWithSecretData { if output.Output.Spec.OTLPHTTP != nil { + internalConfig := OTLPHTTPWrapper{} + internalConfig.mapToOTLPHTTPWrapper(output.Output.Spec.OTLPHTTP) + tenant, err := resourceRelations.FindTenantForOutput(output.Output.NamespacedName()) + if err != nil { + logger.Error(err, "failed to find tenant for output, skipping", "output", output.Output.NamespacedName().String()) + continue + } + if tenant.Spec.PersistenceConfig.EnableFileStorage { + internalConfig.QueueConfig.Storage = utils.ToPtr(fmt.Sprintf("file_storage/%s", tenant.Name)) + } + if output.Output.Spec.Authentication != nil { if output.Output.Spec.Authentication.BasicAuth != nil { - output.Output.Spec.OTLPHTTP.Auth = &v1alpha1.Authentication{ + internalConfig.Auth = &v1alpha1.Authentication{ AuthenticatorID: utils.ToPtr(fmt.Sprintf("basicauth/%s_%s", output.Output.Namespace, output.Output.Name))} } else if output.Output.Spec.Authentication.BearerAuth != nil { - output.Output.Spec.OTLPHTTP.Auth = &v1alpha1.Authentication{ + internalConfig.Auth = &v1alpha1.Authentication{ AuthenticatorID: utils.ToPtr(fmt.Sprintf("bearertokenauth/%s_%s", output.Output.Namespace, output.Output.Name))} } } - otlpHttpValuesMarshaled, err := json.Marshal(output.Output.Spec.OTLPHTTP) + otlpHttpValuesMarshaled, err := json.Marshal(internalConfig) if err != nil { logger.Error(errors.New("failed to compile config for output"), "failed to compile config for output %q", output.Output.NamespacedName().String()) } diff --git a/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter_test.go b/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter_test.go index a66c2b8..074694f 100644 --- a/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter_test.go +++ b/internal/controller/telemetry/pipeline/components/exporter/otlphttp_exporter_test.go @@ -16,6 +16,7 @@ package exporter import ( "context" + "fmt" "testing" "time" @@ -32,45 +33,97 @@ import ( func TestGenerateOTLPHTTPExporters(t *testing.T) { tests := []struct { - name string - outputsWithSecretData []components.OutputWithSecretData - expectedResult map[string]any + name string + resourceRelations components.ResourceRelations + expectedResult map[string]any }{ { name: "Basic auth", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output1", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub1", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPHTTP: &v1alpha1.OTLPHTTP{ - HTTPClientConfig: v1alpha1.HTTPClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output1", + Namespace: "default", }, }, - Authentication: &v1alpha1.OutputAuth{ - BasicAuth: &v1alpha1.BasicAuthConfig{ - SecretRef: &corev1.SecretReference{ - Name: "secret-name", - Namespace: "secret-ns", + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output1", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPHTTP: &v1alpha1.OTLPHTTP{ + HTTPClientConfig: v1alpha1.HTTPClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + }, + }, + Authentication: &v1alpha1.OutputAuth{ + BasicAuth: &v1alpha1.BasicAuthConfig{ + SecretRef: &corev1.SecretReference{ + Name: "secret-name", + Namespace: "secret-ns", + }, + UsernameField: "username", + PasswordField: "password", }, - UsernameField: "username", - PasswordField: "password", }, }, }, + Secret: corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: "secret-ns", + }, + Data: map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + }, + }, }, - Secret: corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret-name", - Namespace: "secret-ns", + }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub1", + Namespace: "default", }, - Data: map[string][]byte{ - "username": []byte("user"), - "password": []byte("pass"), + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub1", + Namespace: "default", + }: { + { + Name: "output1", + Namespace: "default", }, }, }, @@ -81,42 +134,103 @@ func TestGenerateOTLPHTTPExporters(t *testing.T) { "auth": map[string]any{ "authenticator": "basicauth/default_output1", }, + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "Bearer auth", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output2", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub2", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPHTTP: &v1alpha1.OTLPHTTP{ - HTTPClientConfig: v1alpha1.HTTPClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output2", + Namespace: "default", }, }, - Authentication: &v1alpha1.OutputAuth{ - BearerAuth: &v1alpha1.BearerAuthConfig{ - SecretRef: &corev1.SecretReference{ - Name: "secret-name", - Namespace: "secret-ns", + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output2", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPHTTP: &v1alpha1.OTLPHTTP{ + HTTPClientConfig: v1alpha1.HTTPClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + }, + }, + Authentication: &v1alpha1.OutputAuth{ + BearerAuth: &v1alpha1.BearerAuthConfig{ + SecretRef: &corev1.SecretReference{ + Name: "secret-name", + Namespace: "secret-ns", + }, + TokenField: "token", }, - TokenField: "token", }, }, }, + Secret: corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: "secret-ns", + }, + Data: map[string][]byte{ + "token": []byte("token-value"), + }, + }, }, - Secret: corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret-name", - Namespace: "secret-ns", + }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub2", + Namespace: "default", }, - Data: map[string][]byte{ - "token": []byte("token-value"), + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub2", + Namespace: "default", + }: { + { + Name: "output2", + Namespace: "default", }, }, }, @@ -127,86 +241,205 @@ func TestGenerateOTLPHTTPExporters(t *testing.T) { "auth": map[string]any{ "authenticator": "bearertokenauth/default_output2", }, + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "No auth", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output3", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub3", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPHTTP: &v1alpha1.OTLPHTTP{ - HTTPClientConfig: v1alpha1.HTTPClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output3", + Namespace: "default", + }, + }, + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output3", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPHTTP: &v1alpha1.OTLPHTTP{ + HTTPClientConfig: v1alpha1.HTTPClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + }, }, }, }, }, }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub3", + Namespace: "default", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub3", + Namespace: "default", + }: { + { + Name: "output3", + Namespace: "default", + }, + }, + }, }, expectedResult: map[string]any{ "otlphttp/default_output3": map[string]any{ "endpoint": "http://example.com", + "sending_queue": map[string]any{ + "enabled": true, + "queue_size": float64(100), + "storage": fmt.Sprintf("file_storage/%s", testTenantName), + }, + "retry_on_failure": map[string]any{ + "enabled": true, + "max_elapsed_time": float64(0), + }, }, }, }, { name: "All fields set", - outputsWithSecretData: []components.OutputWithSecretData{ - { - Output: v1alpha1.Output{ + resourceRelations: components.ResourceRelations{ + Tenants: []v1alpha1.Tenant{ + { ObjectMeta: metav1.ObjectMeta{ - Name: "output4", + Name: testTenantName, + }, + Spec: v1alpha1.TenantSpec{ + PersistenceConfig: v1alpha1.PersistenceConfig{ + EnableFileStorage: true, + }, + }, + }, + }, + Subscriptions: map[v1alpha1.NamespacedName]v1alpha1.Subscription{ + { + Name: "default", + Namespace: "default", + }: { + ObjectMeta: metav1.ObjectMeta{ + Name: "sub4", Namespace: "default", }, - Spec: v1alpha1.OutputSpec{ - OTLPHTTP: &v1alpha1.OTLPHTTP{ - QueueConfig: &v1alpha1.QueueSettings{ - Enabled: true, - NumConsumers: 10, - QueueSize: 100, - StorageID: "storage-id", + Spec: v1alpha1.SubscriptionSpec{ + Outputs: []v1alpha1.NamespacedName{ + { + Name: "output4", + Namespace: "default", }, - RetryConfig: &v1alpha1.BackOffConfig{ - Enabled: true, - InitialInterval: 5 * time.Second, - RandomizationFactor: "0.1", - Multiplier: "2.0", - MaxInterval: 10 * time.Second, - MaxElapsedTime: 60 * time.Second, - }, - HTTPClientConfig: v1alpha1.HTTPClientConfig{ - Endpoint: utils.ToPtr("http://example.com"), - ProxyURL: utils.ToPtr("http://proxy.example.com"), - TLSSetting: &v1alpha1.TLSClientSetting{ - Insecure: true, - InsecureSkipVerify: true, - ServerName: "server-name", + }, + }, + }, + }, + OutputsWithSecretData: []components.OutputWithSecretData{ + { + Output: v1alpha1.Output{ + ObjectMeta: metav1.ObjectMeta{ + Name: "output4", + Namespace: "default", + }, + Spec: v1alpha1.OutputSpec{ + OTLPHTTP: &v1alpha1.OTLPHTTP{ + QueueConfig: &v1alpha1.QueueSettings{ + NumConsumers: utils.ToPtr(10), + QueueSize: utils.ToPtr(100), + }, + RetryConfig: &v1alpha1.BackOffConfig{ + InitialInterval: utils.ToPtr(5 * time.Second), + RandomizationFactor: utils.ToPtr("0.1"), + Multiplier: utils.ToPtr("2.0"), + MaxInterval: utils.ToPtr(10 * time.Second), + MaxElapsedTime: utils.ToPtr(60 * time.Second), }, - ReadBufferSize: utils.ToPtr(1024), - WriteBufferSize: utils.ToPtr(1024), - Timeout: utils.ToPtr(5 * time.Second), - Headers: &map[string]configopaque.String{ - "header1": configopaque.String("value1"), + HTTPClientConfig: v1alpha1.HTTPClientConfig{ + Endpoint: utils.ToPtr("http://example.com"), + ProxyURL: utils.ToPtr("http://proxy.example.com"), + TLSSetting: &v1alpha1.TLSClientSetting{ + Insecure: true, + InsecureSkipVerify: true, + ServerName: "server-name", + }, + ReadBufferSize: utils.ToPtr(1024), + WriteBufferSize: utils.ToPtr(1024), + Timeout: utils.ToPtr(5 * time.Second), + Headers: &map[string]configopaque.String{ + "header1": configopaque.String("value1"), + }, + Compression: utils.ToPtr(configcompression.Type("gzip")), + MaxIdleConns: utils.ToPtr(10), + MaxIdleConnsPerHost: utils.ToPtr(10), + MaxConnsPerHost: utils.ToPtr(10), + IdleConnTimeout: utils.ToPtr(5 * time.Second), + DisableKeepAlives: utils.ToPtr(true), + HTTP2ReadIdleTimeout: utils.ToPtr(5 * time.Second), + HTTP2PingTimeout: utils.ToPtr(5 * time.Second), }, - Compression: utils.ToPtr(configcompression.Type("gzip")), - MaxIdleConns: utils.ToPtr(10), - MaxIdleConnsPerHost: utils.ToPtr(10), - MaxConnsPerHost: utils.ToPtr(10), - IdleConnTimeout: utils.ToPtr(5 * time.Second), - DisableKeepAlives: utils.ToPtr(true), - HTTP2ReadIdleTimeout: utils.ToPtr(5 * time.Second), - HTTP2PingTimeout: utils.ToPtr(5 * time.Second), }, }, }, }, }, + TenantSubscriptionMap: map[string][]v1alpha1.NamespacedName{ + testTenantName: { + { + Name: "sub4", + Namespace: "default", + }, + }, + }, + SubscriptionOutputMap: map[v1alpha1.NamespacedName][]v1alpha1.NamespacedName{ + { + Name: "sub4", + Namespace: "default", + }: { + { + Name: "output4", + Namespace: "default", + }, + }, + }, }, expectedResult: map[string]any{ "otlphttp/default_output4": map[string]any{ @@ -235,7 +468,7 @@ func TestGenerateOTLPHTTPExporters(t *testing.T) { "enabled": true, "num_consumers": float64(10), "queue_size": float64(100), - "storage": "storage-id", + "storage": fmt.Sprintf("file_storage/%s", testTenantName), }, "retry_on_failure": map[string]any{ "enabled": true, @@ -253,7 +486,7 @@ func TestGenerateOTLPHTTPExporters(t *testing.T) { for _, tt := range tests { ttp := tt t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, ttp.expectedResult, GenerateOTLPHTTPExporters(context.TODO(), ttp.outputsWithSecretData)) + assert.Equal(t, ttp.expectedResult, GenerateOTLPHTTPExporters(context.TODO(), ttp.resourceRelations)) }) } } diff --git a/internal/controller/telemetry/pipeline/components/extension/storage/filestorage.go b/internal/controller/telemetry/pipeline/components/extension/storage/filestorage.go new file mode 100644 index 0000000..cae4b8f --- /dev/null +++ b/internal/controller/telemetry/pipeline/components/extension/storage/filestorage.go @@ -0,0 +1,50 @@ +// Copyright © 2024 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "os" + "path/filepath" + "runtime" +) + +const ( + defaultFileStorageDirectory = "/var/lib/otelcol/file_storage" +) + +var ( + defaultFileStorageDirectoryWindows = filepath.Join(os.Getenv("ProgramData"), "Otelcol", "FileStorage") +) + +func GenerateFileStorageExtensionForTenant(persistDirPath string, tenantName string) map[string]any { + return map[string]any{ + "create_directory": true, + "directory": DetermineFileStorageDirectory(persistDirPath, tenantName), + } +} + +func DetermineFileStorageDirectory(persistDirPath string, tenantName string) string { + if persistDirPath != "" { + return persistDirPath + } + + switch runtime.GOOS { + case "windows": + return fmt.Sprintf("%s/%s", defaultFileStorageDirectoryWindows, tenantName) + default: + return fmt.Sprintf("%s/%s", defaultFileStorageDirectory, tenantName) + } +} diff --git a/internal/controller/telemetry/pipeline/components/receiver/filelog_receiver.go b/internal/controller/telemetry/pipeline/components/receiver/filelog_receiver.go index c25a0e0..f202c19 100644 --- a/internal/controller/telemetry/pipeline/components/receiver/filelog_receiver.go +++ b/internal/controller/telemetry/pipeline/components/receiver/filelog_receiver.go @@ -14,9 +14,13 @@ package receiver -import "fmt" +import ( + "fmt" -func GenerateDefaultKubernetesReceiver(namespaces []string) map[string]any { + "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" +) + +func GenerateDefaultKubernetesReceiver(namespaces []string, tenant v1alpha1.Tenant) map[string]any { // TODO: fix parser-crio operators := []map[string]any{ { @@ -98,18 +102,8 @@ func GenerateDefaultKubernetesReceiver(namespaces []string) map[string]any { }, } - includeList := make([]string, 0, len(namespaces)) - if len(namespaces) > 0 { - for _, ns := range namespaces { - include := fmt.Sprintf("/var/log/pods/%s_*/*/*.log", ns) - includeList = append(includeList, include) - } - } else { - includeList = append(includeList, "/var/log/pods/*/*/*.log") - } - k8sReceiver := map[string]any{ - "include": includeList, + "include": createIncludeList(namespaces, tenant.Spec.SelectFromAllNamespaces), "exclude": []string{"/var/log/pods/*/otc-container/*.log"}, "start_at": "end", "include_file_path": true, @@ -120,6 +114,24 @@ func GenerateDefaultKubernetesReceiver(namespaces []string) map[string]any { "max_elapsed_time": 0, }, } + if tenant.Spec.EnableFileStorage { + k8sReceiver["storage"] = fmt.Sprintf("file_storage/%s", tenant.Name) + } return k8sReceiver } + +func createIncludeList(namespaces []string, includeAll bool) []string { + includeList := make([]string, 0, len(namespaces)) + if includeAll { + return []string{"/var/log/pods/*/*/*.log"} + } + + for _, ns := range namespaces { + if ns != "" { + includeList = append(includeList, fmt.Sprintf("/var/log/pods/%s_*/*/*.log", ns)) + } + } + + return includeList +} diff --git a/internal/controller/telemetry/pipeline/pipeline.go b/internal/controller/telemetry/pipeline/pipeline.go index 956c024..742d100 100644 --- a/internal/controller/telemetry/pipeline/pipeline.go +++ b/internal/controller/telemetry/pipeline/pipeline.go @@ -36,11 +36,12 @@ func GenerateRootPipeline(tenants []v1alpha1.Tenant, tenantName string) *otelv1b for _, tenant := range tenants { if tenant.Name == tenantName { // Add filelog receiver to tenant's pipeline if it has any logsource namespace selectors - if tenant.Spec.LogSourceNamespaceSelectors != nil { + // or if it selects from all namespaces + if tenant.Status.LogSourceNamespaces != nil || tenant.Spec.SelectFromAllNamespaces { receiverName = fmt.Sprintf("filelog/%s", tenantName) } // Add routing connector to tenant's pipeline if it has any subscription namespace selectors - if tenant.Spec.SubscriptionNamespaceSelectors != nil { + if tenant.Status.Subscriptions != nil { exporterName = fmt.Sprintf("routing/tenant_%s_subscriptions", tenantName) } } diff --git a/internal/controller/telemetry/route_controller.go b/internal/controller/telemetry/route_controller.go index 1a3df3e..18bbe17 100644 --- a/internal/controller/telemetry/route_controller.go +++ b/internal/controller/telemetry/route_controller.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" - "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/utils" + "github.com/kube-logging/telemetry-controller/internal/controller/telemetry/pipeline/components" ) const ( @@ -94,7 +94,7 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl r.disownSubscriptions(ctx, subscriptionsToDisown) subscriptionNames := getSubscriptionNamesFromSubscription(subscriptionsForTenant) - utils.SortNamespacedNames(subscriptionNames) + components.SortNamespacedNames(subscriptionNames) tenant.Status.Subscriptions = subscriptionNames for _, subscription := range subscriptionsForTenant { diff --git a/internal/controller/telemetry/utils/utils.go b/internal/controller/telemetry/utils/utils.go index 9ca23da..634af24 100644 --- a/internal/controller/telemetry/utils/utils.go +++ b/internal/controller/telemetry/utils/utils.go @@ -14,19 +14,6 @@ package utils -import ( - "slices" - "strings" - - "github.com/kube-logging/telemetry-controller/api/telemetry/v1alpha1" -) - -func SortNamespacedNames(names []v1alpha1.NamespacedName) { - slices.SortFunc(names, func(a, b v1alpha1.NamespacedName) int { - return strings.Compare(a.String(), b.String()) - }) -} - func ToPtr[T any](v T) *T { return &v }