move Search ETL config to separate struct

PDOK-17304
PDOK · Nov 27, 2024 · 1e605dd · 1e605dd
1 parent 587a490
commit 1e605dd
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 18 deletions.
diff --git a/config/collections.go b/config/collections.go
@@ -71,20 +71,24 @@ type Search struct {
 	// Template that indicates how a search record is displayed. Uses Go text/template syntax to reference fields.
 	DisplayNameTemplate string `yaml:"displayNameTemplate,omitempty" json:"displayNameTemplate,omitempty" validate:"required"`
 
-	// One or more optional templates that make up the autosuggestions. Uses Go text/template syntax to reference fields.
-	SuggestTemplates []string `yaml:"suggestTemplates,omitempty" json:"suggestTemplates,omitempty"`
-
-	// SQLite WHERE clause to filter features when importing/ETL-ing
-	// (Without the WHERE keyword, only the clause)
-	// +Optional
-	ETLFilter string `yaml:"etlFilter,omitempty" json:"etlFilter,omitempty"`
-
 	// Version of the collection used to link to search results
 	Version int `yaml:"version,omitempty" json:"version,omitempty" default:"1"`
 
 	// (Links to) the individual OGC API (feature) collections that are searchable in this collection.
 	// +kubebuilder:validation:MinItems=1
 	OGCCollections []RelatedOGCAPIFeaturesCollection `yaml:"ogcCollections" json:"ogcCollections" validate:"required,min=1"`
+
+	ETL SearchETL `yaml:"etl" json:"etl" validate:"required"`
+}
+
+type SearchETL struct {
+	// One or more optional templates that make up the autosuggestions. Uses Go text/template syntax to reference fields.
+	SuggestTemplates []string `yaml:"suggestTemplates,omitempty" json:"suggestTemplates,omitempty"`
+
+	// SQLite WHERE clause to filter features when importing/ETL-ing
+	// (Without the WHERE keyword, only the clause)
+	// +Optional
+	Filter string `yaml:"filter,omitempty" json:"filter,omitempty"`
 }
 
 type RelatedOGCAPIFeaturesCollection struct {

diff --git a/internal/engine/testdata/config_full.yaml b/internal/engine/testdata/config_full.yaml
@@ -32,9 +32,10 @@ collections:
         - huisletter
         - postcode
         - woonplaats
-      suggestTemplates:
-        - "{{.straatnaam}} {{.huisnummer}} {{.huisletter}} {{.postcode}} {{.woonplaats}}"
-        - "{{.straatnaam}} {{.huisnummer}}{{.huisletter}} {{.postcode}} {{.woonplaats}}"
+      etl:
+        suggestTemplates:
+          - "{{.straatnaam}} {{.huisnummer}} {{.huisletter}} {{.postcode}} {{.woonplaats}}"
+          - "{{.straatnaam}} {{.huisnummer}}{{.huisletter}} {{.postcode}} {{.woonplaats}}"
       version: 1
       links: {}  # placeholder
       ogcCollections:

diff --git a/internal/etl/etl.go b/internal/etl/etl.go
@@ -81,7 +81,7 @@ func ImportFile(cfg *config.Config, searchIndex string, filePath string, table c
 	// import records in batches depending on page size
 	offset := 0
 	for {
-		sourceRecords, err := source.Extract(table, collection.Search.Fields, collection.Search.ETLFilter, pageSize, offset)
+		sourceRecords, err := source.Extract(table, collection.Search.Fields, collection.Search.ETL.Filter, pageSize, offset)
 		if err != nil {
 			return fmt.Errorf("failed extracting source records: %w", err)
 		}

diff --git a/internal/etl/etl_test.go b/internal/etl/etl_test.go
@@ -86,7 +86,7 @@ func TestImportGeoPackage(t *testing.T) {
 		assert.NotNil(t, cfg)
 		for _, collection := range cfg.Collections {
 			if collection.Search != nil {
-				collection.Search.ETLFilter = tt.where
+				collection.Search.ETL.Filter = tt.where
 			}
 		}
 

diff --git a/internal/etl/testdata/config.yaml b/internal/etl/testdata/config.yaml
@@ -22,9 +22,10 @@ collections:
         - component_postaldescriptor
         - component_addressareaname
       displayNameTemplate: "{{ .component_thoroughfarename }} - {{ .component_addressareaname }}"
-      suggestTemplates:
-        - "{{ .component_thoroughfarename }} {{ .component_addressareaname }}"
-        - "{{ .component_thoroughfarename }}, {{ .component_postaldescriptor }} {{ .component_addressareaname }}"
+      etl:
+        suggestTemplates:
+          - "{{ .component_thoroughfarename }} {{ .component_addressareaname }}"
+          - "{{ .component_thoroughfarename }}, {{ .component_postaldescriptor }} {{ .component_addressareaname }}"
       ogcCollections:
         - api: https://example.com/ogc/v1
           collection: addresses

diff --git a/internal/etl/transform/transform.go b/internal/etl/transform/transform.go
@@ -45,8 +45,8 @@ func (t Transformer) Transform(records []RawRecord, collection config.GeoSpatial
 		if err != nil {
 			return nil, err
 		}
-		suggestions := make([]string, 0, len(collection.Search.SuggestTemplates))
-		for _, suggestTemplate := range collection.Search.SuggestTemplates {
+		suggestions := make([]string, 0, len(collection.Search.ETL.SuggestTemplates))
+		for _, suggestTemplate := range collection.Search.ETL.SuggestTemplates {
 			suggestion, err := t.renderTemplate(suggestTemplate, fieldValuesByName)
 			if err != nil {
 				return nil, err