Skip to content

Commit

Permalink
fix: Issue with match generator alongside each generator
Browse files Browse the repository at this point in the history
  • Loading branch information
codingconcepts committed Feb 2, 2024
1 parent 95ace59 commit 44c2c44
Show file tree
Hide file tree
Showing 7 changed files with 108 additions and 11 deletions.
16 changes: 13 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ data_unique_test:
data_const_test:
go run dg.go -c ./examples/const_test/config.yaml -o ./csvs/const_test

data_match:
go run dg.go -c ./examples/match_test/config.yaml -o ./csvs/match -i import.sql

data_each_match:
go run dg.go -c ./examples/each_match_test/config.yaml -o ./csvs/each_match -i import.sql

data: data_many_to_many data_person data_range_test data_input_test data_unique_test data_const_test
echo "done"

Expand All @@ -53,9 +59,13 @@ release: validate_version
GOOS=linux go build -ldflags "-X main.version=${VERSION}" -o dg ;\
tar -zcvf ./releases/dg_${VERSION}_linux.tar.gz ./dg ;\

# macos
GOOS=darwin go build -ldflags "-X main.version=${VERSION}" -o dg ;\
tar -zcvf ./releases/dg_${VERSION}_macOS.tar.gz ./dg ;\
# macos (arm)
GOOS=darwin GOARCH=arm64 go build -ldflags "-X main.version=${VERSION}" -o dg ;\
tar -zcvf ./releases/dg_${VERSION}_macos_arm64.tar.gz ./dg ;\

# macos (amd)
GOOS=darwin GOARCH=amd64 go build -ldflags "-X main.version=${VERSION}" -o dg ;\
tar -zcvf ./releases/dg_${VERSION}_macos_amd64.tar.gz ./dg ;\

# windows
GOOS=windows go build -ldflags "-X main.version=${VERSION}" -o dg ;\
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@ tables:
match_column: timeline_date
```

dg will match rows in the significant_event table with rows in the event column based on the match between `significant_event.date` and `events.date`, and take the value from the `significant_events.events` column where there's a match (otherwise leaving `NULL`). This will result in the following `events` table being generated:
dg will match rows in the significant_event table with rows in the events table based on the match between `significant_event.date` and `events.timeline_date`, and take the value from the `significant_events.event` column where there's a match (otherwise leaving `NULL`). This will result in the following `events` table being generated:

|timeline_date|timeline_event|
|----|-----|
Expand Down
46 changes: 46 additions & 0 deletions examples/each_match_test/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
inputs:

- name: market
type: csv
source:
file_name: market.csv

tables:

- name: product
count: 1
unique_columns: [name]
columns:
- name: id
type: gen
processor:
value: ${uuid}
- name: name
type: gen
processor:
value: ${adjective} ${adverb} ${noun}

- name: market_product
columns:
- name: id
type: gen
processor:
value: ${uuid}
- name: product_id
type: each
processor:
table: product
column: id
- name: market
type: each
processor:
table: market
column: code
- name: region
type: match
processor:
source_table: market
source_column: code
source_value: region
match_column: market

3 changes: 3 additions & 0 deletions examples/each_match_test/market.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
code,region
us,us-east-1
uk,eu-west-1
28 changes: 28 additions & 0 deletions examples/match_test/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
inputs:

- name: market
type: csv
source:
file_name: market.csv

tables:

- name: market_product
count: 10
columns:
- name: id
type: gen
processor:
value: ${uuid}
- name: market
type: set
processor:
values: ["us", "in"]
- name: region
type: match
processor:
source_table: market
source_column: code
source_value: region
match_column: market

3 changes: 3 additions & 0 deletions examples/match_test/market.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
code,region
us,us-east-1
in,ap-south-1
21 changes: 14 additions & 7 deletions internal/pkg/generator/match_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,31 @@ func (g MatchGenerator) Generate(t model.Table, c model.Column, files map[string
valueColumnIndex := lo.IndexOf(sourceTable.Header, g.SourceValue)
valueColumn := sourceTable.Lines[valueColumnIndex]

sourceMap := map[string]string{}
for i := 0; i < len(sourceColumn); i++ {
sourceMap[sourceColumn[i]] = valueColumn[i]
}

matchTable, ok := files[t.Name]
if !ok {
return fmt.Errorf("missing destination table %q for match lookup", t.Name)
}
_, matchColumnIndex, ok := lo.FindIndexOf(t.Columns, func(c model.Column) bool {
return c.Name == g.MatchColumn

// Use the match table headers to determine index, as the each processor
// will re-order columns.
_, matchColumnIndex, ok := lo.FindIndexOf(matchTable.Header, func(c string) bool {
return c == g.MatchColumn
})
if !ok {
return fmt.Errorf("missing match column %q in current table", g.MatchColumn)
}

matchColumn := matchTable.Lines[matchColumnIndex]

lines := make([]string, len(matchColumn))
for sourceI, sourceC := range sourceColumn {
if _, i, ok := lo.FindIndexOf(matchColumn, func(matchCol string) bool {
return matchCol == sourceC
}); ok {
lines[i] = valueColumn[sourceI]
for i, matchC := range matchColumn {
if sourceValue, ok := sourceMap[matchC]; ok {
lines[i] = sourceValue
}
}

Expand Down

0 comments on commit 44c2c44

Please sign in to comment.