From c8aa463b96049dcd39f6fadd73eed06c50c2f25c Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Sat, 20 Apr 2024 10:18:59 -0500 Subject: [PATCH 1/4] feat(forms)!: Improved, unified form with dbt profile browsing Leveraged deeper understanding of huh form library to combine all the forms into one, with conditional visibility based on previous answers. Added ability to browse dbt profiles rather than trying to recall the name. Added more explicit error handling when we can't parse the dbt profile. --- fetch_dbt_profiles.go | 50 ++++ fetch_dbt_profiles_test.go | 28 +++ forms.go | 337 ++++++++++++--------------- get_dbt_profile.go | 48 +--- get_dbt_profile_test.go | 47 ++-- go.mod | 3 +- go.sum | 2 - main.go | 29 ++- set_connection_details.go | 62 ++--- set_connection_details_test.go | 32 ++- sourcerer/get_conn_test.go | 12 +- sourcerer/get_sources_tables_test.go | 14 +- sourcerer/put_columns_on_tables.go | 1 - test_helpers.go | 2 +- 14 files changed, 348 insertions(+), 319 deletions(-) create mode 100644 fetch_dbt_profiles.go create mode 100644 fetch_dbt_profiles_test.go diff --git a/fetch_dbt_profiles.go b/fetch_dbt_profiles.go new file mode 100644 index 0000000..28fd197 --- /dev/null +++ b/fetch_dbt_profiles.go @@ -0,0 +1,50 @@ +package main + +import ( + "log" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +type DbtProfile struct { + Target string `yaml:"target"` + Outputs map[string]struct { + ConnType string `yaml:"type"` + Account string `yaml:"account"` + User string `yaml:"user"` + Role string `yaml:"role"` + Authenticator string `yaml:"authenticator"` + Database string `yaml:"database"` + Schema string `yaml:"schema"` + Project string `yaml:"project"` + Dataset string `yaml:"dataset"` + Path string `yaml:"path"` + Threads int `yaml:"threads"` + } `yaml:"outputs"` +} + +type DbtProfiles map[string]DbtProfile + +func FetchDbtProfiles() (DbtProfiles, error) { + paths := []string{ + filepath.Join(".", "profiles.yml"), + filepath.Join(os.Getenv("HOME"), ".dbt", "profiles.yml"), + } + ps := DbtProfiles{} + for _, path := range paths { + pf := DbtProfiles{} + yf, err := os.ReadFile(path) + if err != nil { + continue + } + if err = yaml.Unmarshal(yf, pf); err != nil { + log.Fatalf("Could not read dbt profile, \nlikely unsupported fields or formatting issues\n please open an issue: %v\n", err) + } + for k, v := range pf { + ps[k] = v + } + } + return ps, nil +} diff --git a/fetch_dbt_profiles_test.go b/fetch_dbt_profiles_test.go new file mode 100644 index 0000000..e826364 --- /dev/null +++ b/fetch_dbt_profiles_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "os" + "testing" +) + +func TestFetchDbtProfiles(t *testing.T) { + CreateTempDbtProfiles(t) + defer os.RemoveAll(os.Getenv("HOME")) + defer os.Unsetenv("HOME") + profiles, err := FetchDbtProfiles() + if err != nil { + t.Fatalf("Error fetching dbt profiles: %v\n", err) + } + if err != nil { + t.Fatalf("Error fetching dbt profiles: %v\n", err) + } + if profiles["elf"].Outputs["dev"].ConnType != "snowflake" { + t.Fatalf("Expected snowflake, got %s\n", profiles["elf"].Outputs["dev"].ConnType) + } + if profiles["human"].Outputs["dev"].ConnType != "bigquery" { + t.Fatalf("Expected bigquery, got %s\n", profiles["human"].Outputs["dev"].ConnType) + } + if profiles["dwarf"].Outputs["dev"].ConnType != "duckdb" { + t.Fatalf("Expected duckdb, got %s\n", profiles["dwarf"].Outputs["dev"].ConnType) + } +} diff --git a/forms.go b/forms.go index 8c314f5..48e5871 100644 --- a/forms.go +++ b/forms.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "github.com/charmbracelet/huh" ) @@ -21,7 +20,7 @@ type FormResponse struct { GenerateDescriptions bool GroqKeyEnvVar string UseDbtProfile bool - DbtProfile string + DbtProfileName string DbtProfileOutput string CreateProfile bool ScaffoldProject bool @@ -29,94 +28,98 @@ type FormResponse struct { Prefix string } -func Forms() (formResponse FormResponse) { - formResponse = FormResponse{} - introForm := huh.NewForm( +var not_empty = func(s string) error { + if len(s) == 0 { + return fmt.Errorf("cannot be empty, please enter a value") + } + return nil +} + +func getProfileOptions(ps DbtProfiles) []huh.Option[string] { + var po []huh.Option[string] + for k := range ps { + po = append(po, huh.Option[string]{ + Key: k, + Value: k, + }) + } + return po +} + +func Forms(ps DbtProfiles) (FormResponse, error) { + dfr := FormResponse{ + BuildDir: "build", + GroqKeyEnvVar: "GROQ_API_KEY", + Prefix: "stg", + } + err := huh.NewForm( huh.NewGroup( huh.NewNote(). Title("🏁 Welcome to tbd! 🏎️✨"). Description(fmt.Sprintf(`A sweet and speedy code generator for dbt. ¸.•✴︎•.¸.•✴︎•.¸.•✴︎•. _%s_ .•✴︎•.¸.•✴︎•.¸.•✴︎•.¸ -Currently supports *Snowflake*, *BigQuery*, and *DuckDB*. - -Generates: -✴︎ YAML sources config -✴︎ SQL staging models -For each table in the designated schema/dataset. - To prepare, make sure you have the following: -✴︎ The name of an existing dbt profile to reference -(Can be found in the profiles.yml file) -*_OR_* -✴︎ The necessary connection details for your warehouse -_Authentication must be handled via SSO._ -_For security, we don't support password auth._ +✴︎ The name of an *_existing dbt profile_* to reference +*_OR_* +✴︎ The necessary *_connection details_* for your warehouse -Platform-specific requirements: -✴︎ _Snowflake_: externalbrowser auth -✴︎ _BigQuery_: gcloud CLI installed and authed -✴︎ _DuckDB_: none if using a local db +_See README for warehouse-specific requirements_ `, Version)), ), - huh.NewGroup( - huh.NewNote(). - Title("🤖 Experimental: LLM Generation 🦙✨"). - Description(`*_Optional_* LLM-powered alpha features. - -Currently generates: -✴︎ column _descriptions_ -✴︎ relevant _tests_ -via the Groq API. -You'll need: -✴︎ A Groq API key -✴︎ Key stored in env var`), - huh.NewConfirm().Affirmative("Sure!").Negative("Nope"). - Title("Do you want to generate column descriptions and tests via LLM?"). - Value(&formResponse.GenerateDescriptions), - ), huh.NewGroup( - huh.NewConfirm().Affirmative("Yes!").Negative("Nah"). + huh.NewConfirm(). Title("Do you have a dbt profile you'd like to connect with?\n(you can enter your credentials manually if not)"). - Value(&formResponse.UseDbtProfile), - huh.NewConfirm().Affirmative("Yeah!").Negative("Nope"). + Value(&dfr.UseDbtProfile), + huh.NewConfirm(). Title("Would you like to scaffold a basic dbt project into the output directory?"). - Value(&formResponse.ScaffoldProject), + Value(&dfr.ScaffoldProject), huh.NewInput(). Title("What prefix do you want to use for your staging files?"). - Value(&formResponse.Prefix). - Placeholder("stg"), + Value(&dfr.Prefix). + Placeholder("stg"). + Validate(not_empty), ), - ) - projectNameForm := huh.NewForm( + huh.NewGroup(huh.NewInput(). Title("What is the name of your dbt project?"). - Value(&formResponse.ProjectName). - Placeholder("gondor_patrol_analytics"), - )) - profileCreateForm := huh.NewForm( + Value(&dfr.ProjectName). + Placeholder("gondor_patrol_analytics"). + Validate(not_empty), + ).WithHideFunc(func() bool { + return !dfr.ScaffoldProject + }), + huh.NewGroup( - huh.NewConfirm().Affirmative("Yes, pls").Negative("No, thx"). - Title("Would you like to generate a profiles.yml file from the info you provide next?"). - Value(&formResponse.CreateProfile), - )) - dbtForm := huh.NewForm( + huh.NewConfirm(). + Title("Would you like to generate a profiles.yml file dfrom the info you provide next?"). + Value(&dfr.CreateProfile), + ).WithHideFunc(func() bool { + return dfr.UseDbtProfile + }), + huh.NewGroup( - huh.NewInput(). - Title("What is the dbt profile name you'd like to use?"). - Value(&formResponse.DbtProfile). - Placeholder("snowflake_sandbox"), + huh.NewSelect[string](). + Title("Choose a dbt profile:"). + Options(getProfileOptions(ps)...), huh.NewInput(). Title("Which 'output' in that profile do you want to use?"). - Value(&formResponse.DbtProfileOutput). - Placeholder("dev"), + Value(&dfr.DbtProfileOutput). + Placeholder("dev"). + Validate(not_empty), huh.NewInput(). Title("What schema/dataset do you want to generate?"). - Value(&formResponse.Schema), - ), - ) - warehouseForm := huh.NewForm( + Value(&dfr.Schema). + Validate(not_empty), + huh.NewInput(). + Title("What project/database is that schema/dataset in?"). + Value(&dfr.Schema). + Validate(not_empty), + ).WithHideFunc(func() bool { + return !dfr.UseDbtProfile + }), + huh.NewGroup( huh.NewSelect[string](). Title("Choose your warehouse."). @@ -125,147 +128,111 @@ You'll need: huh.NewOption("BigQuery", "bigquery"), huh.NewOption("DuckDB", "duckdb"), ). - Value(&formResponse.Warehouse), - ), - ) - snowflakeForm := huh.NewForm( + Value(&dfr.Warehouse), + ).WithHideFunc(func() bool { + return dfr.UseDbtProfile + }), + huh.NewGroup( huh.NewInput(). Title("What is your username?"). - Value(&formResponse.Username).Placeholder("aragorn@dunedain.king"), - + Value(&dfr.Username). + Placeholder("aragorn@dunedain.king"). + Validate(not_empty), huh.NewInput(). Title("What is your Snowflake account id?"). - Value(&formResponse.Account).Placeholder("elfstone-consulting.us-west-1"), - + Value(&dfr.Account). + Placeholder("elfstone-consulting.us-west-1"). + Validate(not_empty), huh.NewInput(). Title("What is the schema you want to generate?"). - Value(&formResponse.Schema).Placeholder("minas-tirith"), - + Value(&dfr.Schema). + Placeholder("minas-tirith"). + Validate(not_empty), huh.NewInput(). Title("What database is that schema in?"). - Value(&formResponse.Database).Placeholder("gondor"), - ), - ) - bigqueryForm := huh.NewForm( + Value(&dfr.Database). + Placeholder("gondor"). + Validate(not_empty), + ).WithHideFunc(func() bool { + return dfr.Warehouse != "snowflake" + }), + huh.NewGroup( - huh.NewInput().Title("What is your GCP project's id?"). - Value(&formResponse.Project).Placeholder("legolas_inc"), - huh.NewInput().Title("What is the dataset you want to generate?"). - Value(&formResponse.Dataset).Placeholder("mirkwood"), - ), - ) - duckdbForm := huh.NewForm( + huh.NewInput(). + Title("What is your GCP project's id?"). + Value(&dfr.Project). + Placeholder("legolas_inc"). + Validate(not_empty), + huh.NewInput(). + Title("What is the dataset you want to generate?"). + Value(&dfr.Dataset). + Placeholder("mirkwood"). + Validate(not_empty), + ).WithHideFunc(func() bool { + return dfr.Warehouse != "bigquery" + }), + huh.NewGroup( - huh.NewInput().Title(`What is the path to your DuckDB database? + huh.NewInput(). + Title(`What is the path to your DuckDB database? Relative to pwd e.g. if db is in this dir -> cool_ducks.db`). - Value(&formResponse.Path).Placeholder("/path/to/duckdb.db"), - huh.NewInput().Title("What is the DuckDB database you want to generate?"). - Value(&formResponse.Database).Placeholder("duckdb"), - huh.NewInput().Title("What is the schema you want to generate?"). - Value(&formResponse.Schema).Placeholder("raw"), - ), - ) - llmForm := huh.NewForm( - huh.NewGroup( + Value(&dfr.Path). + Placeholder("/path/to/duckdb.db"). + Validate(not_empty), huh.NewInput(). - Title("What env var holds your Groq key?"). - Placeholder("GROQ_API_KEY"). - Value(&formResponse.GroqKeyEnvVar), - ), - ) - dirForm := huh.NewForm( + Title("What is the DuckDB database you want to generate?"). + Value(&dfr.Database). + Placeholder("duckdb"). + Validate(not_empty), + huh.NewInput(). + Title("What is the schema you want to generate?"). + Value(&dfr.Schema). + Placeholder("raw"). + Validate(not_empty), + ).WithHideFunc(func() bool { + return dfr.Warehouse != "duckdb" + }), + huh.NewGroup( huh.NewNote(). - Title("🚧🚨 Choose your build directory carefully! 🚨🚧"). - Description(`Choose a _new_ or _empty_ directory. -If you choose an existing, populated directory -tbd will _intentionally error out_.`), + Title("🤖 Experimental: LLM Generation 🦙✨"). + Description(`*_Optional_* LLM-powered alpha features, powered by Groq. + +Currently generates: +✴︎ column _descriptions_ +✴︎ relevant _tests_ + +You'll need: +✴︎ A Groq API key in an env var`), + huh.NewConfirm(). + Title("Do you want to generate column descriptions and tests via LLM?"). + Value(&dfr.GenerateDescriptions), ), + huh.NewGroup( huh.NewInput(). - Title("What directory do you want to build into?"). - Value(&formResponse.BuildDir). - Placeholder("build"), - ), - ) - confirmForm := huh.NewForm( + Title("What env var holds your Groq key?"). + Placeholder("GROQ_API_KEY"). + Value(&dfr.GroqKeyEnvVar). + Validate(not_empty), + ).WithHideFunc(func() bool { + return !dfr.GenerateDescriptions + }), + huh.NewGroup( - huh.NewConfirm().Affirmative("Let's go!").Negative("Nevermind"). + huh.NewInput(). + Title("What directory do you want to build into?\n Must be new or empty."). + Value(&dfr.BuildDir). + Placeholder("build"). + Validate(not_empty), + huh.NewConfirm(). Title("🚦Are you ready to do this thing?🚦"). - Value(&formResponse.Confirm), + Value(&dfr.Confirm), ), - ) - introForm.WithTheme(huh.ThemeCatppuccin()) - profileCreateForm.WithTheme(huh.ThemeCatppuccin()) - projectNameForm.WithTheme(huh.ThemeCatppuccin()) - dbtForm.WithTheme(huh.ThemeCatppuccin()) - warehouseForm.WithTheme(huh.ThemeCatppuccin()) - snowflakeForm.WithTheme(huh.ThemeCatppuccin()) - bigqueryForm.WithTheme(huh.ThemeCatppuccin()) - duckdbForm.WithTheme(huh.ThemeCatppuccin()) - llmForm.WithTheme(huh.ThemeCatppuccin()) - dirForm.WithTheme(huh.ThemeCatppuccin()) - confirmForm.WithTheme(huh.ThemeCatppuccin()) - err := introForm.Run() - if err != nil { - log.Fatalf("Error running intro form %v\n", err) - } - if formResponse.UseDbtProfile { - err = dbtForm.Run() - if err != nil { - log.Fatalf("Error running dbt form %v\n", err) - } - } else { - err = profileCreateForm.Run() - if err != nil { - log.Fatalf("Error running profile create form %v\n", err) - } - if formResponse.ScaffoldProject { - err = projectNameForm.Run() - if err != nil { - log.Fatalf("Error running project name form %v\n", err) - } - } - err = warehouseForm.Run() - if err != nil { - log.Fatalf("Error running warehouse form %v\n", err) - } - switch formResponse.Warehouse { - case "snowflake": - err = snowflakeForm.Run() - if err != nil { - log.Fatalf("Error running snowflake form %v\n", err) - } - case "bigquery": - { - err = bigqueryForm.Run() - if err != nil { - log.Fatalf("Error running bigquery form %v\n", err) - } - } - case "duckdb": - { - err = duckdbForm.Run() - if err != nil { - log.Fatalf("Error running duckdb form %v\n", err) - } - } - } - } - if formResponse.GenerateDescriptions { - err = llmForm.Run() - if err != nil { - log.Fatalf("Error running LLM features form %v\n", err) - } - } - err = dirForm.Run() - if err != nil { - log.Fatalf("Error running build directory form %v\n", err) - } - err = confirmForm.Run() + ).WithTheme(huh.ThemeCatppuccin()).Run() if err != nil { - log.Fatalf("Error running confirmation form %v\n", err) + return dfr, err } - return formResponse + return dfr, nil } diff --git a/get_dbt_profile.go b/get_dbt_profile.go index 2f1dcb2..ec1c5f3 100644 --- a/get_dbt_profile.go +++ b/get_dbt_profile.go @@ -2,52 +2,12 @@ package main import ( "fmt" - "log" - "os" - "path/filepath" - - "gopkg.in/yaml.v2" ) -type DbtProfile struct { - Target string `yaml:"target"` - Outputs map[string]struct { - ConnType string `yaml:"type"` - Account string `yaml:"account"` - User string `yaml:"user"` - Role string `yaml:"role"` - Authenticator string `yaml:"authenticator"` - Database string `yaml:"database"` - Schema string `yaml:"schema"` - Project string `yaml:"project"` - Dataset string `yaml:"dataset"` - Path string `yaml:"path"` - Threads int `yaml:"threads"` - } `yaml:"outputs"` -} - -func GetDbtProfile(dbtProfile string) (*DbtProfile, error) { - paths := []string{ - filepath.Join(".", "profiles.yml"), - filepath.Join(os.Getenv("HOME"), ".dbt", "profiles.yml"), - } - profileMap := make(map[string]DbtProfile) - var selectedProfile *DbtProfile - for _, path := range paths { - yamlFile, err := os.ReadFile(path) - if err == nil { - if err := yaml.Unmarshal(yamlFile, &profileMap); err != nil { - log.Fatalf("Could not unmarshal dbt profile: %v\n", err) - } - - if profile, ok := profileMap[dbtProfile]; ok { - selectedProfile = &profile - } - } - } - if selectedProfile != nil { - return selectedProfile, nil +func GetDbtProfile(pn string, ps DbtProfiles) (DbtProfile, error) { + if p, ok := ps[pn]; ok { + return p, nil } else { - return nil, fmt.Errorf("no profile named %s", dbtProfile) + return DbtProfile{}, fmt.Errorf("no profile named %s", pn) } } diff --git a/get_dbt_profile_test.go b/get_dbt_profile_test.go index fb17ec9..06538f7 100644 --- a/get_dbt_profile_test.go +++ b/get_dbt_profile_test.go @@ -6,47 +6,50 @@ import ( ) func TestGetDbtProfile(t *testing.T) { - CreateTempDbtProfile(t) + CreateTempDbtProfiles(t) defer os.RemoveAll(os.Getenv("HOME")) defer os.Unsetenv("HOME") - - // Profile exists - profile, err := GetDbtProfile("elf") + ps, err := FetchDbtProfiles() + if err != nil { + t.Errorf("Error fetching dbt profiles: %v", err) + } + // Profile exists + p, err := GetDbtProfile("elf", ps) if err != nil { t.Errorf("GetDbtProfile returned an error for an existing profile: %v", err) } - if profile.Target != "dev" { - t.Errorf("Expected target 'dev', got '%s'", profile.Target) + if p.Target != "dev" { + t.Errorf("Expected target 'dev', got '%s'", p.Target) } - if profile.Outputs["dev"].ConnType != "snowflake" { - t.Errorf("Expected connection type 'snowflake', got '%s'", profile.Outputs["dev"].ConnType) + if p.Outputs["dev"].ConnType != "snowflake" { + t.Errorf("Expected connection type 'snowflake', got '%s'", p.Outputs["dev"].ConnType) } - // Profile exists, DuckDB - profile, err = GetDbtProfile("dwarf") + // Profile exists, DuckDB + p, err = GetDbtProfile("dwarf", ps) if err != nil { t.Errorf("GetDbtProfile returned an error for an existing profile: %v", err) } - if profile.Target != "dev" { - t.Errorf("Expected target 'dev', got '%s'", profile.Target) + if p.Target != "dev" { + t.Errorf("Expected target 'dev', got '%s'", p.Target) } - if profile.Outputs["dev"].ConnType != "duckdb" { - t.Errorf("Expected connection type 'duckdb', got '%s'", profile.Outputs["dev"].ConnType) + if p.Outputs["dev"].ConnType != "duckdb" { + t.Errorf("Expected connection type 'duckdb', got '%s'", p.Outputs["dev"].ConnType) } - if profile.Outputs["dev"].Schema != "balins_tomb" { - t.Errorf("Expected schema 'balins_tomb', got '%s'", profile.Outputs["dev"].Schema) + if p.Outputs["dev"].Schema != "balins_tomb" { + t.Errorf("Expected schema 'balins_tomb', got '%s'", p.Outputs["dev"].Schema) } - // If using dbt profile with DuckDB, path should be unedited - if profile.Outputs["dev"].Path != "/usr/local/var/dwarf.db" { - t.Errorf("Expected path '/usr/local/var/dwarf.db', got '%s'", profile.Outputs["dev"].Path) + // If using dbt profile with DuckDB, path should be unedited + if p.Outputs["dev"].Path != "/usr/local/var/dwarf.db" { + t.Errorf("Expected path '/usr/local/var/dwarf.db', got '%s'", p.Outputs["dev"].Path) } // Profile does not exist - profile, err = GetDbtProfile("dunedain") + p, err = GetDbtProfile("dunedain", ps) if err == nil { t.Error("GetDbtProfile did not return an error for a non-existing profile") } - if profile != nil { - t.Error("GetDbtProfile returned a non-nil profile for a non-existing profile") + if p.Outputs != nil { + t.Error("GetDbtProfile returned a non-empty profile for a non-existing profile") } } diff --git a/go.mod b/go.mod index 844cda1..84067f6 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( cloud.google.com/go/bigquery v1.60.0 github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/charmbracelet/huh v0.3.1-0.20240306161957-71f31c155b08 - github.com/charmbracelet/huh/spinner v0.0.0-20240306161957-71f31c155b08 github.com/jarcoal/httpmock v1.3.1 github.com/marcboeker/go-duckdb v1.6.3 + github.com/schollz/progressbar/v3 v3.14.2 github.com/snowflakedb/gosnowflake v1.9.0 google.golang.org/api v0.170.0 gopkg.in/yaml.v2 v2.4.0 @@ -84,7 +84,6 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/schollz/progressbar/v3 v3.14.2 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect diff --git a/go.sum b/go.sum index 0300954..bfa66cb 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,6 @@ github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/huh v0.3.1-0.20240306161957-71f31c155b08 h1:2SLao5UVJt6qBafbtLG6Zem7AyTEeOKcipTKK9pstMY= github.com/charmbracelet/huh v0.3.1-0.20240306161957-71f31c155b08/go.mod h1:3bNz/oITPP07NndDV0YFj7mxLytf16ZOHrvhpW2O7CI= -github.com/charmbracelet/huh/spinner v0.0.0-20240306161957-71f31c155b08 h1:kO5eMMxyCJ6m7gdpGQ7OomrMdfsKVPgC4aB/focl/HE= -github.com/charmbracelet/huh/spinner v0.0.0-20240306161957-71f31c155b08/go.mod h1:nrBG0YEHaxdbqHXW1xvG1hPqkuac9Eg7RTMvogiXuz0= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/charmbracelet/x/exp/strings v0.0.0-20240304160204-3835fda67169 h1:1QdJhraY3DLWdZqVys2qYB2MT5/uo/WrpeEjRpPWcvg= diff --git a/main.go b/main.go index 80e2da8..c794e1d 100644 --- a/main.go +++ b/main.go @@ -20,17 +20,24 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - formResponse := Forms() - if !formResponse.Confirm { + ps, err := FetchDbtProfiles() + if err != nil { + log.Fatalf("Error fetching dbt profiles: %v\n", err) + } + fr, err := Forms(ps) + if err != nil { + log.Fatalf("Error running form: %v\n", err) + } + if !fr.Confirm { log.Fatal("⛔ User cancelled.") } - cd := SetConnectionDetails(formResponse) + cd := SetConnectionDetails(fr, ps) e := Elapsed{} e.DbStart = time.Now() - bd := formResponse.BuildDir - err := PrepBuildDir(bd) + bd := fr.BuildDir + err = PrepBuildDir(bd) if err != nil { log.Fatalf("Error preparing build directory: %v\n", err) } @@ -55,23 +62,23 @@ func main() { // End of database interaction, start of processing e.ProcessingStart = time.Now() - if formResponse.GenerateDescriptions { + if fr.GenerateDescriptions { GenerateColumnDescriptions(ts) } - if formResponse.CreateProfile { + if fr.CreateProfile { WriteProfile(cd, bd) } - if formResponse.ScaffoldProject { - s, err := WriteScaffoldProject(cd, bd, formResponse.ProjectName) + if fr.ScaffoldProject { + s, err := WriteScaffoldProject(cd, bd, fr.ProjectName) if err != nil { log.Fatalf("Error scaffolding project: %v\n", err) } bd = s } - err = WriteFiles(ts, bd, formResponse.Prefix) + err = WriteFiles(ts, bd, fr.Prefix) if err != nil { log.Fatalf("Error writing files: %v\n", err) } e.ProcessingElapsed = time.Since(e.ProcessingStart).Seconds() - fmt.Printf("\n🏁 Done in %.1fs fetching data and %.1fs writing files!\nYour YAML and SQL files are in the %s directory.", e.DbElapsed, e.ProcessingElapsed, formResponse.BuildDir) + fmt.Printf("\n🏁 Done in %.1fs fetching data and %.1fs writing files!\nYour YAML and SQL files are in the %s directory.", e.DbElapsed, e.ProcessingElapsed, fr.BuildDir) } diff --git a/set_connection_details.go b/set_connection_details.go index d47308f..dbddd75 100644 --- a/set_connection_details.go +++ b/set_connection_details.go @@ -8,64 +8,64 @@ import ( "github.com/gwenwindflower/tbd/shared" ) -func SetConnectionDetails(formResponse FormResponse) shared.ConnectionDetails { +func SetConnectionDetails(fr FormResponse, ps DbtProfiles) shared.ConnectionDetails { var cd shared.ConnectionDetails - if formResponse.UseDbtProfile { - profile, err := GetDbtProfile(formResponse.DbtProfile) + if fr.UseDbtProfile { + profile, err := GetDbtProfile(fr.DbtProfileName, ps) if err != nil { log.Fatalf("Could not get dbt profile %v\n", err) } - switch profile.Outputs[formResponse.DbtProfileOutput].ConnType { + switch profile.Outputs[fr.DbtProfileOutput].ConnType { case "snowflake": { cd = shared.ConnectionDetails{ - ConnType: profile.Outputs[formResponse.DbtProfileOutput].ConnType, - Username: profile.Outputs[formResponse.DbtProfileOutput].User, - Account: profile.Outputs[formResponse.DbtProfileOutput].Account, - Database: profile.Outputs[formResponse.DbtProfileOutput].Database, - Schema: formResponse.Schema, + ConnType: profile.Outputs[fr.DbtProfileOutput].ConnType, + Username: profile.Outputs[fr.DbtProfileOutput].User, + Account: profile.Outputs[fr.DbtProfileOutput].Account, + Database: profile.Outputs[fr.DbtProfileOutput].Database, + Schema: fr.Schema, } } case "bigquery": { cd = shared.ConnectionDetails{ - ConnType: profile.Outputs[formResponse.DbtProfileOutput].ConnType, - Project: profile.Outputs[formResponse.DbtProfileOutput].Project, - Dataset: formResponse.Schema, + ConnType: profile.Outputs[fr.DbtProfileOutput].ConnType, + Project: profile.Outputs[fr.DbtProfileOutput].Project, + Dataset: fr.Schema, } } case "duckdb": { cd = shared.ConnectionDetails{ - ConnType: profile.Outputs[formResponse.DbtProfileOutput].ConnType, - Path: profile.Outputs[formResponse.DbtProfileOutput].Path, - Database: profile.Outputs[formResponse.DbtProfileOutput].Database, - Schema: formResponse.Schema, + ConnType: profile.Outputs[fr.DbtProfileOutput].ConnType, + Path: profile.Outputs[fr.DbtProfileOutput].Path, + Database: profile.Outputs[fr.DbtProfileOutput].Database, + Schema: fr.Schema, } } default: { - log.Fatalf("Unsupported connection type %v\n", profile.Outputs[formResponse.DbtProfileOutput].ConnType) + log.Fatalf("Unsupported connection type %v\n", profile.Outputs[fr.DbtProfileOutput].ConnType) } } } else { - switch formResponse.Warehouse { + switch fr.Warehouse { case "snowflake": { cd = shared.ConnectionDetails{ - ConnType: formResponse.Warehouse, - Username: formResponse.Username, - Account: formResponse.Account, - Schema: formResponse.Schema, - Database: formResponse.Database, + ConnType: fr.Warehouse, + Username: fr.Username, + Account: fr.Account, + Schema: fr.Schema, + Database: fr.Database, } } case "bigquery": { cd = shared.ConnectionDetails{ - ConnType: formResponse.Warehouse, - Project: formResponse.Project, - Dataset: formResponse.Dataset, + ConnType: fr.Warehouse, + Project: fr.Project, + Dataset: fr.Dataset, } } case "duckdb": @@ -75,15 +75,15 @@ func SetConnectionDetails(formResponse FormResponse) shared.ConnectionDetails { } { cd = shared.ConnectionDetails{ - ConnType: formResponse.Warehouse, - Path: filepath.Join(wd, formResponse.Path), - Database: formResponse.Database, - Schema: formResponse.Schema, + ConnType: fr.Warehouse, + Path: filepath.Join(wd, fr.Path), + Database: fr.Database, + Schema: fr.Schema, } } default: { - log.Fatalf("Unsupported connection type %v\n", formResponse.Warehouse) + log.Fatalf("Unsupported connection type %v\n", fr.Warehouse) } } } diff --git a/set_connection_details_test.go b/set_connection_details_test.go index 49fcdba..3c83412 100644 --- a/set_connection_details_test.go +++ b/set_connection_details_test.go @@ -19,7 +19,11 @@ func TestSetConnectionDetailsWithoutDbtProfile(t *testing.T) { BuildDir: "test_build", Confirm: true, } - connectionDetails := SetConnectionDetails(formResponse) + ps, err := FetchDbtProfiles() + if err != nil { + t.Errorf("Error fetching dbt profiles: %v", err) + } + connectionDetails := SetConnectionDetails(formResponse, ps) want := shared.ConnectionDetails{ ConnType: "snowflake", Username: "aragorn", @@ -33,19 +37,23 @@ func TestSetConnectionDetailsWithoutDbtProfile(t *testing.T) { } func TestSetConnectionDetailsWithDbtProfile(t *testing.T) { - CreateTempDbtProfile(t) + CreateTempDbtProfiles(t) + ps, err := FetchDbtProfiles() + if err != nil { + t.Errorf("Error fetching dbt profiles: %v", err) + } defer os.RemoveAll(os.Getenv("HOME")) defer os.Unsetenv("HOME") formResponse := FormResponse{ UseDbtProfile: true, - DbtProfile: "elf", + DbtProfileName: "elf", DbtProfileOutput: "dev", Schema: "hall_of_thranduil", GenerateDescriptions: false, BuildDir: "test_build", Confirm: true, } - connectionDetails := SetConnectionDetails(formResponse) + connectionDetails := SetConnectionDetails(formResponse, ps) want := shared.ConnectionDetails{ ConnType: "snowflake", Username: "legolas", @@ -59,19 +67,23 @@ func TestSetConnectionDetailsWithDbtProfile(t *testing.T) { } func TestSetConnectionDetailsWithDuckDBDbtProfile(t *testing.T) { - CreateTempDbtProfile(t) + CreateTempDbtProfiles(t) + ps, err := FetchDbtProfiles() + if err != nil { + t.Errorf("Error fetching dbt profiles: %v", err) + } defer os.RemoveAll(os.Getenv("HOME")) defer os.Unsetenv("HOME") formResponse := FormResponse{ UseDbtProfile: true, - DbtProfile: "dwarf", + DbtProfileName: "dwarf", DbtProfileOutput: "dev", Schema: "balins_tomb", GenerateDescriptions: false, BuildDir: "test_build", Confirm: true, } - connectionDetails := SetConnectionDetails(formResponse) + connectionDetails := SetConnectionDetails(formResponse, ps) want := shared.ConnectionDetails{ ConnType: "duckdb", Path: "/usr/local/var/dwarf.db", @@ -94,7 +106,11 @@ func TestSetConnectionDetailsWithDuckDBWithoutDbtProfile(t *testing.T) { BuildDir: "test_build", Confirm: true, } - connectionDetails := SetConnectionDetails(formResponse) + ps, err := FetchDbtProfiles() + if err != nil { + t.Errorf("Error fetching dbt profiles: %v", err) + } + connectionDetails := SetConnectionDetails(formResponse, ps) wd, err := os.Getwd() if err != nil { t.Errorf("Failed to get working directory: %v", err) diff --git a/sourcerer/get_conn_test.go b/sourcerer/get_conn_test.go index d2e9aec..ea35d8f 100644 --- a/sourcerer/get_conn_test.go +++ b/sourcerer/get_conn_test.go @@ -22,11 +22,11 @@ func TestGetConnSnowflake(t *testing.T) { if conn == nil { t.Errorf("GetConn failed: conn is nil") } - SfConn, ok := conn.(*SfConn) + sfc, ok := conn.(*SfConn) if !ok { t.Errorf("GetConn failed: conn is not of type SfConn") } - if SfConn.Account != "DUNEDAIN.SNOWFLAKECOMPUTING.COM" { + if sfc.Account != "DUNEDAIN.SNOWFLAKECOMPUTING.COM" { t.Errorf("GetConn failed: Account is not correct") } } @@ -44,11 +44,11 @@ func TestGetConnBigQuery(t *testing.T) { if conn == nil { t.Errorf("GetConn failed: conn is nil") } - BqConn, ok := conn.(*BqConn) + bqc, ok := conn.(*BqConn) if !ok { t.Errorf("GetConn failed: conn is not of type BqConn") } - if BqConn.Dataset != "hall_of_thranduil" { + if bqc.Dataset != "hall_of_thranduil" { t.Errorf("GetConn failed: Account is not correct") } } @@ -67,11 +67,11 @@ func TestGetConnDuckDB(t *testing.T) { if conn == nil { t.Errorf("GetConn failed: conn is nil") } - DuckConn, ok := conn.(*DuckConn) + dc, ok := conn.(*DuckConn) if !ok { t.Errorf("GetConn failed: conn is not of type DuckConn") } - if DuckConn.Path != "/path/to/duckdb.db" { + if dc.Path != "/path/to/duckdb.db" { t.Errorf("GetConn failed: Account is not correct") } } diff --git a/sourcerer/get_sources_tables_test.go b/sourcerer/get_sources_tables_test.go index 1a83fc9..5dcf30f 100644 --- a/sourcerer/get_sources_tables_test.go +++ b/sourcerer/get_sources_tables_test.go @@ -19,13 +19,15 @@ func TestGetSourceTablesSnowflake(t *testing.T) { Schema: "minas-tirith", } conn, err := GetConn(cd) + // TODO: look at testify to clean up assertions if err != nil { t.Errorf("GetConn failed: %v", err) } if conn == nil { t.Errorf("GetConn failed: conn is nil") } - SfConn, ok := conn.(*SfConn) + // TODO: look for capital vars and lower + sfc, ok := conn.(*SfConn) if !ok { t.Errorf("GetConn failed: conn is not of type SfConn") } @@ -33,12 +35,12 @@ func TestGetSourceTablesSnowflake(t *testing.T) { if err != nil { t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) } - SfConn.Db = db - SfConn.Cancel = cancel - defer SfConn.Db.Close() - q := fmt.Sprintf("SELECT table_name FROM information_schema.tables WHERE table_schema = '%s'", SfConn.Schema) + sfc.Db = db + sfc.Cancel = cancel + defer sfc.Db.Close() + q := fmt.Sprintf("SELECT table_name FROM information_schema.tables WHERE table_schema = '%s'", sfc.Schema) mock.ExpectQuery(q).WillReturnRows(sqlmock.NewRows([]string{"table_name"}).AddRow("table1").AddRow("table2")) - ts, err := SfConn.GetSourceTables(ctx) + ts, err := sfc.GetSourceTables(ctx) if err != nil { t.Errorf("GetSources failed: %v", err) } diff --git a/sourcerer/put_columns_on_tables.go b/sourcerer/put_columns_on_tables.go index f0f5325..934a78d 100644 --- a/sourcerer/put_columns_on_tables.go +++ b/sourcerer/put_columns_on_tables.go @@ -24,7 +24,6 @@ func PutColumnsOnTables(ctx context.Context, ts shared.SourceTables, dbc DbConn) bar := progressbar.NewOptions(len(ts.SourceTables), progressbar.OptionShowCount(), progressbar.OptionShowElapsedTimeOnFinish(), - progressbar.OptionFullWidth(), progressbar.OptionEnableColorCodes(true), progressbar.OptionSetDescription("🏎️✨"), ) diff --git a/test_helpers.go b/test_helpers.go index 888d36b..e44c41b 100644 --- a/test_helpers.go +++ b/test_helpers.go @@ -8,7 +8,7 @@ import ( "github.com/gwenwindflower/tbd/shared" ) -func CreateTempDbtProfile(t *testing.T) string { +func CreateTempDbtProfiles(t *testing.T) string { content := []byte(` elf: target: dev From f1bf806502a13f8f84ac9bbf8058767752c767ab Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Sat, 20 Apr 2024 10:23:28 -0500 Subject: [PATCH 2/4] style(form placeholders): Add placeholders for dbt profile details --- forms.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/forms.go b/forms.go index 48e5871..61652d0 100644 --- a/forms.go +++ b/forms.go @@ -102,7 +102,8 @@ _See README for warehouse-specific requirements_ huh.NewGroup( huh.NewSelect[string](). Title("Choose a dbt profile:"). - Options(getProfileOptions(ps)...), + Options(getProfileOptions(ps)...). + Value(&dfr.DbtProfileName), huh.NewInput(). Title("Which 'output' in that profile do you want to use?"). Value(&dfr.DbtProfileOutput). @@ -111,10 +112,12 @@ _See README for warehouse-specific requirements_ huh.NewInput(). Title("What schema/dataset do you want to generate?"). Value(&dfr.Schema). + Placeholder("raw"). Validate(not_empty), huh.NewInput(). Title("What project/database is that schema/dataset in?"). Value(&dfr.Schema). + Placeholder("jaffle_shop"). Validate(not_empty), ).WithHideFunc(func() bool { return !dfr.UseDbtProfile From 17e78f524a1f9434174242ef8f44b4a9dc97a1e5 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Sat, 20 Apr 2024 10:34:32 -0500 Subject: [PATCH 3/4] fix(form input): Database/Project properly maps to database It was accidentally mapped to schema. --- forms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forms.go b/forms.go index 61652d0..dc93239 100644 --- a/forms.go +++ b/forms.go @@ -116,7 +116,7 @@ _See README for warehouse-specific requirements_ Validate(not_empty), huh.NewInput(). Title("What project/database is that schema/dataset in?"). - Value(&dfr.Schema). + Value(&dfr.Database). Placeholder("jaffle_shop"). Validate(not_empty), ).WithHideFunc(func() bool { From 697258350cb38802bf40da42e07aa8044e376d89 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Sat, 20 Apr 2024 10:38:35 -0500 Subject: [PATCH 4/4] fix(typo in form) --- forms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forms.go b/forms.go index dc93239..a503f8e 100644 --- a/forms.go +++ b/forms.go @@ -93,7 +93,7 @@ _See README for warehouse-specific requirements_ huh.NewGroup( huh.NewConfirm(). - Title("Would you like to generate a profiles.yml file dfrom the info you provide next?"). + Title("Would you like to generate a profiles.yml file?\n(from the info you provide next)"). Value(&dfr.CreateProfile), ).WithHideFunc(func() bool { return dfr.UseDbtProfile