-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
6 changed files
with
123 additions
and
115 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package config | ||
|
|
||
| import "github.com/artie-labs/transfer/lib/config/constants" | ||
|
|
||
| type S3Settings struct { | ||
| FolderName string `yaml:"folderName"` | ||
| Bucket string `yaml:"bucket"` | ||
| AwsAccessKeyID string `yaml:"awsAccessKeyID"` | ||
| AwsSecretAccessKey string `yaml:"awsSecretAccessKey"` | ||
| OutputFormat constants.S3OutputFormat `yaml:"outputFormat"` | ||
| } | ||
|
|
||
| type MSSQL struct { | ||
| Host string `yaml:"host"` | ||
| Port int `yaml:"port"` | ||
| Username string `yaml:"username"` | ||
| Password string `yaml:"password"` | ||
| Database string `yaml:"database"` | ||
| } | ||
|
|
||
| type BigQuery struct { | ||
| // PathToCredentials is _optional_ if you have GOOGLE_APPLICATION_CREDENTIALS set as an env var | ||
| // Links to credentials: https://cloud.google.com/docs/authentication/application-default-credentials#GAC | ||
| PathToCredentials string `yaml:"pathToCredentials"` | ||
| DefaultDataset string `yaml:"defaultDataset"` | ||
| ProjectID string `yaml:"projectID"` | ||
| Location string `yaml:"location"` | ||
| } | ||
|
|
||
| type Redshift struct { | ||
| Host string `yaml:"host"` | ||
| Port int `yaml:"port"` | ||
| Database string `yaml:"database"` | ||
| Username string `yaml:"username"` | ||
| Password string `yaml:"password"` | ||
| Bucket string `yaml:"bucket"` | ||
| OptionalS3Prefix string `yaml:"optionalS3Prefix"` | ||
| // https://docs.aws.amazon.com/redshift/latest/dg/copy-parameters-authorization.html | ||
| CredentialsClause string `yaml:"credentialsClause"` | ||
| } | ||
|
|
||
| type Snowflake struct { | ||
| AccountID string `yaml:"account"` | ||
| Username string `yaml:"username"` | ||
| // If pathToPrivateKey is specified, the password field will be ignored | ||
| PathToPrivateKey string `yaml:"pathToPrivateKey,omitempty"` | ||
| Password string `yaml:"password,omitempty"` | ||
|
|
||
| Warehouse string `yaml:"warehouse"` | ||
| Region string `yaml:"region"` | ||
| Host string `yaml:"host"` | ||
| Application string `yaml:"application"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package config | ||
|
|
||
| import ( | ||
| "github.com/artie-labs/transfer/lib/config/constants" | ||
| "github.com/artie-labs/transfer/lib/kafkalib" | ||
| ) | ||
|
|
||
| type Mode string | ||
|
|
||
| const ( | ||
| History Mode = "history" | ||
| Replication Mode = "replication" | ||
| ) | ||
|
|
||
| type Sentry struct { | ||
| DSN string `yaml:"dsn"` | ||
| } | ||
|
|
||
| type Pubsub struct { | ||
| ProjectID string `yaml:"projectID"` | ||
| TopicConfigs []*kafkalib.TopicConfig `yaml:"topicConfigs"` | ||
| PathToCredentials string `yaml:"pathToCredentials"` | ||
| } | ||
| type Kafka struct { | ||
| // Comma-separated Kafka servers to port. | ||
| // e.g. host1:port1,host2:port2,... | ||
| // Following kafka's spec mentioned here: https://kafka.apache.org/documentation/#producerconfigs_bootstrap.servers | ||
| BootstrapServer string `yaml:"bootstrapServer"` | ||
| GroupID string `yaml:"groupID"` | ||
| TopicConfigs []*kafkalib.TopicConfig `yaml:"topicConfigs"` | ||
|
|
||
| // Optional parameters | ||
| Username string `yaml:"username,omitempty"` | ||
| Password string `yaml:"password,omitempty"` | ||
| EnableAWSMSKIAM bool `yaml:"enableAWSMKSIAM,omitempty"` | ||
| DisableTLS bool `yaml:"disableTLS,omitempty"` | ||
| } | ||
|
|
||
| type Config struct { | ||
| Mode Mode `yaml:"mode"` | ||
| Output constants.DestinationKind `yaml:"outputSource"` | ||
| Queue constants.QueueKind `yaml:"queue"` | ||
|
|
||
| // Flush rules | ||
| FlushIntervalSeconds int `yaml:"flushIntervalSeconds"` | ||
| FlushSizeKb int `yaml:"flushSizeKb"` | ||
| BufferRows uint `yaml:"bufferRows"` | ||
|
|
||
| // Supported message queues | ||
| Pubsub *Pubsub `yaml:"pubsub,omitempty"` | ||
| Kafka *Kafka `yaml:"kafka,omitempty"` | ||
|
|
||
| // Supported destinations | ||
| MSSQL *MSSQL `yaml:"mssql,omitempty"` | ||
| BigQuery *BigQuery `yaml:"bigquery,omitempty"` | ||
| Snowflake *Snowflake `yaml:"snowflake,omitempty"` | ||
| Redshift *Redshift `yaml:"redshift,omitempty"` | ||
| S3 *S3Settings `yaml:"s3,omitempty"` | ||
|
|
||
| Reporting struct { | ||
| Sentry *Sentry `yaml:"sentry"` | ||
| } | ||
|
|
||
| Telemetry struct { | ||
| Metrics struct { | ||
| Provider constants.ExporterKind `yaml:"provider"` | ||
| Settings map[string]any `yaml:"settings,omitempty"` | ||
| } | ||
| } | ||
| } |