diff --git a/runtime/compilers/rillv1/connectors.go b/runtime/compilers/rillv1/connectors.go index 4e9f1c42c29..63d3b62f936 100644 --- a/runtime/compilers/rillv1/connectors.go +++ b/runtime/compilers/rillv1/connectors.go @@ -128,9 +128,7 @@ func driverSourceForAnonAccessCheck(connector string, src *runtimev1.SourceSpec) } case "motherduck": return &drivers.DatabaseSource{} - case "postgres_ext": - return &drivers.DatabaseSource{} - case "sqlite_ext": + case "sqlite": return &drivers.DatabaseSource{} default: return nil diff --git a/runtime/compilers/rillv1beta/connector.go b/runtime/compilers/rillv1beta/connector.go index 519adb0b810..4df147113cc 100644 --- a/runtime/compilers/rillv1beta/connector.go +++ b/runtime/compilers/rillv1beta/connector.go @@ -180,9 +180,7 @@ func source(connector string, src *runtimev1.Source) drivers.Source { } case "motherduck": return &drivers.DatabaseSource{} - case "postgres_ext": - return &drivers.DatabaseSource{} - case "sqlite_ext": + case "sqlite": return &drivers.DatabaseSource{} case "bigquery": return &drivers.DatabaseSource{ diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 45366adcb4b..e3a41dafd42 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -26,62 +26,27 @@ import ( func init() { drivers.Register("duckdb", Driver{name: "duckdb"}) drivers.Register("motherduck", Driver{name: "motherduck"}) - drivers.Register("postgres_ext", Driver{name: "postgres_ext"}) - drivers.Register("sqlite_ext", Driver{name: "sqlite_ext"}) drivers.RegisterAsConnector("motherduck", Driver{name: "motherduck"}) - drivers.RegisterAsConnector("postgres_ext", Driver{name: "postgres_ext"}) - drivers.RegisterAsConnector("sqlite_ext", Driver{name: "sqlite_ext"}) } -var specs map[string]drivers.Spec = map[string]drivers.Spec{ - "motherduck": { - DisplayName: "MotherDuck", - Description: "Import data from MotherDuck.", - SourceProperties: []drivers.PropertySchema{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "SQL", - Description: "Query to extract data from MotherDuck.", - Placeholder: "select * from my_db.my_table;", - }, - }, - ConfigProperties: []drivers.PropertySchema{ - { - Key: "token", - Secret: true, - }, - }, - }, - "postgres_ext": { - DisplayName: "Postgres", - Description: "Import data from Postgres table to DuckDB.", - SourceProperties: []drivers.PropertySchema{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "SQL", - Description: "Query to extract data from postgres", - Placeholder: "SELECT * FROM postgres_scan('dbname=postgres user=postgres password=*** host=127.0.0.1', 'public', 'users');", - Hint: "https://duckdb.org/docs/extensions/postgres_scanner.html#querying-individual-tables", - }, +// spec for duckdb as motherduck connector +var spec = drivers.Spec{ + DisplayName: "MotherDuck", + Description: "Import data from MotherDuck.", + SourceProperties: []drivers.PropertySchema{ + { + Key: "sql", + Type: drivers.StringPropertyType, + Required: true, + DisplayName: "SQL", + Description: "Query to extract data from MotherDuck.", + Placeholder: "select * from my_db.my_table;", }, }, - "sqlite_ext": { - DisplayName: "SQLite", - Description: "Import data from SQLite table to DuckDB.", - SourceProperties: []drivers.PropertySchema{ - { - Key: "sql", - Type: drivers.StringPropertyType, - Required: true, - DisplayName: "SQL", - Description: "Query to extract data from SQLite", - Placeholder: "SELECT * FROM sqlite_scan('sakila.db', 'film');", - Hint: "https://duckdb.org/docs/extensions/sqlite_scanner#querying-individual-tables", - }, + ConfigProperties: []drivers.PropertySchema{ + { + Key: "token", + Secret: true, }, }, } @@ -170,7 +135,7 @@ func (d Driver) Drop(config map[string]any, logger *zap.Logger) error { } func (d Driver) Spec() drivers.Spec { - return specs[d.name] + return spec } func (d Driver) HasAnonymousSourceAccess(ctx context.Context, src drivers.Source, logger *zap.Logger) (bool, error) { @@ -269,7 +234,7 @@ func (c *connection) AsTransporter(from, to drivers.Handle) (drivers.Transporter if from == to { return transporter.NewDuckDBToDuckDB(olap, c.logger), true } - if from.Driver() == "postgres_ext" || from.Driver() == "sqlite_ext" { + if from.Driver() == "sqlite" { return transporter.NewSQLExtensionToDuckDB(from, olap, c.logger), true } if from.Driver() == "motherduck" { diff --git a/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB.go b/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB.go index efd0b07bf99..46591081c8c 100644 --- a/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB.go @@ -18,6 +18,7 @@ type sqlextensionToDuckDB struct { var _ drivers.Transporter = &sqlextensionToDuckDB{} // NewSQLExtensionToDuckDB returns a transporter to transfer data to duckdb from a sql extension supported by duckdb. +// Currently only sqlite extension is supported. Postgres is not supported due to licensing issues. func NewSQLExtensionToDuckDB(from drivers.Handle, to drivers.OLAPStore, logger *zap.Logger) drivers.Transporter { return &sqlextensionToDuckDB{ to: to, diff --git a/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB_test.go b/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB_test.go index 209460f951d..59571b7b439 100644 --- a/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB_test.go +++ b/runtime/drivers/duckdb/transporter/sqlextension_to_duckDB_test.go @@ -7,6 +7,8 @@ import ( "testing" "github.com/rilldata/rill/runtime/drivers" + _ "github.com/rilldata/rill/runtime/drivers/sqlite" + "github.com/rilldata/rill/runtime/pkg/activity" "github.com/stretchr/testify/require" "go.uber.org/zap" _ "modernc.org/sqlite" @@ -27,9 +29,9 @@ func Test_sqlextensionToDuckDB_Transfer(t *testing.T) { require.NoError(t, err) db.Close() - from, err := drivers.Open("sqlite_ext", map[string]any{"dsn": ""}, false, zap.NewNop()) + from, err := drivers.Open("sqlite", map[string]any{"dsn": ""}, false, activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) - to, err := drivers.Open("duckdb", map[string]any{"dsn": ""}, false, zap.NewNop()) + to, err := drivers.Open("duckdb", map[string]any{"dsn": ""}, false, activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) olap, _ := to.AsOLAP("") diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index c1961b4f91f..98288f203d8 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -17,6 +17,7 @@ import ( func init() { drivers.Register("sqlite", driver{}) + drivers.RegisterAsConnector("sqlite", driver{}) } type driver struct{} @@ -55,7 +56,21 @@ func (d driver) Drop(config map[string]any, logger *zap.Logger) error { } func (d driver) Spec() drivers.Spec { - return drivers.Spec{} + return drivers.Spec{ + DisplayName: "SQLite", + Description: "Import data from SQLite table to DuckDB.", + SourceProperties: []drivers.PropertySchema{ + { + Key: "sql", + Type: drivers.StringPropertyType, + Required: true, + DisplayName: "SQL", + Description: "Query to extract data from SQLite", + Placeholder: "SELECT * FROM sqlite_scan('/Users/kanshul/.config/gcloud/access_tokens.db', 'film');", + Hint: "https://duckdb.org/docs/extensions/sqlite_scanner#querying-individual-tables", + }, + }, + } } func (d driver) HasAnonymousSourceAccess(ctx context.Context, src drivers.Source, logger *zap.Logger) (bool, error) { diff --git a/runtime/reconcilers/source.go b/runtime/reconcilers/source.go index a955180205f..efbb7b3de31 100644 --- a/runtime/reconcilers/source.go +++ b/runtime/reconcilers/source.go @@ -398,16 +398,7 @@ func driversSource(conn drivers.Handle, propsPB *structpb.Struct) (drivers.Sourc SQL: query, Database: db, }, nil - case "postgres_ext": - query, ok := props["sql"].(string) - if !ok { - return nil, fmt.Errorf("property \"sql\" is mandatory for connector \"postgres\"") - } - - return &drivers.DatabaseSource{ - SQL: query, - }, nil - case "sqlite_ext": + case "sqlite": query, ok := props["sql"].(string) if !ok { return nil, fmt.Errorf("property \"sql\" is mandatory for connector \"sqlite\"") diff --git a/runtime/services/catalog/migrator/sources/sources.go b/runtime/services/catalog/migrator/sources/sources.go index 2c0e110dbbe..9f389393466 100644 --- a/runtime/services/catalog/migrator/sources/sources.go +++ b/runtime/services/catalog/migrator/sources/sources.go @@ -403,16 +403,7 @@ func source(connector string, src *runtimev1.Source) (drivers.Source, error) { SQL: query, Props: props, }, nil - case "postgres_ext": - query, ok := props["sql"].(string) - if !ok { - return nil, fmt.Errorf("property \"sql\" is mandatory for connector \"postgres\"") - } - - return &drivers.DatabaseSource{ - SQL: query, - }, nil - case "sqlite_ext": + case "sqlite": query, ok := props["sql"].(string) if !ok { return nil, fmt.Errorf("property \"sql\" is mandatory for connector \"sqlite\"") @@ -452,9 +443,7 @@ func connectorVariables(src *runtimev1.Source, env map[string]string, repoRoot s case "motherduck": vars["token"] = env["token"] vars["dsn"] = "" - case "postgres_ext": - vars["dsn"] = "" - case "sqlite_ext": + case "sqlite": vars["dsn"] = "" case "local_file": vars["dsn"] = repoRoot diff --git a/web-common/src/features/sources/modal/AddSourceModal.svelte b/web-common/src/features/sources/modal/AddSourceModal.svelte index d60d216bec5..de50a5ab278 100644 --- a/web-common/src/features/sources/modal/AddSourceModal.svelte +++ b/web-common/src/features/sources/modal/AddSourceModal.svelte @@ -27,8 +27,7 @@ "https", "local_file", "motherduck", - "postgres_ext", - "sqlite_ext", + "sqlite", "bigquery", ]; @@ -93,7 +92,7 @@
- {#if selectedConnector?.name === "gcs" || selectedConnector?.name === "s3" || selectedConnector?.name === "https" || selectedConnector?.name === "motherduck" || selectedConnector?.name === "postgres_ext" || selectedConnector?.name === "sqlite_ext" || selectedConnector?.name === "bigquery"} + {#if selectedConnector?.name === "gcs" || selectedConnector?.name === "s3" || selectedConnector?.name === "https" || selectedConnector?.name === "motherduck" || selectedConnector?.name === "postgres_ext" || selectedConnector?.name === "sqlite"} {#key selectedConnector} {/key} diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 5ea06c87c23..2c4962946e5 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -57,18 +57,7 @@ export function getYupSchema(connector: V1ConnectorSpec) { ) .required("Source name is required"), }); - case "postgres_ext": - return yup.object().shape({ - sql: yup.string().required("sql is required"), - sourceName: yup - .string() - .matches( - /^[a-zA-Z_][a-zA-Z0-9_]*$/, - "Source name must start with a letter or underscore and contain only letters, numbers, and underscores" - ) - .required("Source name is required"), - }); - case "sqlite_ext": + case "sqlite": return yup.object().shape({ sql: yup.string().required("sql is required"), sourceName: yup