diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index 5294a150e94..a15c6474b4f 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -274,6 +274,8 @@ func (c *connection) reopenDB() error { "LOAD 'parquet'", "INSTALL 'httpfs'", "LOAD 'httpfs'", + "INSTALL 'sqlite'", + "LOAD 'sqlite'", "SET max_expression_depth TO 250", "SET timezone='UTC'", } diff --git a/runtime/drivers/duckdb/transporter/duckDB_to_duckDB.go b/runtime/drivers/duckdb/transporter/duckDB_to_duckDB.go index ce1c707eaf2..1b10fe0806f 100644 --- a/runtime/drivers/duckdb/transporter/duckDB_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter/duckDB_to_duckDB.go @@ -119,6 +119,7 @@ func rewriteLocalPaths(ast *duckdbsql.AST, basePath string, allowHostAccess bool Function: t.Function, Paths: res, Properties: t.Properties, + Params: t.Params, }, true }) if resolveErr != nil { diff --git a/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go b/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go index d2121bd64ed..66ff5c39e90 100644 --- a/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go @@ -303,6 +303,7 @@ func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL s Paths: allFiles, Function: table.Function, Properties: table.Properties, + Params: table.Params, }, true }) if err != nil { diff --git a/runtime/drivers/duckdb/transporter/sqlite_to_duckDB_test.go b/runtime/drivers/duckdb/transporter/sqlite_to_duckDB_test.go new file mode 100644 index 00000000000..d1e070f7c1e --- /dev/null +++ b/runtime/drivers/duckdb/transporter/sqlite_to_duckDB_test.go @@ -0,0 +1,51 @@ +package transporter + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/rilldata/rill/runtime/drivers" + _ "github.com/rilldata/rill/runtime/drivers/sqlite" + "github.com/rilldata/rill/runtime/pkg/activity" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + _ "modernc.org/sqlite" +) + +func Test_sqliteToDuckDB_Transfer(t *testing.T) { + tempDir := t.TempDir() + + dbPath := fmt.Sprintf("%s.db", tempDir) + db, err := sql.Open("sqlite", dbPath) + require.NoError(t, err) + + _, err = db.Exec(` + drop table if exists t; + create table t(i); + insert into t values(42), (314); + `) + require.NoError(t, err) + db.Close() + + to, err := drivers.Open("duckdb", map[string]any{"dsn": ""}, false, activity.NewNoopClient(), zap.NewNop()) + require.NoError(t, err) + olap, _ := to.AsOLAP("") + + tr := &duckDBToDuckDB{ + to: olap, + logger: zap.NewNop(), + } + query := fmt.Sprintf("SELECT * FROM sqlite_scan('%s', 't');", dbPath) + err = tr.Transfer(context.Background(), map[string]any{"sql": query}, map[string]any{"table": "test"}, &drivers.TransferOptions{Progress: drivers.NoOpProgress{}}) + require.NoError(t, err) + + res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) from test"}) + require.NoError(t, err) + res.Next() + var count int + err = res.Scan(&count) + require.NoError(t, err) + require.Equal(t, 2, count) +} diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index b34b94e6554..7d36d2e5c38 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -17,6 +17,7 @@ import ( func init() { drivers.Register("sqlite", driver{}) + drivers.RegisterAsConnector("sqlite", driver{}) } type driver struct{} @@ -55,11 +56,32 @@ func (d driver) Drop(config map[string]any, logger *zap.Logger) error { } func (d driver) Spec() drivers.Spec { - return drivers.Spec{} + return drivers.Spec{ + DisplayName: "SQLite", + Description: "Import data from SQLite into DuckDB.", + SourceProperties: []drivers.PropertySchema{ + { + Key: "db", + Type: drivers.StringPropertyType, + Required: true, + DisplayName: "DB", + Description: "Path to SQLite db file", + Placeholder: "sqlite.db", + }, + { + Key: "table", + Type: drivers.StringPropertyType, + Required: true, + DisplayName: "Table", + Description: "SQLite table name", + Placeholder: "table", + }, + }, + } } func (d driver) HasAnonymousSourceAccess(ctx context.Context, src map[string]any, logger *zap.Logger) (bool, error) { - return false, nil + return true, nil } type connection struct { diff --git a/runtime/pkg/duckdbsql/ast.go b/runtime/pkg/duckdbsql/ast.go index 50caa9380e7..f1dc06d1e82 100644 --- a/runtime/pkg/duckdbsql/ast.go +++ b/runtime/pkg/duckdbsql/ast.go @@ -82,6 +82,12 @@ func (a *AST) RewriteTableRefs(fn func(table *TableRef) (*TableRef, bool)) error } } else if newRef.Function != "" { switch newRef.Function { + case "sqlite_scan": + newRef.Params[0] = newRef.Paths[0] + err := node.rewriteToSqliteScanFunction(newRef.Function, newRef.Params) + if err != nil { + return err + } case "read_csv_auto", "read_csv", "read_parquet", "read_json", "read_json_auto", "read_json_objects", "read_json_objects_auto", @@ -155,6 +161,8 @@ type TableRef struct { Paths []string Properties map[string]any LocalAlias bool + // Params passed to sqlite_scan + Params []string } // ColumnRef has information about a column in the select list of a DuckDB SQL statement diff --git a/runtime/pkg/duckdbsql/ast_rewrite.go b/runtime/pkg/duckdbsql/ast_rewrite.go index fbe76349f1b..33f0298eb94 100644 --- a/runtime/pkg/duckdbsql/ast_rewrite.go +++ b/runtime/pkg/duckdbsql/ast_rewrite.go @@ -62,6 +62,15 @@ func (sn *selectNode) rewriteLimit(limit, offset int) error { return nil } +func (fn *fromNode) rewriteToSqliteScanFunction(name string, params []string) error { + baseTable, err := createSqliteScanTableFunction(params, fn.ast) + if err != nil { + return err + } + fn.parent[fn.childKey] = baseTable + return nil +} + func createBaseTable(name string, ast astNode) (astNode, error) { // TODO: validation and fill in other fields from ast var n astNode @@ -357,3 +366,37 @@ func createFunctionCall(key, name, schema string) (astNode, error) { }`, key, name, schema)), &n) return n, err } + +func createSqliteScanTableFunction(params []string, ast astNode) (astNode, error) { + var n astNode + err := json.Unmarshal([]byte(`{ + "type": "TABLE_FUNCTION", + "alias": "", + "sample": null, + "function": {}, + "column_name_alias": [] +}`), &n) + if err != nil { + return nil, err + } + + fn, err := createFunctionCall("", "sqlite_scan", "") + if err != nil { + return nil, err + } + n[astKeyFunction] = fn + + var list []astNode + for _, v := range params { + vn, err := createGenericValue("", v) + if err != nil { + return nil, err + } + if vn == nil { + continue + } + list = append(list, vn) + } + fn[astKeyChildren] = list + return n, nil +} diff --git a/runtime/pkg/duckdbsql/ast_test.go b/runtime/pkg/duckdbsql/ast_test.go index 088f44d05e6..3dfd0825f6f 100644 --- a/runtime/pkg/duckdbsql/ast_test.go +++ b/runtime/pkg/duckdbsql/ast_test.go @@ -126,6 +126,18 @@ select * from read_json( {Name: "tbl3", LocalAlias: true}, }, }, + { + "sqlite_scan", + `select * from sqlite_scan('mydatabase.db', 'table')`, + []*TableRef{ + { + Function: "sqlite_scan", + Paths: []string{"mydatabase.db"}, + Params: []string{"mydatabase.db", "table"}, + Properties: make(map[string]any), + }, + }, + }, { "other table functions", `select * from generate_series(TIMESTAMP '2001-04-10', TIMESTAMP '2001-04-11', INTERVAL 30 MINUTE)`, @@ -470,6 +482,19 @@ func TestAST_RewriteWithFunctionRef(t *testing.T) { }, `SELECT * FROM read_csv(main.list_value('/path/to/AdBids.csv'), ("columns" = main.struct_pack(L := main.list_value('INT32', 'INT64'))))`, }, + { + "sqlite_scan", + `select * from AdBids`, + []*TableRef{ + { + Function: "sqlite_scan", + Paths: []string{"/path/to/data.db"}, + Properties: map[string]any{}, + Params: []string{"/path/to/data.db", "table"}, + }, + }, + `SELECT * FROM sqlite_scan('/path/to/data.db', 'table')`, + }, } for _, tt := range sqlVariations { diff --git a/runtime/pkg/duckdbsql/ast_traversal.go b/runtime/pkg/duckdbsql/ast_traversal.go index eb70ac87627..4f96ec9e8bb 100644 --- a/runtime/pkg/duckdbsql/ast_traversal.go +++ b/runtime/pkg/duckdbsql/ast_traversal.go @@ -172,6 +172,27 @@ func (a *AST) traverseTableFunction(parent astNode, childKey string) { // TODO: add to local alias switch functionName { + case "sqlite_scan": + a.newFromNode(node, parent, childKey, ref) + ref.Params = make([]string, 0) + for _, argument := range arguments { + typ := toString(argument, astKeyType) + switch typ { + case "VALUE_CONSTANT": + ref.Params = append(ref.Params, getListOfValues[string](argument)...) + case "COLUMN_REF": + columnNames := toArray(argument, astKeyColumnNames) + for _, column := range columnNames { + ref.Params = append(ref.Params, column.(string)) + } + default: + } + } + if len(ref.Params) >= 1 { + // first param is path to local db file + ref.Paths = ref.Params[:1] + } + return case "read_csv_auto", "read_csv", "read_parquet", "read_json", "read_json_auto", "read_json_objects", "read_json_objects_auto", diff --git a/runtime/services/catalog/migrator/sources/sources.go b/runtime/services/catalog/migrator/sources/sources.go index 57d04b497a1..79f35639bae 100644 --- a/runtime/services/catalog/migrator/sources/sources.go +++ b/runtime/services/catalog/migrator/sources/sources.go @@ -290,6 +290,8 @@ func mergeFromParsedQuery(apiSource *runtimev1.Source, env map[string]string, re return errors.New("invalid source, only a single path for source is supported") } + // TODO :: it looks at path to determine connector which is not correct for sqlite_scan + // but it works since behaviour is same as local_file p, c, ok := parseEmbeddedSourceConnector(ref.Paths[0]) if !ok { return errors.New("unknown source") @@ -334,6 +336,7 @@ func rewriteLocalRelativePath(ast *duckdbsql.AST, repoRoot string, allowRootAcce Function: table.Function, Paths: newPaths, Properties: table.Properties, + Params: table.Params, }, true }) if resolveErr != nil { diff --git a/web-common/src/components/icons/connectors/SQLite.svelte b/web-common/src/components/icons/connectors/SQLite.svelte new file mode 100644 index 00000000000..0866df04640 --- /dev/null +++ b/web-common/src/components/icons/connectors/SQLite.svelte @@ -0,0 +1,38 @@ + + + + + + + + + + + + + diff --git a/web-common/src/features/sources/modal/AddSourceModal.svelte b/web-common/src/features/sources/modal/AddSourceModal.svelte index f12bfcacd54..b3454b55225 100644 --- a/web-common/src/features/sources/modal/AddSourceModal.svelte +++ b/web-common/src/features/sources/modal/AddSourceModal.svelte @@ -13,6 +13,7 @@ import LocalFile from "../../../components/icons/connectors/LocalFile.svelte"; import MotherDuck from "../../../components/icons/connectors/MotherDuck.svelte"; import Postgres from "../../../components/icons/connectors/Postgres.svelte"; + import SQLite from "../../../components/icons/connectors/SQLite.svelte"; import { appScreen } from "../../../layout/app-store"; import { behaviourEvent } from "../../../metrics/initMetrics"; import { @@ -39,6 +40,7 @@ // athena "motherduck", "postgres", + "sqlite", "local_file", "https", ]; @@ -52,6 +54,7 @@ // athena: AmazonAthena, motherduck: MotherDuck, postgres: Postgres, + sqlite: SQLite, local_file: LocalFile, https: Https, }; diff --git a/web-common/src/features/sources/modal/yupSchemas.ts b/web-common/src/features/sources/modal/yupSchemas.ts index 0fba22f26d7..ce10fcfb671 100644 --- a/web-common/src/features/sources/modal/yupSchemas.ts +++ b/web-common/src/features/sources/modal/yupSchemas.ts @@ -57,6 +57,18 @@ export function getYupSchema(connector: V1ConnectorSpec) { ) .required("Source name is required"), }); + case "sqlite": + return yup.object().shape({ + db: yup.string().required("db is required"), + table: yup.string().required("table is required"), + sourceName: yup + .string() + .matches( + /^[a-zA-Z_][a-zA-Z0-9_]*$/, + "Source name must start with a letter or underscore and contain only letters, numbers, and underscores" + ) + .required("Source name is required"), + }); case "bigquery": return yup.object().shape({ sql: yup.string().required("sql is required"), diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts index 11f1ffa611c..f534ff1424a 100644 --- a/web-common/src/features/sources/sourceUtils.ts +++ b/web-common/src/features/sources/sourceUtils.ts @@ -30,6 +30,14 @@ export function compileCreateSourceYAML( values.sql = buildDuckDbQuery(values.path as string); delete values.path; break; + case "sqlite": + connectorName = "duckdb"; + values.sql = `SELECT * FROM sqlite_scan('${values.db as string}', '${ + values.table as string + }');`; + delete values.db; + delete values.table; + break; } const compiledKeyValues = Object.entries(values)