Runtime: duckdb sqlite extensions based connector for sqlite

runtime/drivers/duckdb/duckdb.go

@@ -274,6 +274,8 @@ func (c *connection) reopenDB() error {
 		"LOAD 'parquet'",
 		"INSTALL 'httpfs'",
 		"LOAD 'httpfs'",
+		"INSTALL 'sqlite'",
+		"LOAD 'sqlite'",
 		"SET max_expression_depth TO 250",
 		"SET timezone='UTC'",
 	}

runtime/drivers/duckdb/transporter/duckDB_to_duckDB.go

@@ -119,6 +119,7 @@ func rewriteLocalPaths(ast *duckdbsql.AST, basePath string, allowHostAccess bool
 			Function:   t.Function,
 			Paths:      res,
 			Properties: t.Properties,
+			Params:     t.Params,
 		}, true
 	})
 	if resolveErr != nil {

runtime/drivers/duckdb/transporter/objectStore_to_duckDB.go

@@ -303,6 +303,7 @@ func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL s
 			Paths:      allFiles,
 			Function:   table.Function,
 			Properties: table.Properties,
+			Params:     table.Params,
 		}, true
 	})
 	if err != nil {

runtime/drivers/duckdb/transporter/sqlite_to_duckDB_test.go

@@ -0,0 +1,51 @@
+package transporter
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+
+	"github.com/rilldata/rill/runtime/drivers"
+	_ "github.com/rilldata/rill/runtime/drivers/sqlite"
+	"github.com/rilldata/rill/runtime/pkg/activity"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	_ "modernc.org/sqlite"
+)
+
+func Test_sqliteToDuckDB_Transfer(t *testing.T) {
+	tempDir := t.TempDir()
+
+	dbPath := fmt.Sprintf("%s.db", tempDir)
+	db, err := sql.Open("sqlite", dbPath)
+	require.NoError(t, err)
+
+	_, err = db.Exec(`
+	drop table if exists t;
+	create table t(i);
+	insert into t values(42), (314);
+	`)
+	require.NoError(t, err)
+	db.Close()
+
+	to, err := drivers.Open("duckdb", map[string]any{"dsn": ""}, false, activity.NewNoopClient(), zap.NewNop())
+	require.NoError(t, err)
+	olap, _ := to.AsOLAP("")
+
+	tr := &duckDBToDuckDB{
+		to:     olap,
+		logger: zap.NewNop(),
+	}
+	query := fmt.Sprintf("SELECT * FROM sqlite_scan('%s', 't');", dbPath)
+	err = tr.Transfer(context.Background(), map[string]any{"sql": query}, map[string]any{"table": "test"}, &drivers.TransferOptions{Progress: drivers.NoOpProgress{}})
+	require.NoError(t, err)
+
+	res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) from test"})
+	require.NoError(t, err)
+	res.Next()
+	var count int
+	err = res.Scan(&count)
+	require.NoError(t, err)
+	require.Equal(t, 2, count)
+}

runtime/drivers/sqlite/sqlite.go

@@ -17,6 +17,7 @@ import (
 
 func init() {
 	drivers.Register("sqlite", driver{})
+	drivers.RegisterAsConnector("sqlite", driver{})
 }
 
 type driver struct{}
@@ -55,11 +56,32 @@ func (d driver) Drop(config map[string]any, logger *zap.Logger) error {
 }
 
 func (d driver) Spec() drivers.Spec {
-	return drivers.Spec{}
+	return drivers.Spec{
+		DisplayName: "SQLite",
+		Description: "Import data from SQLite into DuckDB.",
+		SourceProperties: []drivers.PropertySchema{
+			{
+				Key:         "db",
+				Type:        drivers.StringPropertyType,
+				Required:    true,
+				DisplayName: "DB",
+				Description: "Path to SQLite db file",
+				Placeholder: "sqlite.db",
+			},
+			{
+				Key:         "table",
+				Type:        drivers.StringPropertyType,
+				Required:    true,
+				DisplayName: "Table",
+				Description: "SQLite table name",
+				Placeholder: "table",
+			},
+		},
+	}
 }
 
 func (d driver) HasAnonymousSourceAccess(ctx context.Context, src map[string]any, logger *zap.Logger) (bool, error) {
-	return false, nil
+	return true, nil
 }
 
 type connection struct {

runtime/pkg/duckdbsql/ast.go

@@ -82,6 +82,12 @@ func (a *AST) RewriteTableRefs(fn func(table *TableRef) (*TableRef, bool)) error
 			}
 		} else if newRef.Function != "" {
 			switch newRef.Function {
+			case "sqlite_scan":
+				newRef.Params[0] = newRef.Paths[0]
+				err := node.rewriteToSqliteScanFunction(newRef.Function, newRef.Params)
+				if err != nil {
+					return err
+				}
 			case "read_csv_auto", "read_csv",
 				"read_parquet",
 				"read_json", "read_json_auto", "read_json_objects", "read_json_objects_auto",
@@ -155,6 +161,8 @@ type TableRef struct {
 	Paths      []string
 	Properties map[string]any
 	LocalAlias bool
+	// Params passed to sqlite_scan
+	Params []string
 }
 
 // ColumnRef has information about a column in the select list of a DuckDB SQL statement

runtime/pkg/duckdbsql/ast_rewrite.go

@@ -62,6 +62,15 @@ func (sn *selectNode) rewriteLimit(limit, offset int) error {
 	return nil
 }
 
+func (fn *fromNode) rewriteToSqliteScanFunction(name string, params []string) error {
+	baseTable, err := createSqliteScanTableFunction(params, fn.ast)
+	if err != nil {
+		return err
+	}
+	fn.parent[fn.childKey] = baseTable
+	return nil
+}
+
 func createBaseTable(name string, ast astNode) (astNode, error) {
 	// TODO: validation and fill in other fields from ast
 	var n astNode
@@ -357,3 +366,37 @@ func createFunctionCall(key, name, schema string) (astNode, error) {
 }`, key, name, schema)), &n)
 	return n, err
 }
+
+func createSqliteScanTableFunction(params []string, ast astNode) (astNode, error) {
+	var n astNode
+	err := json.Unmarshal([]byte(`{
+  "type": "TABLE_FUNCTION",
+  "alias": "",
+  "sample": null,
+  "function": {},
+  "column_name_alias": []
+}`), &n)
+	if err != nil {
+		return nil, err
+	}
+
+	fn, err := createFunctionCall("", "sqlite_scan", "")
+	if err != nil {
+		return nil, err
+	}
+	n[astKeyFunction] = fn
+
+	var list []astNode
+	for _, v := range params {
+		vn, err := createGenericValue("", v)
+		if err != nil {
+			return nil, err
+		}
+		if vn == nil {
+			continue
+		}
+		list = append(list, vn)
+	}
+	fn[astKeyChildren] = list
+	return n, nil
+}

runtime/pkg/duckdbsql/ast_test.go

@@ -126,6 +126,18 @@ select * from read_json(
 				{Name: "tbl3", LocalAlias: true},
 			},
 		},
+		{
+			"sqlite_scan",
+			`select * from sqlite_scan('mydatabase.db', 'table')`,
+			[]*TableRef{
+				{
+					Function:   "sqlite_scan",
+					Paths:      []string{"mydatabase.db"},
+					Params:     []string{"mydatabase.db", "table"},
+					Properties: make(map[string]any),
+				},
+			},
+		},
 		{
 			"other table functions",
 			`select * from generate_series(TIMESTAMP '2001-04-10', TIMESTAMP '2001-04-11', INTERVAL 30 MINUTE)`,
@@ -470,6 +482,19 @@ func TestAST_RewriteWithFunctionRef(t *testing.T) {
 			},
 			`SELECT * FROM read_csv(main.list_value('/path/to/AdBids.csv'), ("columns" = main.struct_pack(L := main.list_value('INT32', 'INT64'))))`,
 		},
+		{
+			"sqlite_scan",
+			`select * from AdBids`,
+			[]*TableRef{
+				{
+					Function:   "sqlite_scan",
+					Paths:      []string{"/path/to/data.db"},
+					Properties: map[string]any{},
+					Params:     []string{"/path/to/data.db", "table"},
+				},
+			},
+			`SELECT * FROM sqlite_scan('/path/to/data.db', 'table')`,
+		},
 	}
 
 	for _, tt := range sqlVariations {

runtime/pkg/duckdbsql/ast_traversal.go

@@ -172,6 +172,27 @@ func (a *AST) traverseTableFunction(parent astNode, childKey string) {
 	// TODO: add to local alias
 
 	switch functionName {
+	case "sqlite_scan":
+		a.newFromNode(node, parent, childKey, ref)
+		ref.Params = make([]string, 0)
+		for _, argument := range arguments {
+			typ := toString(argument, astKeyType)
+			switch typ {
+			case "VALUE_CONSTANT":
+				ref.Params = append(ref.Params, getListOfValues[string](argument)...)
+			case "COLUMN_REF":
+				columnNames := toArray(argument, astKeyColumnNames)
+				for _, column := range columnNames {
+					ref.Params = append(ref.Params, column.(string))
+				}
+			default:
+			}
+		}
+		if len(ref.Params) >= 1 {
+			// first param is path to local db file
+			ref.Paths = ref.Params[:1]
+		}
+		return
 	case "read_csv_auto", "read_csv",
 		"read_parquet",
 		"read_json", "read_json_auto", "read_json_objects", "read_json_objects_auto",

runtime/services/catalog/migrator/sources/sources.go

@@ -290,6 +290,8 @@ func mergeFromParsedQuery(apiSource *runtimev1.Source, env map[string]string, re
 		return errors.New("invalid source, only a single path for source is supported")
 	}
 
+	// TODO :: it looks at path to determine connector which is not correct for sqlite_scan
+	// but it works since behaviour is same as local_file
 	p, c, ok := parseEmbeddedSourceConnector(ref.Paths[0])
 	if !ok {
 		return errors.New("unknown source")
@@ -334,6 +336,7 @@ func rewriteLocalRelativePath(ast *duckdbsql.AST, repoRoot string, allowRootAcce
 			Function:   table.Function,
 			Paths:      newPaths,
 			Properties: table.Properties,
+			Params:     table.Params,
 		}, true
 	})
 	if resolveErr != nil {

web-common/src/components/icons/connectors/SQLite.svelte

@@ -0,0 +1,38 @@
+<svg
+  width="75"
+  height="36"
+  viewBox="0 0 75 36"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    d="M28.9767 17.5234C28.0295 17.5234 27.2593 17.8038 26.6689 18.3643C26.0784 18.9253 25.7789 19.6607 25.7789 20.5617C25.7789 21.0286 25.8533 21.4544 25.9998 21.8445C26.1464 22.2354 26.3745 22.5975 26.6811 22.9248C26.9878 23.2523 27.6016 23.6972 28.5164 24.2628C29.6388 24.9481 30.3734 25.5042 30.726 25.9384C31.0788 26.3723 31.2538 26.827 31.2538 27.301C31.2538 27.9357 31.0449 28.4424 30.6155 28.8232C30.1854 29.2042 29.6101 29.3941 28.8969 29.3941C28.1446 29.3941 27.4894 29.1289 26.9267 28.6023C26.3639 28.0749 26.0799 27.3774 26.0735 26.5031H25.7236V29.6764H26.0735C26.1803 29.3759 26.3281 29.2222 26.5216 29.2222C26.6146 29.2222 26.8326 29.2864 27.1722 29.4063C27.9983 29.7003 28.6767 29.8421 29.2099 29.8421C30.1289 29.8421 30.9133 29.5203 31.5669 28.8662C32.2194 28.2124 32.5489 27.4238 32.5489 26.5031C32.5489 25.7892 32.3314 25.154 31.9045 24.6004C31.4775 24.0461 30.6472 23.4013 29.4063 22.6608C28.3386 22.0193 27.6455 21.4968 27.3256 21.0895C27.0052 20.6828 26.8407 20.2329 26.8407 19.7392C26.8407 19.2052 27.0369 18.7763 27.4238 18.4564C27.8107 18.1362 28.3177 17.9776 28.9521 17.9776C29.6661 17.9776 30.2612 18.1895 30.7321 18.616C31.2025 19.0432 31.4746 19.6352 31.5546 20.396H31.9045V17.6462H31.5791C31.5394 17.7864 31.5025 17.876 31.4687 17.9163C31.4358 17.9563 31.3724 17.9776 31.2784 17.9776C31.1652 17.9776 30.9634 17.93 30.6769 17.8365C30.0631 17.6297 29.4972 17.5234 28.9767 17.5234ZM40.4115 17.5234C39.2574 17.5234 38.2095 17.7955 37.2628 18.3398C36.3146 18.8831 35.5641 19.6376 35.0102 20.5985C34.4566 21.5594 34.1816 22.5799 34.1816 23.6674C34.1816 25.1285 34.6599 26.4302 35.624 27.5711C36.5886 28.7113 37.7445 29.4218 39.0857 29.6948C39.3924 29.8546 39.8311 30.2671 40.4054 30.9347C41.0526 31.6881 41.6001 32.2315 42.0503 32.555C42.5001 32.8788 42.9857 33.1161 43.4989 33.2732C44.0125 33.4296 44.5678 33.5064 45.1684 33.5064C45.8956 33.5064 46.5467 33.38 47.1202 33.1197L46.9913 32.8006C46.6584 32.9206 46.3034 32.9786 45.9295 32.9786C45.422 32.9786 44.9091 32.8112 44.395 32.4753C43.8816 32.1385 43.2396 31.4959 42.4738 30.548C42.1139 30.0938 41.8655 29.8072 41.725 29.6948C43.1925 29.4082 44.3992 28.697 45.3402 27.5588C46.281 26.4211 46.7519 25.1227 46.7519 23.6674C46.7519 21.9395 46.1372 20.4874 44.9167 19.3034C43.6953 18.1193 42.1932 17.5234 40.4115 17.5234ZM47.5499 17.5234L47.5683 17.8978C48.3367 17.8978 48.7671 18.1242 48.8633 18.5791C48.8992 18.7419 48.9156 19.0425 48.9186 19.4753L48.9125 27.7552C48.9061 28.3736 48.8179 28.7685 48.6485 28.9398C48.4789 29.11 48.1921 29.2165 47.777 29.259L47.7585 29.6334H55.4186L55.615 27.7552H55.2652C55.1652 28.2669 54.9362 28.6334 54.5716 28.8416C54.206 29.0514 53.5596 29.1547 52.6259 29.1547H51.9016C51.0615 29.1547 50.5753 28.8509 50.4469 28.2401C50.4204 28.1193 50.4113 27.9897 50.4101 27.8473L50.4408 19.4753C50.4406 18.8579 50.5186 18.4441 50.6802 18.2416C50.8434 18.0398 51.1378 17.9268 51.564 17.8978L51.5456 17.5234H47.5499ZM40.5343 17.9776C41.836 17.9776 42.8988 18.4903 43.7198 19.5244C44.5406 20.5588 44.9474 21.9765 44.9474 23.7718C44.9474 25.4729 44.5347 26.835 43.7076 27.8596C42.8804 28.8838 41.7795 29.3941 40.4115 29.3941C39.0973 29.3941 38.0315 28.8679 37.2137 27.8105C36.3966 26.7531 35.9923 25.3802 35.9923 23.692C35.9923 21.9572 36.4015 20.5685 37.226 19.5305C38.0499 18.4937 39.1533 17.9776 40.5343 17.9776ZM58.1499 19.7576C57.9631 19.7576 57.8207 19.8207 57.7141 19.9479C57.6051 20.0746 57.5717 20.228 57.6098 20.4144C57.6466 20.5951 57.7485 20.7536 57.9106 20.887C58.0716 21.0203 58.2454 21.0895 58.4323 21.0895C58.613 21.0895 58.7507 21.0203 58.8496 20.887C58.9486 20.7536 58.9788 20.5951 58.9417 20.4144C58.9037 20.228 58.8052 20.0746 58.6532 19.9479C58.4995 19.8207 58.3306 19.7576 58.1499 19.7576ZM63.2014 21.0282C62.8839 22.2494 62.1958 22.9103 61.1391 23.0168L61.1513 23.3667H62.3851L62.3605 27.5036C62.3625 28.2109 62.3841 28.6818 62.4342 28.9214C62.5557 29.4952 62.9303 29.7869 63.5574 29.7869C64.4646 29.7869 65.4162 29.2343 66.4115 28.1297L66.1107 27.8719C65.3921 28.5993 64.7567 28.9644 64.2019 28.9644C63.8608 28.9644 63.6495 28.7683 63.5697 28.3813C63.5478 28.2873 63.539 28.1775 63.539 28.0499L63.5512 23.3667H65.4356L65.4172 22.8081H63.5574V21.0282H63.2014ZM70.487 22.5872C69.433 22.5872 68.5757 23.0988 67.9091 24.1155C67.246 25.1339 67.0407 26.2622 67.3015 27.5036C67.4549 28.2307 67.7606 28.7943 68.2283 29.1915C68.6953 29.5885 69.2877 29.7869 69.996 29.7869C70.6554 29.7869 71.5737 29.6199 71.9601 29.2836C72.3475 28.9474 72.7045 28.4029 73.0342 27.657L72.7703 27.3808C72.2444 28.3484 71.1831 28.8355 70.3827 28.8355C69.2823 28.8355 68.6084 28.2318 68.3572 27.031C68.3245 26.8777 68.3002 26.713 68.2835 26.5399C69.5926 26.3327 70.5847 25.9657 71.2543 25.4351C71.9233 24.9043 72.595 24.3416 72.4696 23.7472C72.3949 23.394 72.2115 23.1163 71.9294 22.9063C71.6436 22.6965 70.9017 22.5872 70.487 22.5872ZM58.9663 22.6301L56.6953 23.1519V23.557L57.4809 23.4587C57.8615 23.4587 58.0854 23.6311 58.1561 23.9743C58.1799 24.0892 58.1949 24.2505 58.199 24.4531L58.1745 28.1665C58.1681 28.6803 58.1111 28.9791 57.9965 29.0687C57.8808 29.1586 57.5759 29.2038 57.0819 29.2038L57.0697 29.5536H60.6726L60.6665 29.2038C60.1659 29.2038 59.8413 29.1643 59.6967 29.0872C59.5545 29.0105 59.4566 28.8711 59.4143 28.6575C59.3817 28.5036 59.3674 28.2384 59.3652 27.8719L59.3775 22.6301H58.9663ZM69.9101 23.3176C70.1293 23.3176 70.3412 23.4023 70.5545 23.5692C70.7649 23.7356 70.8931 23.9206 70.9351 24.1216C71.1406 25.1089 70.2653 25.7914 68.2958 26.1717C68.2396 25.4518 68.3643 24.7983 68.6825 24.2076C68.9982 23.6174 69.4095 23.3176 69.9101 23.3176Z"
+    fill="#003B57"
+  />
+  <path
+    d="M23.8901 3.3457H4.34594C3.03607 3.3457 1.96436 4.41762 1.96436 5.72744V27.2809C1.96436 28.5907 3.03607 29.6624 4.34594 29.6624H17.2182C17.0721 23.2576 19.2593 10.8278 23.8901 3.3457Z"
+    fill="#0F80CC"
+  />
+  <path
+    d="M23.1797 4.04102H4.34564C3.41556 4.04102 2.65869 4.79769 2.65869 5.72794V25.7089C6.92421 24.0718 13.3262 22.6592 17.7528 22.7234C18.6424 18.0722 21.2567 8.95684 23.1797 4.04102Z"
+    fill="url(#paint0_linear_5444_176962)"
+  />
+  <path
+    d="M28.4485 2.6405C27.11 1.44681 25.4893 1.9263 23.8898 3.34586C23.6524 3.55676 23.4155 3.79073 23.1798 4.04067C20.4435 6.94338 17.9037 12.3201 17.1146 16.4264C17.422 17.0497 17.6621 17.8452 17.8202 18.4528C17.8607 18.6088 17.8973 18.7551 17.9265 18.8796C17.996 19.1741 18.0333 19.3652 18.0333 19.3652C18.0333 19.3652 18.0088 19.2724 17.9081 18.9805C17.889 18.9245 17.8676 18.8633 17.8423 18.7914C17.8315 18.7617 17.8165 18.7256 17.8001 18.6871C17.6216 18.2721 17.1279 17.3964 16.9106 17.0151C16.7247 17.5634 16.5605 18.0762 16.423 18.5403C17.0503 19.688 17.4325 21.6549 17.4325 21.6549C17.4325 21.6549 17.3994 21.5274 17.2418 21.0825C17.1018 20.689 16.4048 19.4678 16.2397 19.1823C15.9573 20.2249 15.8451 20.9289 15.9463 21.1003C16.1427 21.4324 16.3299 22.0054 16.4942 22.6391C16.8654 24.0668 17.1234 25.8047 17.1234 25.8047C17.1234 25.8047 17.1318 25.9199 17.1458 26.0972C17.0942 27.2961 17.1252 28.5391 17.218 29.6626C17.341 31.1499 17.5725 32.4275 17.8676 33.1113L18.0679 33.0021C17.6346 31.6549 17.4586 29.8894 17.5357 27.8533C17.6523 24.7412 18.3685 20.9881 19.6918 17.0763C21.9274 11.1711 25.0293 6.43321 27.8681 4.17055C25.2807 6.50735 21.7786 14.0714 20.7303 16.8725C19.5565 20.0092 18.7247 22.9527 18.2234 25.7728C19.0883 23.1289 21.885 21.9925 21.885 21.9925C21.885 21.9925 23.2566 20.3008 24.8596 17.884C23.8994 18.103 22.3227 18.4779 21.7946 18.6998C21.0156 19.0266 20.8057 19.1381 20.8057 19.1381C20.8057 19.1381 23.3291 17.6014 25.494 16.9056C28.4714 12.2163 31.7151 5.55451 28.4485 2.6405Z"
+    fill="#003B57"
+  />
+  <defs>
+    <linearGradient
+      id="paint0_linear_5444_176962"
+      x1="14.4914"
+      y1="4.48444"
+      x2="14.4914"
+      y2="24.5039"
+      gradientUnits="userSpaceOnUse"
+    >
+      <stop stop-color="#97D9F6" />
+      <stop offset="0.920245" stop-color="#0F80CC" />
+      <stop offset="1" stop-color="#0F80CC" />
+    </linearGradient>
+  </defs>
+</svg>

web-common/src/features/sources/modal/AddSourceModal.svelte

@@ -13,6 +13,7 @@
   import LocalFile from "../../../components/icons/connectors/LocalFile.svelte";
   import MotherDuck from "../../../components/icons/connectors/MotherDuck.svelte";
   import Postgres from "../../../components/icons/connectors/Postgres.svelte";
+  import SQLite from "../../../components/icons/connectors/SQLite.svelte";
   import { appScreen } from "../../../layout/app-store";
   import { behaviourEvent } from "../../../metrics/initMetrics";
   import {
@@ -39,6 +40,7 @@
     // athena
     "motherduck",
     "postgres",
+    "sqlite",
     "local_file",
     "https",
   ];
@@ -52,6 +54,7 @@
     // athena: AmazonAthena,
     motherduck: MotherDuck,
     postgres: Postgres,
+    sqlite: SQLite,
     local_file: LocalFile,
     https: Https,
   };

web-common/src/features/sources/modal/yupSchemas.ts

@@ -57,6 +57,18 @@ export function getYupSchema(connector: V1ConnectorSpec) {
           )
           .required("Source name is required"),
       });
+    case "sqlite":
+      return yup.object().shape({
+        db: yup.string().required("db is required"),
+        table: yup.string().required("table is required"),
+        sourceName: yup
+          .string()
+          .matches(
+            /^[a-zA-Z_][a-zA-Z0-9_]*$/,
+            "Source name must start with a letter or underscore and contain only letters, numbers, and underscores"
+          )
+          .required("Source name is required"),
+      });
     case "bigquery":
       return yup.object().shape({
         sql: yup.string().required("sql is required"),

web-common/src/features/sources/sourceUtils.ts

@@ -30,6 +30,14 @@ export function compileCreateSourceYAML(
       values.sql = buildDuckDbQuery(values.path as string);
       delete values.path;
       break;
+    case "sqlite":
+      connectorName = "duckdb";
+      values.sql = `SELECT * FROM sqlite_scan('${values.db as string}', '${
+        values.table as string
+      }');`;
+      delete values.db;
+      delete values.table;
+      break;
   }
 
   const compiledKeyValues = Object.entries(values)