Runtime: duckdb sqlite extensions based connector for sqlite

runtime/drivers/duckdb/duckdb.go

@@ -274,6 +274,8 @@ func (c *connection) reopenDB() error {
 		"LOAD 'parquet'",
 		"INSTALL 'httpfs'",
 		"LOAD 'httpfs'",
+		"INSTALL 'sqlite'",
+		"LOAD 'sqlite'",
 		"SET max_expression_depth TO 250",
 		"SET timezone='UTC'",
 	}

runtime/drivers/duckdb/transporter/sqlite_to_duckDB_test.go

@@ -0,0 +1,51 @@
+package transporter
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+
+	"github.com/rilldata/rill/runtime/drivers"
+	_ "github.com/rilldata/rill/runtime/drivers/sqlite"
+	"github.com/rilldata/rill/runtime/pkg/activity"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	_ "modernc.org/sqlite"
+)
+
+func Test_sqliteToDuckDB_Transfer(t *testing.T) {
+	tempDir := t.TempDir()
+
+	dbPath := fmt.Sprintf("%s.db", tempDir)
+	db, err := sql.Open("sqlite", dbPath)
+	require.NoError(t, err)
+
+	_, err = db.Exec(`
+	drop table if exists t;
+	create table t(i);
+	insert into t values(42), (314);
+	`)
+	require.NoError(t, err)
+	db.Close()
+
+	to, err := drivers.Open("duckdb", map[string]any{"dsn": ""}, false, activity.NewNoopClient(), zap.NewNop())
+	require.NoError(t, err)
+	olap, _ := to.AsOLAP("")
+
+	tr := &duckDBToDuckDB{
+		to:     olap,
+		logger: zap.NewNop(),
+	}
+	query := fmt.Sprintf("SELECT * FROM sqlite_scan('%s', 't');", dbPath)
+	err = tr.Transfer(context.Background(), map[string]any{"sql": query}, map[string]any{"table": "test"}, &drivers.TransferOptions{Progress: drivers.NoOpProgress{}})
+	require.NoError(t, err)
+
+	res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) from test"})
+	require.NoError(t, err)
+	res.Next()
+	var count int
+	err = res.Scan(&count)
+	require.NoError(t, err)
+	require.Equal(t, 2, count)
+}

runtime/drivers/sqlite/sqlite.go

@@ -17,6 +17,7 @@ import (
 
 func init() {
 	drivers.Register("sqlite", driver{})
+	drivers.RegisterAsConnector("sqlite", driver{})
 }
 
 type driver struct{}
@@ -55,11 +56,25 @@ func (d driver) Drop(config map[string]any, logger *zap.Logger) error {
 }
 
 func (d driver) Spec() drivers.Spec {
-	return drivers.Spec{}
+	return drivers.Spec{
+		DisplayName: "SQLite",
+		Description: "Import data from SQLite table to DuckDB using duckdb SQLite scanner.",
+		SourceProperties: []drivers.PropertySchema{
+			{
+				Key:         "sql",
+				Type:        drivers.StringPropertyType,
+				Required:    true,
+				DisplayName: "SQL",
+				Description: "Query to extract data from SQLite",
+				Placeholder: "SELECT * FROM sqlite_scan('sqlite.db', 'film');",
+				Hint:        "https://duckdb.org/docs/extensions/sqlite_scanner#querying-individual-tables",
+			},
+		},
+	}
 }
 
 func (d driver) HasAnonymousSourceAccess(ctx context.Context, src map[string]any, logger *zap.Logger) (bool, error) {
-	return false, nil
+	return true, nil
 }
 
 type connection struct {

runtime/pkg/duckdbsql/ast_traversal.go

@@ -175,7 +175,7 @@ func (a *AST) traverseTableFunction(parent astNode, childKey string) {
 	case "read_csv_auto", "read_csv",
 		"read_parquet",
 		"read_json", "read_json_auto", "read_json_objects", "read_json_objects_auto",
-		"read_ndjson_objects", "read_ndjson", "read_ndjson_auto":
+		"read_ndjson_objects", "read_ndjson", "read_ndjson_auto", "sqlite_scan":
 		ref.Paths = getListOfValues[string](arguments[0])
 	default:
 		// only read_... are supported for now

runtime/services/catalog/migrator/sources/sources.go

@@ -385,6 +385,8 @@ func connectorVariables(src *runtimev1.Source, env map[string]string, repoRoot s
 	case "motherduck":
 		vars["token"] = env["token"]
 		vars["dsn"] = ""
+	case "sqlite":
+		vars["dsn"] = ""
 	case "local_file":
 		vars["dsn"] = repoRoot
 	case "bigquery":

web-common/src/features/sources/modal/AddSourceModal.svelte

@@ -39,6 +39,7 @@
     // athena
     "motherduck",
     "postgres",
+    "sqlite",
     "local_file",
     "https",
   ];

web-common/src/features/sources/modal/yupSchemas.ts

@@ -57,6 +57,17 @@ export function getYupSchema(connector: V1ConnectorSpec) {
           )
           .required("Source name is required"),
       });
+    case "sqlite":
+      return yup.object().shape({
+        sql: yup.string().required("sql is required"),
+        sourceName: yup
+          .string()
+          .matches(
+            /^[a-zA-Z_][a-zA-Z0-9_]*$/,
+            "Source name must start with a letter or underscore and contain only letters, numbers, and underscores"
+          )
+          .required("Source name is required"),
+      });
     case "bigquery":
       return yup.object().shape({
         sql: yup.string().required("sql is required"),

web-common/src/features/sources/sourceUtils.ts

@@ -30,6 +30,9 @@ export function compileCreateSourceYAML(
       values.sql = buildDuckDbQuery(values.path as string);
       delete values.path;
       break;
+    case "sqlite":
+      connectorName = "duckdb";
+      break;
   }
 
   const compiledKeyValues = Object.entries(values)