Merge pull request #38 from TFMV/feature/flight

Add support for FlightSQL
data-catering · Jun 17, 2024 · d401166 · d401166
2 parents 0decc60 + a14f153
commit d401166
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 38 deletions.
diff --git a/README.md b/README.md
@@ -78,41 +78,43 @@ If any data is persisted from the services to carry across sessions, it gets pus
 
 ## Services
 
-| Service Type               | Service       | Supported |
-|----------------------------|---------------|-----------|
-| Change Data Capture        | debezium      | ✅         |
-| Database                   | cassandra     | ✅         |
-| Database                   | cockroachdb   | ✅         |
-| Database                   | elasticsearch | ✅         |
-| Database                   | mariadb       | ✅         |
-| Database                   | mongodb       | ✅         |
-| Database                   | mysql         | ✅         |
-| Database                   | neo4j         | ✅         |
-| Database                   | postgres      | ✅         |
-| Database                   | opensearch    | ❌         |
-| Data Catalog               | marquez       | ✅         |
-| Data Catalog               | unitycatalog  | ✅         |
-| Data Catalog               | amundsen      | ❌         |
-| Data Catalog               | datahub       | ❌         |
-| Data Catalog               | openmetadata  | ❌         |
-| Distributed Coordination   | zookeeper     | ✅         |
-| Distribute Data Processing | flink         | ✅         |
-| HTTP                       | httpbin       | ✅         |
-| Identity Management        | keycloak      | ✅         |
-| Job Orchestrator           | airflow       | ✅         |
-| Job Orchestrator           | dagster       | ✅         |
-| Job Orchestrator           | mage-ai       | ✅         |
-| Job Orchestrator           | prefect       | ✅         |
-| Messaging                  | kafka         | ✅         |
-| Messaging                  | solace        | ✅         |
-| Messaging                  | activemq      | ✅         |
-| Messaging                  | rabbitmq      | ✅         |
-| Object Storage             | minio         | ✅         |
-| Query Engine               | duckdb        | ✅         |
-| Query Engine               | presto        | ✅         |
-| Query Engine               | trino         | ✅         |
-| Real-time OLAP             | clickhouse    | ✅         |
-| Real-time OLAP             | doris         | ✅         |
-| Real-time OLAP             | druid         | ✅         |
-| Real-time OLAP             | pinot         | ✅         |
-| Test Data Management       | data-caterer  | ✅         |
+| Service Type             | Service       | Supported |
+|--------------------------|---------------|-----------|
+| Change Data Capture      | debezium      | ✅         |
+| Database                 | cassandra     | ✅         |
+| Database                 | cockroachdb   | ✅         |
+| Database                 | elasticsearch | ✅         |
+| Database                 | mariadb       | ✅         |
+| Database                 | mongodb       | ✅         |
+| Database                 | mysql         | ✅         |
+| Database                 | neo4j         | ✅         |
+| Database                 | postgres      | ✅         |
+| Database                 | opensearch    | ❌         |
+| Database                 | spanner       | ✅         |
+| Database                 | sqlite        | ✅         |
+| Data Catalog             | marquez       | ✅         |
+| Data Catalog             | amundsen      | ❌         |
+| Data Catalog             | datahub       | ❌         |
+| Data Catalog             | openmetadata  | ❌         |
+| Distributed Coordination | zookeeper     | ✅         |
+| HTTP                     | httpbin       | ✅         |
+| Identity Management      | keycloak      | ✅         |
+| Job Orchestrator         | airflow       | ✅         |
+| Job Orchestrator         | dagster       | ✅         |
+| Job Orchestrator         | mage-ai       | ✅         |
+| Job Orchestrator         | prefect       | ✅         |
+| Messaging                | kafka         | ✅         |
+| Messaging                | solace        | ✅         |
+| Messaging                | activemq      | ✅         |
+| Messaging                | rabbitmq      | ✅         |
+| Object Storage           | minio         | ✅         |
+| Query Engine             | duckdb        | ✅         |
+| Query Engine             | flight-sql    | ✅         |
+| Query Engine             | presto        | ✅         |
+| Query Engine             | trino         | ✅         |
+| Real-time OLAP           | clickhouse    | ✅         |
+| Real-time OLAP           | doris         | ✅         |
+| Real-time OLAP           | druid         | ✅         |
+| Real-time OLAP           | pinot         | ✅         |
+| Test Data Management     | data-caterer  | ✅         |
+
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,5 +1,6 @@
 version: "3.9"
 services:
+
   #change data capture
   debezium-connect:
     container_name: "debezium-connect"
@@ -233,6 +234,13 @@ services:
     depends_on:
       - "marquez-data"
 
+  spanner-emulator:
+    image: gcr.io/cloud-spanner-emulator/emulator
+    container_name: spanner-emulator
+    ports:
+      - "9010:9010"
+      - "9020:9020"
+
   #distributed coordination
   zookeeper:
     container_name: "zookeeper"
@@ -566,6 +574,22 @@ services:
       postgres:
         condition: "service_healthy"
 
+  flight-sql:
+    container_name: "flight-sql"
+    image: "voltrondata/flight-sql:latest"
+    ports:
+      - "31337:31337"
+    environment:
+      - TLS_ENABLED=1
+      - FLIGHT_PASSWORD=flight_password
+      - PRINT_QUERIES=1
+    command: ["tail", "-f", "/dev/null"]
+    volumes:
+      - "./data/flight-sql:/opt/data"
+    depends_on:
+      - duckdb
+      - sqlite
+
   presto:
     container_name: "presto"
     image: "prestodb/presto:0.287"
@@ -578,6 +602,14 @@ services:
       postgres:
         condition: "service_healthy"
 
+  sqlite:
+    container_name: "sqlite"
+    image: "nouchka/sqlite3:latest"
+    platform: "linux/amd64"
+    volumes:
+      - "./data/sqlite:/opt/data"
+    command: ["tail", "-f", "/dev/null"]
+
   trino:
     container_name: "trino"
     image: "trinodb/trino:449"

diff --git a/run.sh b/run.sh
@@ -16,6 +16,7 @@ cockroachdb='./cockroach sql --insecure'
 doris='mysql -uroot -P9030 -h127.0.0.1'
 duckdb='./duckdb'
 elasticsearch='elasticsearch-sql-cli http://elastic:elasticsearch@localhost:9200'
+flight-sql='flight_sql_client --command Execute --host localhost --port 31337 --username flight_username --password flight_password --query 'SELECT version()' --use-tls --tls-skip-verify'
 mariadb='mariadb --user=user --password=password'
 mongodb-connect='mongosh mongodb://root:root@mongodb'
 mysql='mysql -u root -proot'