extra trino worker

docker-compose/airflow/docker-compose.yml

@@ -41,7 +41,7 @@ x-airflow-common:
   &airflow-common
 #  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.0.2}
 #  image: harbor.ukserp.ac.uk/dare/airflow/dags:pr-73 #1.4.4
-  image: harbor.ukserp.ac.uk/dlm/airflow/dags:1.13.1
+  image: harbor.ukserp.ac.uk/dlm/airflow/dags:1.14.1
   environment:
     &airflow-common-env
     AIRFLOW__CORE__EXECUTOR: CeleryExecutor
@@ -54,7 +54,7 @@ x-airflow-common:
     AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
     #AIRFLOW__LOGGING__LOGGING_LEVEL: DEBUG
   volumes:
-#    - ../../dags:/opt/airflow/dags
+    #- ../../dags:/opt/airflow/dags
     - ../../../DLM-Dags/dags:/opt/airflow/dags
     - ./logs:/opt/airflow/logs
     - ./plugins:/opt/airflow/plugins

docker-compose/minio-trino/docker-compose.yml

@@ -193,6 +193,23 @@ services:
     networks:
       - miniotrino
 
+  trino-worker:
+    image: 'trinodb/trino:438'
+    hostname: trino-worker
+    depends_on:
+      nginx:
+          condition: service_started
+      hive-metastore:
+          condition: service_started
+      trino-coordinator:
+          condition: service_started
+    ports:
+      - '8091:8080'
+    volumes:
+      - ./etc2:/etc/trino
+    networks:
+      - miniotrino
+
 #  mariadb:
 #    image: 'mariadb:lts'
 #    hostname: mariadb

docker-compose/minio-trino/etc2/catalog/iceberg.properties

@@ -0,0 +1,7 @@
+connector.name=iceberg
+hive.metastore.uri=thrift://hive-metastore:9083
+hive.s3.path-style-access=true
+hive.s3.endpoint=http://minio:9000
+hive.s3.aws-access-key=minio
+hive.s3.aws-secret-key=minioadmin
+iceberg.file-format=PARQUET

docker-compose/minio-trino/etc2/catalog/minio.properties

@@ -0,0 +1,9 @@
+connector.name=hive
+hive.metastore.uri=thrift://hive-metastore:9083
+hive.s3.path-style-access=true
+hive.s3.endpoint=http://minio:9000
+hive.s3.aws-access-key=minio
+hive.s3.aws-secret-key=minioadmin
+hive.non-managed-table-writes-enabled=true
+hive.storage-format=ORC
+hive.allow-drop-table=true

docker-compose/minio-trino/etc2/config.properties

@@ -0,0 +1,3 @@
+coordinator=false
+http-server.http.port=8080
+discovery.uri=http://trino-coordinator:8080

docker-compose/minio-trino/etc2/jvm.config

@@ -0,0 +1,11 @@
+-server
+-Xmx10G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+UseGCOverheadLimit
+-XX:+ExitOnOutOfMemoryError
+-XX:ReservedCodeCacheSize=256M
+-Djdk.attach.allowAttachSelf=true
+-Djdk.nio.maxCachedBufferSize=2000000

docker-compose/minio-trino/etc2/node.properties

@@ -0,0 +1,3 @@
+node.environment=docker
+node.data-dir=/data/trino
+plugin.dir=/usr/lib/trino/plugin