SwanseaUniversityMedical · JossWhittle · Apr 10, 2024 · Apr 10, 2024 · Apr 10, 2024 · Apr 10, 2024
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
 .idea
 .coverage
 docker-compose/airflow/logs/**
+.DS_Store
+**/.DS_Store
+
+**/__pycache__
diff --git a/containers/dags/Dockerfile b/containers/dags/Dockerfile
@@ -1,17 +1,30 @@
-FROM apache/airflow:2.8.4-python3.10 as build
+FROM apache/airflow:2.9.0-python3.12 as build
 
 # Ensure that release notes get picked up from our repo and not from the apache airflow base repo
 # If the apache airflow notes are used we will incorrectly pickup the release notes from our version
 # string searched against their release history....
 LABEL org.opencontainers.image.source=https://github.com/SwanseaUniversityMedical/DARE-Airflow
 
+# Add trusted certificates
+USER root
+ADD containers/dags/tls-certs/chi-root-ca.pem /usr/local/share/ca-certificates/chi-root-ca.crt
+ADD containers/dags/tls-certs/chi-domain-ca.pem /usr/local/share/ca-certificates/chi-domain-ca.crt
+ADD containers/dags/tls-certs/bundle.pem /usr/local/share/ca-certificates/bundle.pem
+ENV SSL_CERT_FILE="/usr/local/share/ca-certificates/bundle.pem"
+
+RUN chmod 644 /usr/local/share/ca-certificates/bundle.pem && \
+    chmod 644 /usr/local/share/ca-certificates/chi-root-ca.crt && \
+    chmod 644 /usr/local/share/ca-certificates/chi-domain-ca.crt && \
+    update-ca-certificates
+
+USER airflow
+ENV SSL_CERT_FILE="/usr/local/share/ca-certificates/bundle.pem"
+ENV REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
 # Add directory to the python path so we can import modules nested within that directory
 # DAGs injected via volume mount live in /opt/airflow/dags
 # DAGs injected via git-sync live in /opt/airflow/dags/repo/dags
 ENV PYTHONPATH="/opt/airflow/dags/modules:/opt/airflow/dags/repo/dags/modules:${PYTHONPATH}"
 
-USER airflow
-
 # Install additional python dependencies
 COPY containers/dags/requirements.txt .
 RUN pip install --no-cache-dir pyclean && \
@@ -23,6 +36,5 @@ RUN pip install --no-cache-dir pyclean && \
 RUN duckcli -D ":memory:" -e "INSTALL httpfs; LOAD httpfs" && \
     python3 -c "import duckdb; duckdb.connect(':memory:').sql('LOAD httpfs')"
 
-
 # Copy dags code
 COPY dags /opt/airflow/dags
diff --git a/containers/dags/requirements.txt b/containers/dags/requirements.txt
@@ -1,12 +1,16 @@
-pika==1.3.2
-aio-pika==9.3.0
-duckdb==0.10.1
+pika
+aio-pika
+duckdb==1.0.0
 duckcli==0.2.1
-apache-airflow-providers-amazon==8.10.0
-airflow-provider-rabbitmq==0.6.1
-ydata-profiling==4.6.1
-pyarrow==14.0.1
-s3fs==2023.10.0
-pandas==2.0.3
-polars==0.19.12
-apache-airflow-providers-trino==5.4.0
+apache-airflow-providers-amazon
+airflow-provider-rabbitmq
+ydata-profiling==4.8.3
+pyarrow
+s3fs
+pandas
+polars
+apache-airflow-providers-trino
+apache-airflow-providers-hashicorp
+beartype
+minio
+jinja2
diff --git a/containers/dags/tls-certs/bundle.pem b/containers/dags/tls-certs/bundle.pem
@@ -0,0 +1,65 @@
+-----BEGIN CERTIFICATE-----
+MIIGCzCCA/OgAwIBAgIRAKGIqJbj5x6d4e4OiI2WQN0wDQYJKoZIhvcNAQELBQAw
+LjEsMCoGA1UECgwjQ0hJIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEFuc2libGUw
+HhcNMjIxMDI4MTUzMjEyWhcNMzIxMDI1MTUzMjEyWjBCMQwwCgYDVQQKDANDaGkx
+HzAdBgNVBAsMFks4cyBQcm9kIENISSBEb21haW4gQ0ExETAPBgNVBAMMCEs4cyBQ
+cm9kMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArqoz/Tp6IpdyDl3f
+OYm6JiHXBCKPNVO0bExorfMTVhc5An+2fT6eLbh/u3xavFAE3VHTuDjkP6oC0739
+te8HpzO1P0uFmlA02YEZQdgzXbgPn6fMlue+HjYQVX09fy/5/3K7QY+Bp7OTNDQh
+QWZlnaELJAin9Gkv/tAEb6PKC7NIhMeCMZihjl6OExTKhExyp3wfuV0gbBbZF5RX
+wa0nnHPaW2qPMPL1L70GYm7B+NE26ihSZ0Q7WZiUJ8fhnaQKoQapcA0hxiCyre67
+ehzJn53zoDZi0XaBAkFbcSyPKryCkVwPJ2PfGvSubGqxO//PDgX4zciYnnviqduX
+CUDBqpqiDakzYUecUuH10Y6Yb4S9wyIYBV2A8pzf5b/6C5Gs/z1B42qpru0TA+ov
+L2FqEaqDPkv3KuQOohgvflZicfAtBo5qW3LXfBRBr1vsSOv1jIuiQMuA8vtIlYSr
+rnuNbtjFmaIM/GBLTeqSmVGwb4+MGozTQJp/SAPwejen2TFh8zsIIlvYy9Gy+GM9
+9oO/IuVvaFSMrySq8DVLrifcb2bPeT3+T03nsHoveQbzpzEdxl9NwUEvvMhVebaY
+w0saplk/7HegUbGhd618K9ltl2ntalutlHWe/HiIscJ+Cve23sGS+KexMUhj28dr
+SeOEwKXG/9JfpoQO8/GQoeS80IECAwEAAaOCAQ4wggEKMG8GCCsGAQUFBwEBBGMw
+YTAuBggrBgEFBQcwAoYiaHR0cDovL3Jvb3QtY2EuY2hpLnN3YW4uYWMudWsvY3J0
+LzAvBggrBgEFBQcwAYYjaHR0cDovL3Jvb3QtY2EuY2hpLnN3YW4uYWMudWsvb2Nz
+cC8wHwYDVR0jBBgwFoAUvO8HQ3rGyrb6KU5fyKDMAcwxpOQwEgYDVR0TAQH/BAgw
+BgEB/wIBADAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vcm9vdC1jYS5jaGkuc3dh
+bi5hYy51ay9jcmwvMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU3jyk9fu4Tn0b
+UBO3fTYkjRKbgTMwDQYJKoZIhvcNAQELBQADggIBAK9LKO+AeoOZ/lfdRD1w0gxq
+Za8ILmdB7XZVMLFvIWUAWW4yDhatUSfzr0B+VHnzgIUpdPQOjme0SjtmsKyFPV8v
+fEh4Lp03B81UyqzZo9GVTfgobyOSK2QaCT76qH84vUmmaoRILK0f6aBOXqmVcybB
+KGw+pMkTxKdxRgPR+R7ATYXwcqMyi0axMPhylepjcbT/ccK23pOPuTcJsP8kBztI
+l6qLFUXuymD1sqP6wR8ZFpAEQyZdUF7I9p/MrFLPPTYzGCYyJqAjuhZvrgrP18/E
+6pwaagtaeckGNVH1Aqh+Ac1dBhnZNh2Uw+i7Nr96Tp+DpXfsmlnwX5z14mj9AIgy
+V9aGxxAOA+FUVBFEN/2zyDfNarhNlh9lxTYk6y2+BUelntw6oU+ouIseXjsJjKOh
+Yczt1iB0AvsyTBVAD/YrAGuJf8kse1yMf/oMmySQ+GXaWuXo04eXgFfu43hwZajw
+6ku3WFfCOrHc6OCAexBxyADeAfAOgUO+eTw5QuzhHMNkQNgHuGQPm1WMbg5EHJ8E
+sjxSjpCNTH/KsVw9qpamtcjZXoF03IOpR+fZ77grltWUYsVZu8xDIi+VYwkT7/Yv
+zKA3btTPvR01eDEgLrXOvlwu31ebpeVTwQr4eWYdmvsbn+CuXbDnQpYmII2TeUPq
+4TnY4JX7ejhj3XPielyR
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFKTCCAxGgAwIBAgIRAKGIqJbj5x6d4e4OiI2WQNgwDQYJKoZIhvcNAQELBQAw
+LjEsMCoGA1UECgwjQ0hJIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEFuc2libGUw
+HhcNMTYxMTE3MTYyODQzWhcNMjgxMTE0MTYyODQzWjAuMSwwKgYDVQQKDCNDSEkg
+Q2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gQW5zaWJsZTCCAiIwDQYJKoZIhvcNAQEB
+BQADggIPADCCAgoCggIBANrN76lFRk3Lq1HYzt9rBBgJJqKfWihFX4kaQZZO22ad
+H7rvmuxpNgBMzm3BuQnIFd0JYN26hx9wLOUPAvVyDHumoFa5mI3FI+FjuyMX+hQy
+GmgEz92Qn+e5T12pidMK0qVQXdxVnTRwMCjieaDLx5s3BYlEJCGvkl+v+uK1UeGd
+tRE8kle6wpLwZa1sJnSbQcoBYQCYTUUlqvCBxEnBMhACJR97ga0OBNUsk5+S/Pye
+Riga7/nzmtCvMg9PoM4UCzpeN1trNoJftGDSwPuhf7PvZ4EEYzRMKheIv47Yp2py
+6J8++3THOvE8qk2evkeNgYsb5qS9iUBNOw41TByK5v+lYpis9T0YsqDKUKkkq4OD
+/S5gh1GNbKfnkteLUHvGtPjOuYYxSM1P37URKiqH5rNQwH30ICW6V5wVyq2DM48W
+c9/QFOYRY27bf+cZOezFjadIYQnu8m8G118S6DIVelJ4qlLVdOfZ/a2lUmoAl6HP
+XL2ipI758EsFdJcTahkAy9do3hUJHVd5Ja9rV4shfyl+lVqW4gb0r2F59/TsIWmb
+tI4cGdwG6+QmdYa1DqIwaGDm81NdXy70qK7qgOvu2sew6r+u0jU1aJYQB/TwNvsa
+VoOLslSi1u179sPX92bO83e0hpaU70Y5iMGUXjBC5eHug2gTlcJSwS9aY/w5435l
+AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1Ud
+DgQWBBS87wdDesbKtvopTl/IoMwBzDGk5DANBgkqhkiG9w0BAQsFAAOCAgEAt6dI
+jSOB0Sl0+2FNoULN36H5pV5LpVK8JK2bCY//ntvXqOcDbCom86CQzdUcbTGhdxvK
+bpuhG4Rt5acQQaTvUyQL6f9oIQB6TBPocV9pJbgoE+2OKa+jNE5AS9fdNGc2AAhG
+OWzJ/620nAbVZBLdDyETtX6cix94+KLDDbuVmKtN+Nao9pUdvHJ4IEhhaldmGPD2
+hrWsA9g3CtP8NFhKfYo7qpIZNbWWRx1svz7655yP9ukvOj3Siyj/nbGGt3vqoSvM
+F2fpg5SZRCIejxc+DPUwdpMoaetRVM0okP7vZfT8ryqM1IA43US7p/ye2gOEvpAk
+lpXnpY9rXu8hdp6y/LHCjlXk5VjsX/BiRxua39idqbhMivJRAmAonr4q4PKgI1O3
+IBF9ZNo3AxinzEttDX9/CJLiQ9ejHqlECKW+nl6lMsWhV+1RskMvNnCsPUyuT4Hs
+cSxyGhcuB+1ZSpEfWinJeZ6NJAn4k4AGbQhPH3xs3mjrer3r8kg5d+1y60D5MDq6
+bzoeUuVmSTbgUBOGKQkuOz9J99gfG0PTNriy3rcs3/Kul8/flIUHc0Vcxn5LpR8O
+igzn89sJC+VxVbjleUychdZTzuRs7JhhMPKUuhBXc3D836E69jJHtioJo6oXnX4B
+OjKpgF8vJG4E0zd6sDJdvY/yPpmVjoJ7ha2cwSA=
+-----END CERTIFICATE-----
diff --git a/containers/dags/tls-certs/chi-domain-ca.pem b/containers/dags/tls-certs/chi-domain-ca.pem
@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGCzCCA/OgAwIBAgIRAKGIqJbj5x6d4e4OiI2WQN0wDQYJKoZIhvcNAQELBQAw
+LjEsMCoGA1UECgwjQ0hJIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEFuc2libGUw
+HhcNMjIxMDI4MTUzMjEyWhcNMzIxMDI1MTUzMjEyWjBCMQwwCgYDVQQKDANDaGkx
+HzAdBgNVBAsMFks4cyBQcm9kIENISSBEb21haW4gQ0ExETAPBgNVBAMMCEs4cyBQ
+cm9kMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArqoz/Tp6IpdyDl3f
+OYm6JiHXBCKPNVO0bExorfMTVhc5An+2fT6eLbh/u3xavFAE3VHTuDjkP6oC0739
+te8HpzO1P0uFmlA02YEZQdgzXbgPn6fMlue+HjYQVX09fy/5/3K7QY+Bp7OTNDQh
+QWZlnaELJAin9Gkv/tAEb6PKC7NIhMeCMZihjl6OExTKhExyp3wfuV0gbBbZF5RX
+wa0nnHPaW2qPMPL1L70GYm7B+NE26ihSZ0Q7WZiUJ8fhnaQKoQapcA0hxiCyre67
+ehzJn53zoDZi0XaBAkFbcSyPKryCkVwPJ2PfGvSubGqxO//PDgX4zciYnnviqduX
+CUDBqpqiDakzYUecUuH10Y6Yb4S9wyIYBV2A8pzf5b/6C5Gs/z1B42qpru0TA+ov
+L2FqEaqDPkv3KuQOohgvflZicfAtBo5qW3LXfBRBr1vsSOv1jIuiQMuA8vtIlYSr
+rnuNbtjFmaIM/GBLTeqSmVGwb4+MGozTQJp/SAPwejen2TFh8zsIIlvYy9Gy+GM9
+9oO/IuVvaFSMrySq8DVLrifcb2bPeT3+T03nsHoveQbzpzEdxl9NwUEvvMhVebaY
+w0saplk/7HegUbGhd618K9ltl2ntalutlHWe/HiIscJ+Cve23sGS+KexMUhj28dr
+SeOEwKXG/9JfpoQO8/GQoeS80IECAwEAAaOCAQ4wggEKMG8GCCsGAQUFBwEBBGMw
+YTAuBggrBgEFBQcwAoYiaHR0cDovL3Jvb3QtY2EuY2hpLnN3YW4uYWMudWsvY3J0
+LzAvBggrBgEFBQcwAYYjaHR0cDovL3Jvb3QtY2EuY2hpLnN3YW4uYWMudWsvb2Nz
+cC8wHwYDVR0jBBgwFoAUvO8HQ3rGyrb6KU5fyKDMAcwxpOQwEgYDVR0TAQH/BAgw
+BgEB/wIBADAzBgNVHR8ELDAqMCigJqAkhiJodHRwOi8vcm9vdC1jYS5jaGkuc3dh
+bi5hYy51ay9jcmwvMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU3jyk9fu4Tn0b
+UBO3fTYkjRKbgTMwDQYJKoZIhvcNAQELBQADggIBAK9LKO+AeoOZ/lfdRD1w0gxq
+Za8ILmdB7XZVMLFvIWUAWW4yDhatUSfzr0B+VHnzgIUpdPQOjme0SjtmsKyFPV8v
+fEh4Lp03B81UyqzZo9GVTfgobyOSK2QaCT76qH84vUmmaoRILK0f6aBOXqmVcybB
+KGw+pMkTxKdxRgPR+R7ATYXwcqMyi0axMPhylepjcbT/ccK23pOPuTcJsP8kBztI
+l6qLFUXuymD1sqP6wR8ZFpAEQyZdUF7I9p/MrFLPPTYzGCYyJqAjuhZvrgrP18/E
+6pwaagtaeckGNVH1Aqh+Ac1dBhnZNh2Uw+i7Nr96Tp+DpXfsmlnwX5z14mj9AIgy
+V9aGxxAOA+FUVBFEN/2zyDfNarhNlh9lxTYk6y2+BUelntw6oU+ouIseXjsJjKOh
+Yczt1iB0AvsyTBVAD/YrAGuJf8kse1yMf/oMmySQ+GXaWuXo04eXgFfu43hwZajw
+6ku3WFfCOrHc6OCAexBxyADeAfAOgUO+eTw5QuzhHMNkQNgHuGQPm1WMbg5EHJ8E
+sjxSjpCNTH/KsVw9qpamtcjZXoF03IOpR+fZ77grltWUYsVZu8xDIi+VYwkT7/Yv
+zKA3btTPvR01eDEgLrXOvlwu31ebpeVTwQr4eWYdmvsbn+CuXbDnQpYmII2TeUPq
+4TnY4JX7ejhj3XPielyR
+-----END CERTIFICATE-----
diff --git a/containers/dags/tls-certs/chi-root-ca.pem b/containers/dags/tls-certs/chi-root-ca.pem
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFKTCCAxGgAwIBAgIRAKGIqJbj5x6d4e4OiI2WQNgwDQYJKoZIhvcNAQELBQAw
+LjEsMCoGA1UECgwjQ0hJIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEFuc2libGUw
+HhcNMTYxMTE3MTYyODQzWhcNMjgxMTE0MTYyODQzWjAuMSwwKgYDVQQKDCNDSEkg
+Q2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gQW5zaWJsZTCCAiIwDQYJKoZIhvcNAQEB
+BQADggIPADCCAgoCggIBANrN76lFRk3Lq1HYzt9rBBgJJqKfWihFX4kaQZZO22ad
+H7rvmuxpNgBMzm3BuQnIFd0JYN26hx9wLOUPAvVyDHumoFa5mI3FI+FjuyMX+hQy
+GmgEz92Qn+e5T12pidMK0qVQXdxVnTRwMCjieaDLx5s3BYlEJCGvkl+v+uK1UeGd
+tRE8kle6wpLwZa1sJnSbQcoBYQCYTUUlqvCBxEnBMhACJR97ga0OBNUsk5+S/Pye
+Riga7/nzmtCvMg9PoM4UCzpeN1trNoJftGDSwPuhf7PvZ4EEYzRMKheIv47Yp2py
+6J8++3THOvE8qk2evkeNgYsb5qS9iUBNOw41TByK5v+lYpis9T0YsqDKUKkkq4OD
+/S5gh1GNbKfnkteLUHvGtPjOuYYxSM1P37URKiqH5rNQwH30ICW6V5wVyq2DM48W
+c9/QFOYRY27bf+cZOezFjadIYQnu8m8G118S6DIVelJ4qlLVdOfZ/a2lUmoAl6HP
+XL2ipI758EsFdJcTahkAy9do3hUJHVd5Ja9rV4shfyl+lVqW4gb0r2F59/TsIWmb
+tI4cGdwG6+QmdYa1DqIwaGDm81NdXy70qK7qgOvu2sew6r+u0jU1aJYQB/TwNvsa
+VoOLslSi1u179sPX92bO83e0hpaU70Y5iMGUXjBC5eHug2gTlcJSwS9aY/w5435l
+AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1Ud
+DgQWBBS87wdDesbKtvopTl/IoMwBzDGk5DANBgkqhkiG9w0BAQsFAAOCAgEAt6dI
+jSOB0Sl0+2FNoULN36H5pV5LpVK8JK2bCY//ntvXqOcDbCom86CQzdUcbTGhdxvK
+bpuhG4Rt5acQQaTvUyQL6f9oIQB6TBPocV9pJbgoE+2OKa+jNE5AS9fdNGc2AAhG
+OWzJ/620nAbVZBLdDyETtX6cix94+KLDDbuVmKtN+Nao9pUdvHJ4IEhhaldmGPD2
+hrWsA9g3CtP8NFhKfYo7qpIZNbWWRx1svz7655yP9ukvOj3Siyj/nbGGt3vqoSvM
+F2fpg5SZRCIejxc+DPUwdpMoaetRVM0okP7vZfT8ryqM1IA43US7p/ye2gOEvpAk
+lpXnpY9rXu8hdp6y/LHCjlXk5VjsX/BiRxua39idqbhMivJRAmAonr4q4PKgI1O3
+IBF9ZNo3AxinzEttDX9/CJLiQ9ejHqlECKW+nl6lMsWhV+1RskMvNnCsPUyuT4Hs
+cSxyGhcuB+1ZSpEfWinJeZ6NJAn4k4AGbQhPH3xs3mjrer3r8kg5d+1y60D5MDq6
+bzoeUuVmSTbgUBOGKQkuOz9J99gfG0PTNriy3rcs3/Kul8/flIUHc0Vcxn5LpR8O
+igzn89sJC+VxVbjleUychdZTzuRs7JhhMPKUuhBXc3D836E69jJHtioJo6oXnX4B
+OjKpgF8vJG4E0zd6sDJdvY/yPpmVjoJ7ha2cwSA=
+-----END CERTIFICATE-----
diff --git a/dags/__pycache__/ingest_csv_to_iceberg.cpython-310.pyc b/dags/__pycache__/ingest_csv_to_iceberg.cpython-310.pyc
diff --git a/dags/__pycache__/ingest_csv_to_iceberg2.cpython-310.pyc b/dags/__pycache__/ingest_csv_to_iceberg2.cpython-310.pyc
diff --git a/dags/__pycache__/ingest_csv_to_parquet.cpython-310.pyc b/dags/__pycache__/ingest_csv_to_parquet.cpython-310.pyc
diff --git a/dags/__pycache__/register_minio_objects.cpython-310.pyc b/dags/__pycache__/register_minio_objects.cpython-310.pyc
diff --git a/dags/constants.py b/dags/constants.py
@@ -0,0 +1,87 @@
+rabbitmq_queue_minio_event='afload'
+rabbitmq_queue_object_event='afobjectload'
+rabbitmq_queue_minio_register='afregister'
+
+rabbitmq_exchange_load='load'
+rabbitmq_exchange_load_key_s3file='s3'
+
+rabbitmq_exchange_notify='notify'
+rabbitmq_exchange_notify_key_s3file='s3'
+rabbitmq_exchange_notify_key_trino='hive'
+rabbitmq_exchange_notify_key_trino_iceberg='iceberg'
+
+rabbitmq_exchange_schema='schemarequest'
+rabbitmq_queue_schema_request_key='request'
+
+redis_expiry = 30
+
+assets3_url = 'https://cat-hdp.demo.ukserp.ac.uk/doc/GetFilteredData2?profile=dlm&Filter=%22Dataset='
+
+
+process_s3_option_default   = "default"
+process_s3_option_load   = "load"
+process_s3_option_manual = "manual"
+process_s3_option_whatif = "whatif"
+
+process_s3_formoption_yesauto = "yesAlways"
+process_s3_formoption_yesmanual = "yesManual"
+process_s3_formoption_no = "no"
+
+sql_trackingtable='''               
+CREATE TABLE IF NOT EXISTS trackingtable (
+            id VARCHAR(150), 
+            dataset VARCHAR(150), 
+            version VARCHAR(150), 
+            label VARCHAR(150), 
+            dated timestamp,
+            bucket VARCHAR(150),
+            key VARCHAR(350), 
+            tablename VARCHAR(150), 
+            physical VARCHAR(200)
+        );
+        '''
+
+sql_tracking='''                
+CREATE TABLE IF NOT EXISTS tracking (
+id VARCHAR(350), 
+bucket VARCHAR(100),
+path VARCHAR(500),
+s_marker timestamp,
+e_marker timestamp,
+d_marker INT,
+dataset VARCHAR(100),
+version VARCHAR(100),
+label VARCHAR(200),
+schema_hive VARCHAR(150),
+tablename_hive VARCHAR(200),
+location_hive VARCHAR(300),
+schema_ice VARCHAR(150),
+tablename_ice VARCHAR(200),
+location_ice VARCHAR(300),
+filesize BIGINT,
+filesize_par BIGINT,
+columns INT,
+s_download timestamp,
+e_download timestamp,
+d_download INT,
+s_convert timestamp,
+e_convert timestamp,
+d_convert INT,
+s_par timestamp,
+e_par timestamp,
+d_par INT,
+s_upload timestamp,
+e_upload timestamp,
+d_upload INT,
+s_schema timestamp,
+e_schema timestamp,
+d_schema INT,
+s_hive timestamp,
+e_hive timestamp,
+d_hive INT,
+s_iceberg timestamp,
+e_iceberg timestamp,
+d_iceberg INT,
+params VARCHAR(1000)
+        );
+        '''