feat(openchallenges): add Mariadb Connection and Load the EDAM Concepts

apps/openchallenges/edam-etl/project.json

@@ -4,6 +4,13 @@
   "sourceRoot": "apps/openchallenges/edam-etl/src",
   "projectType": "application",
   "targets": {
+    "install-libmariadb-dev": {
+      "executor": "nx:run-commands",
+      "options": {
+        "command": "pip wheel --no-cache-dir --use-pep517 'mariadb (==1.1.10)'",
+        "cwd": "{projectRoot}"
+      }
+    },
     "create-config": {
       "executor": "nx:run-commands",
       "options": {

apps/openchallenges/edam-etl/pyproject.toml

@@ -11,6 +11,9 @@ python = "3.12.0"
 pandas = "2.2.1"
 requests = "2.31.0"
 regex = "2023.12.25"
+mariadb = "^1.1.10"
+mysql-connector = "^2.2.9"
+sqlalchemy = "^2.0.30"
 
 [tool.poetry.group.dev.dependencies]
 

apps/openchallenges/edam-etl/src/main.py

@@ -3,6 +3,10 @@
 import requests
 from os import getenv
 from typing import Optional
+import mariadb 
+import sys
+import mysql.connector
+from sqlalchemy import create_engine
 
 # Get config from the environment variables
 
@@ -11,7 +15,66 @@
 print(f"EDAM Version: {VERSION}")
 print(f"OC DB URL: {OC_DB_URL}")
 
+# Intialize required connection variables from environment variables
 
+USERNAME = getenv("USERNAME")
+PASSWORD = getenv("PASSWORD")
+PORT = getenv("PORT")
+DB = getenv("DB")
+HOST = getenv("HOST")
+
+def connect_to_mariadb(username: str, password: str, port: str, host: str, database: str, df: pd.DataFrame) -> None:
+    """Connect to the MariaDB database"""
+    try:
+        conn = mariadb.connect(
+            user = username,
+            password = password,
+            host = host,
+            port = int(port),
+            database = database
+        )
+        print("Establishing a connection to the MariaDB Platform.")
+
+        # Get the cursor
+        cur = conn.cursor()
+        print("Connection has been established to MariaDB Platform!")
+
+        # Commit the transaction
+        conn.commit()
+        print("The table edam_etl has been added to the edam database!")
+
+        # Create a SQLAlchemy engine from the MySQL Connector connection
+        engine = create_engine(f'mysql+mysqlconnector://{username}:{password}@{host}/{database}')
+
+        # Drop the table if it exists
+        cur.execute("DROP TABLE IF EXISTS edam_etl")
+
+        # Create the table with columns
+        cur.execute("""
+            CREATE TABLE edam_etl (
+                id INT PRIMARY KEY,
+                class_id VARCHAR(255),
+                preferred_label VARCHAR(255)
+            )
+        """)
+
+        #Load the concepts
+        df.to_sql(
+            name = "edam_etl",
+            con = engine,
+            if_exists = "append",
+            index = False
+        )
+
+        print("The table edam_etl has been populated with the EDAM concepts!")
+
+        # Close the connection
+        conn.close()
+
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB Platform: {e}")
+        sys.exit(1)
+
 def download_edam_csv(url: str, version: str) -> Optional[bool]:
     """Download EDAM concepts from GitHub or S3 bucket (CSV file)"""
     print("Downloading the EDAM concepts from GitHub (CSV file)...")
@@ -92,6 +155,8 @@ def print_info_statistics(df: pd.DataFrame) -> None:
     else:
         print("No data available.")
 
+# def load_edam_dataframe(df: pd.DataFrame) -> None:
+
 
 def main() -> None:
     """Main function to execute preceding functions"""
@@ -101,6 +166,7 @@ def main() -> None:
     if download_edam_csv(url, VERSION):
         df: pd.DataFrame = transform_to_dataframe(VERSION)
         print_info_statistics(df)
+    connect_to_mariadb(USERNAME, PASSWORD, PORT, HOST, DB, df)
 
 
 if __name__ == "__main__":