add server config

conf/config.yaml

@@ -1,11 +1,15 @@
+defaults:
+  - server: serenity
+
 model:
   path:
 
 check_annotations: false
 label_studio:
-  project_name: "BOEM"
+  project_name: "Bureau of Ocean Energy Management"
   url: "https://labelstudio.naturecast.org/"
   api_key: "${oc.env:LABEL_STUDIO_API_KEY}"
+  folder_name: "/pgsql/retrieverdash/everglades-label-studio/everglades-data"
 
 pipeline:
   confidence_threshold: 0.5

conf/server/serenity.yaml

@@ -0,0 +1,4 @@
+# Remote server for image hosting
+user: 'ben'
+host: 'serenity.ifas.ufl.edu'
+key_filename: '/Users/benweinstein/.ssh/id_rsa.pub'

src/label_studio.py

@@ -196,22 +196,24 @@ def connect_to_label_studio(url, project_name, label_config=None):
 
     # Look up existing name
     projects = ls.list_projects()
-    project = [x for x in projects if x.get_params()["title"] == project_name][0]
+    project = [x for x in projects if x.get_params()["title"] == project_name]
 
-    if project is None:
+    if len(project) == 0:
         # Create a project with the specified title and labeling configuration
 
         project = ls.create_project(
             title=project_name,
             label_config=label_config
         )
+    else:
+        project = project[0]
 
     return project
 
 def create_project(ls, project_name):
     ls.create_project(title=project_name)
 
-def create_client(user, host, key_filename):
+def create_sftp_client(user, host, key_filename):
     # Download annotations from Label Studio
     # SSH connection with a user prompt for password
     ssh = paramiko.SSHClient()

src/pipeline.py

@@ -1,20 +1,19 @@
-import hydra
 from omegaconf import DictConfig
 from src.data_ingestion import DataIngestion
 from src.pipeline_evaluation import PipelineEvaluation
 from src.active_learning import choose_images
 from src.reporting import Reporting
 from src.pre_annotation_prediction import PreAnnotationPrediction
-from src.label_studio import check_for_new_annotations, upload_to_label_studio
+from src.label_studio import check_for_new_annotations, upload_to_label_studio, create_sftp_client, connect_to_label_studio
 from src.model import preprocess_and_train
 from src.data_processing import preprocess_images
 
 class Pipeline:
     def __init__(self, cfg: DictConfig):
         """Initialize the pipeline with optional configuration"""
         self.config = cfg
-        self.label_studio_client = None
-        self.pre_annotation_predictor = None
+        self.label_studio_project = connect_to_label_studio(url=self.config.label_studio.url, project_name=self.config.label_studio.project_name)
+        self.sftp_client = create_sftp_client(**self.config.server) 
 
     def run(self):
         # Check for new annotations if the check_annotations flag is set
@@ -53,5 +52,5 @@ def run(self):
             print(f"Images auto-annotated: {len(auto_annotated_images)}")
 
             # Run the annotation pipeline
-            annotations = upload_to_label_studio(images_for_human_review, **self.config)
+            annotations = upload_to_label_studio(self.sftp_client, images_for_human_review, **self.config)
             reporting.generate_reports(trained_model)

tests/conftest.py

@@ -5,6 +5,8 @@
 from hydra import initialize, compose
 import pandas as pd
 
+IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true"
+
 def get_api_key():
     """Get Label Studio API key from .comet.config file"""
     config_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.comet.config')
@@ -22,7 +24,7 @@ def config(tmpdir_factory):
     with initialize(version_base=None, config_path="../conf"):
         cfg = compose(config_name="config")
 
-    cfg["train"]["train_csv_folder"] = tmpdir_factory.mktemp("data").strpath
+    cfg.train.train_csv_folder = tmpdir_factory.mktemp("data").strpath
 
     # Create sample bounding box annotations
     data = {
@@ -44,6 +46,7 @@ def config(tmpdir_factory):
 
     return cfg
 
+@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.")
 @pytest.fixture(scope="session")
 def label_studio_client(config):
     """Initialize Label Studio client with API key from .comet.config"""
@@ -69,16 +72,23 @@ def label_studio_client(config):
             label_config=label_config
         )
 
-        # Only try to upload images if we have a valid client
-        if os.path.exists("tests/data"):
-            images = ["tests/data/" + f for f in os.listdir("tests/data/")]
-            label_studio.upload_to_label_studio(images, ls, "test_BOEM", "tests/data")
+        sftp_client = label_studio.create_sftp_client(user=config.server.user, host=config.server.host, key_filename=config.server.key_filename)
+        images = ["tests/data/" + f for f in os.listdir("tests/data/")]
+        label_studio.upload_to_label_studio(
+            images=images,
+            sftp_client=sftp_client,
+            label_studio_project=ls,
+            images_to_annotate_dir="tests/data",
+            folder_name=config.label_studio.folder_name,
+            preannotations=None
+        )
 
         return ls
     except Exception as e:
         print(f"Warning: Failed to initialize Label Studio client: {str(e)}")
         return None
 
+@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work in Github Actions.")
 @pytest.fixture(scope="session", autouse=True)
 def cleanup_label_studio(label_studio_client, request) -> Generator:
     """
@@ -88,21 +98,6 @@ def cleanup_label_studio(label_studio_client, request) -> Generator:
     # Setup: yield to allow tests to run
     yield
 
-    # Teardown: Clean up Label Studio projects only if we have a valid client
     def cleanup() -> None:
-        if label_studio_client is None:
-            return
-
-        try:
-            # Get all test projects
-            projects = label_studio_client.get_projects()
-
-            # Delete test projects
-            for project in projects:
-                if project.title.startswith('test_'):
-                    label_studio_client.delete_project(project.id)
-                    print(f"Cleaned up test project: {project.title}")
-        except Exception as e:
-            print(f"Warning: Failed to cleanup Label Studio projects: {str(e)}")
-
+        label_studio.delete_all_tasks()
     request.addfinalizer(cleanup)