fixes for importing into workflows

darwin/dataset/remote_dataset.py

@@ -264,7 +264,7 @@ def fetch_remote_files(self):
         base_url = f"/datasets/{self.dataset_id}/items"
         if not self.client.feature_enabled("WORKFLOW", self.team):
             base_url = f"/datasets/{self.dataset_id}/dataset_images"
-        cursor = ""
+        cursor = "?page[size]=500"
         while True:
             response = self.client.get(f"{base_url}{cursor}", team=self.team)
             yield from response["items"]

darwin/importer/formats/csvtags.py

@@ -14,6 +14,8 @@ def parse_file(path: Path) -> Optional[List[dt.AnnotationFile]]:
         reader = csv.reader(f)
         for row in reader:
             filename, *tags = map(lambda s: s.strip(), row)
+            if filename == "":
+                continue
             annotations = [dt.make_tag(tag) for tag in tags if len(tag) > 0]
             annotation_classes = set([annotation.annotation_class for annotation in annotations])
             files.append(dt.AnnotationFile(path, filename, annotation_classes, annotations))

darwin/importer/importer.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 from typing import Callable, List, Union
+from tqdm import tqdm
 
 import darwin.datatypes as dt
 from darwin.utils import secure_continue_request
@@ -93,7 +94,7 @@ def import_annotations(
         parsed_files = importer(local_path)
         if type(parsed_files) is not list:
             parsed_files = [parsed_files]
-        for parsed_file in parsed_files:
+        for parsed_file in tqdm(parsed_files):
             image_id = remote_files[parsed_file.filename]
             _import_annotations(dataset.client, image_id, remote_classes, parsed_file.annotations, dataset)
 
@@ -108,6 +109,8 @@ def _import_annotations(client: "Client", id: int, remote_classes, annotations,
         )
 
     if client.feature_enabled("WORKFLOW", dataset.team):
-        client.post(f"/items/{id}/import", payload={"annotations": serialized_annotations})
+        res = client.post(f"/dataset_items/{id}/import", payload={"annotations": serialized_annotations})
+        if res["status_code"] != 200:
+            print(f"warning, failed to upload annotation to {id}")
     else:
         client.post(f"/dataset_images/{id}/import", payload={"annotations": serialized_annotations})

docs/torch.md

@@ -158,7 +158,7 @@ for epoch in range(10):
 ```
 
 
-## Darwin છ Detectron2
+## Darwin ✕ Detectron2
 
 This tutorial shows how to train [Detectron2](https://github.com/facebookresearch/detectron2) models in your Darwin datasets. If you do not have Detectron2 installed yet, please follow these [installation instructions](https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md).
 

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="darwin-py",
-    version="0.5.5",
+    version="0.5.6",
     author="V7",
     author_email="[email protected]",
     description="Library and command line interface for darwin.v7labs.com",