TreeBoxes match DeepForest training schema

data_prep/NEON_points.py

@@ -0,0 +1,65 @@
+# NEON Point Locations
+import geopandas as gpd
+import pandas as pd
+from deepforest.utilities import crop_raster, read_file
+from utilities import find_sensor_path
+from deepforest.visualize import plot_results
+from matplotlib import pyplot as plt
+
+import glob
+import os
+
+points = gpd.read_file("/orange/idtrees-collab/DeepTreeAttention/fe902d874c4a41e4b8e5e0ddcfc9cb92/canopy_points.shp")
+
+os.makedirs("/orange/ewhite/MillionTrees/NEON_points/cropped_tiles", exist_ok=True)
+
+# Only the plot locations
+plot_counts = points['plotID'].value_counts()
+filtered_plots = plot_counts[plot_counts > 5].index
+filtered_points = points[points['plotID'].isin(filtered_plots)]
+
+annotations = []
+for plot_id in filtered_points['plotID'].unique():
+    print(plot_id)
+    plot_points = filtered_points[filtered_points['plotID'] == plot_id]
+    rgb_pool = glob.glob("/orange/ewhite/NeonData/*/DP3.30010.001/**/Camera/**/*.tif", recursive=True)
+    bounds = plot_points.total_bounds
+
+    # add small buffer of 2 meters on all sides
+    bounds[0] -= 2
+    bounds[1] -= 2
+    bounds[2] += 2
+    bounds[3] += 2
+
+    sensor_path = find_sensor_path(bounds=bounds, lookup_pool=rgb_pool)
+    #if not os.path.exists(f"/orange/ewhite/MillionTrees/NEON_points/cropped_tiles/{plot_id}.tif"):
+    try:
+        cropped_tile = crop_raster(rgb_path=sensor_path, bounds=plot_points.total_bounds, savedir="/orange/ewhite/MillionTrees/NEON_points/cropped_tiles", filename=plot_id)
+    except:
+        print(f"Failed to crop {plot_id}")
+        continue
+
+    xmin, ymin, xmax, ymax = bounds
+    plot_points['x'] = plot_points.geometry.x
+    plot_points['y'] = plot_points.geometry.y
+    plot_points['x'] -= xmin
+    plot_points['y'] -= ymin
+
+    # scale by resolution
+    resolution = 0.1
+    plot_points['x'] /= resolution
+    plot_points['y'] /= resolution
+
+    plot_points['image_path'] = cropped_tile
+
+    plotdf = plot_points[["image_path", "x", "y"]]
+    gdf = read_file(plotdf)
+    gdf["source"] = "NEON_points"
+    gdf["label"] = "Tree"
+    gdf.root_dir = "/orange/ewhite/MillionTrees/NEON_points/cropped_tiles"
+    gdf["score"] = 1
+    plot_results(gdf)
+
+    annotations.append(gdf)
+
+pd.concat(annotations).to_csv("/orange/ewhite/MillionTrees/NEON_points/annotations.csv", index=False)

data_prep/Tonga.py

@@ -0,0 +1,34 @@
+import geopandas as gpd
+from deepforest.utilities import read_file
+from deepforest.preprocess import split_raster
+from deepforest.visualize import plot_results
+import rasterio as rio
+from rasterio import warp
+import os
+from matplotlib import pyplot as plt
+
+gdf = gpd.read_file("/orange/ewhite/DeepForest/Tonga/Kolovai-Trees-20180108_projected.shp")
+
+gdf["label"] = "Tree"
+gdf["source"] = "Kolovai-Trees"
+gdf["image_path"] = "Kolovai-Trees-20180108.tif"
+
+df = read_file(gdf, root_dir="/orange/ewhite/DeepForest/Tonga")
+split_files = split_raster(df,
+            path_to_raster="/orange/ewhite/DeepForest/Tonga/Kolovai-Trees-20180108.tif",
+            patch_overlap=0,
+            patch_size=1000,
+            allow_empty=False,
+            save_dir="/orange/ewhite/DeepForest/Tonga/crops/")
+
+for image in split_files.image_path.unique():
+    image_df = split_files[split_files.image_path==image]
+    image_df.root_dir = "/orange/ewhite/DeepForest/Tonga/crops"
+    image_df["score"] = 1
+    plot_results(image_df)
+
+split_files["image_path"] = split_files["image_path"].apply(lambda x: os.path.join("/orange/ewhite/DeepForest/Tonga/crops/", x))
+split_files.to_csv("/orange/ewhite/DeepForest/Tonga/annotations.csv")
+
+
+

data_prep/collect_tasks.py

@@ -22,7 +22,8 @@
 TreePoints = [
     "/orange/ewhite/DeepForest/TreeFormer/all_images/annotations.csv",
     "/orange/ewhite/DeepForest/Ventura_2022/urban-tree-detection-data/images/annotations.csv",
-    "/orange/ewhite/MillionTrees/NEON_points/annotations.csv"]
+    "/orange/ewhite/MillionTrees/NEON_points/annotations.csv",
+    "/orange/ewhite/DeepForest/Tonga/annotations.csv"]
 
 TreePolygons = [
     "/orange/ewhite/DeepForest/Jansen_2023/pngs/annotations.csv",

data_prep/justdiggit.py

@@ -1,28 +1,32 @@
 import json
 import pandas as pd
-import random
+from deepforest.utilities import read_file
+from deepforest.visualize import plot_results
+from matplotlib import pyplot as plt
 
-def justdiggit():
-    with open("/blue/ewhite/DeepForest/justdiggit-drone/label_sample/Annotations_trees_only.json") as jsonfile:
-        data = json.load(jsonfile)    
-        ids = [x["id"] for x in data["images"]]
-        image_paths = [x["file_name"] for x in data["images"]]
-        id_df = pd.DataFrame({"id":ids,"image_path":image_paths})
-        annotation_df = []
-        for row in data["annotations"]:
-            b = {"id":row["id"],"xmin":row["bbox"][0],"ymin":row["bbox"][1],"xmax":row["bbox"][2],"ymax":row["bbox"][3]}
-            annotation_df.append(b)
-    annotation_df = pd.DataFrame(annotation_df)
-    annotations = annotation_df.merge(id_df)
-    annotations["label"] = "Tree"
-    annotations["source"] = "Justdiggit et al. 2023"
+with open("/orange/ewhite/DeepForest/justdiggit-drone/label_sample/Annotations_trees_only.json") as jsonfile:
+    data = json.load(jsonfile)    
+    ids = [x["id"] for x in data["images"]]
+    image_paths = [x["file_name"] for x in data["images"]]
+    id_df = pd.DataFrame({"id":ids,"image_path":image_paths})
+    annotation_df = []
+    for row in data["annotations"]:
+        b = {"id":row["id"],"xmin":row["bbox"][0],"ymin":row["bbox"][1],"xmax":row["bbox"][2],"ymax":row["bbox"][3]}
+        annotation_df.append(b)
+annotation_df = pd.DataFrame(annotation_df)
+annotations = annotation_df.merge(id_df)
+annotations["label"] = "Tree"
+annotations["source"] = "Justdiggit et al. 2023"
 
-    print("There are {} annotations in {} images".format(annotations.shape[0], len(annotations.image_path.unique())))
-    images = annotations.image_path.unique()
-    random.shuffle(images)
-    train_images = images[0:int(len(images)*0.8)]
-    train = annotations[annotations.image_path.isin(train_images)]
-    test = annotations[~(annotations.image_path.isin(train_images))]    
+print("There are {} annotations in {} images".format(annotations.shape[0], len(annotations.image_path.unique())))
 
-    train.to_csv("/blue/ewhite/DeepForest/justdiggit-drone/label_sample/train.csv")
-    test.to_csv("/blue/ewhite/DeepForest/justdiggit-drone/label_sample/test.csv")
+annotations["image_path"] = "/orange/ewhite/DeepForest/justdiggit-drone/label_sample/" + annotations["image_path"]
+annotations = read_file(annotations, root_dir="/orange/ewhite/DeepForest/justdiggit-drone/label_sample/")
+
+for image in annotations.image_path.unique():
+    print(image)
+    gdf = annotations[annotations.image_path == image]
+    gdf.root_dir = "/orange/ewhite/DeepForest/justdiggit-drone/label_sample/"
+    plot_results(gdf)
+    plt.savefig("current.png")
+annotations.to_csv("/orange/ewhite/DeepForest/justdiggit-drone/label_sample/train.csv")

docs/datasets.md

@@ -89,10 +89,22 @@ International Journal of Applied Earth Observation and Geoinformation, 130, 1038
 
 ### National Ecological Observatory Network
 
+![sample_image](public/NEON_points.png)
+
 **Location:** Multiple sites across the United States, see [NEON Field Sites](https://www.neonscience.org/field-sites/explore-field-sites)
 
 **Link:** [https://data.neonscience.org/data-products/DP1.10098.001](https://data.neonscience.org/data-products/DP1.10098.001)
 
+### Tonga Trees
+
+![sample_image](public/Kolovai-Trees.png)
+
+**Location:** Tonga
+
+**Link:** The provenance of this project is unclear, it was used several times in ArcGIS tutorials, but no citation was given.
+
+[https://learn.arcgis.com/en/projects/detect-objects-with-a-deep-learning-pretrained-model/](https://learn.arcgis.com/en/projects/detect-objects-with-a-deep-learning-pretrained-model/)
+
 ## Polygons
 
 ### Araujo et al. 2020