New mapping system

aero_vloc/__init__.py

@@ -17,8 +17,9 @@
 from aero_vloc.index_searchers import FaissSearcher, SequentialSearcher
 from aero_vloc.localization_pipeline import LocalizationPipeline
 from aero_vloc.map_downloader import MapDownloader
+from aero_vloc.maps import Map
 from aero_vloc.metrics import reference_recall, retrieval_recall
-from aero_vloc.primitives import Map, UAVSeq
+from aero_vloc.primitives import UAVSeq
 from aero_vloc.retrieval_system import RetrievalSystem
 from aero_vloc.utils import visualize_matches
 from aero_vloc.vpr_systems import AnyLoc, CosPlace, EigenPlaces, MixVPR, NetVLAD, SALAD

aero_vloc/feature_matchers/feature_matcher.py

@@ -11,10 +11,10 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+import numpy as np
 import torch
 
 from abc import ABC, abstractmethod
-from pathlib import Path
 
 
 class FeatureMatcher(ABC):
@@ -28,10 +28,10 @@ def __init__(self, resize: int, gpu_index: int = 0):
         print('Running inference on device "{}"'.format(self.device))
 
     @abstractmethod
-    def get_feature(self, image_path: Path):
+    def get_feature(self, image: np.ndarray):
         """
         Gets features of RGB image given
-        :param image_path: Path to the image for which features should be calculated
+        :param image: The image for which features should be calculated in OpenCV format
         :return: Features for image
         """
         pass

aero_vloc/feature_matchers/lightglue/lightglue.py

@@ -14,15 +14,14 @@
 import numpy as np
 import torch
 
-from pathlib import Path
 from tqdm import tqdm
 
 from aero_vloc.feature_detectors import SuperPoint
 from aero_vloc.feature_matchers import FeatureMatcher
 from aero_vloc.feature_matchers.lightglue.model.lightglue_matcher import (
     LightGlueMatcher,
 )
-from aero_vloc.utils import load_image_for_sp
+from aero_vloc.utils import transform_image_for_sp
 
 
 class LightGlue(FeatureMatcher):
@@ -42,8 +41,8 @@ def __init__(self, resize: int = 800, gpu_index: int = 0):
             LightGlueMatcher(features="superpoint").eval().to(self.device)
         )
 
-    def get_feature(self, image_path: Path):
-        img = load_image_for_sp(image_path, self.resize).to(self.device)
+    def get_feature(self, image: np.ndarray):
+        img = transform_image_for_sp(image, self.resize).to(self.device)
         shape = img.shape[-2:][::-1]
         with torch.no_grad():
             feats = self.super_point({"image": img})

aero_vloc/feature_matchers/superglue/superglue.py

@@ -46,15 +46,14 @@
 import numpy as np
 import torch
 
-from pathlib import Path
 from tqdm import tqdm
 
 from aero_vloc.feature_detectors import SuperPoint
 from aero_vloc.feature_matchers.feature_matcher import FeatureMatcher
 from aero_vloc.feature_matchers.superglue.model.superglue_matcher import (
     SuperGlueMatcher,
 )
-from aero_vloc.utils import load_image_for_sp
+from aero_vloc.utils import transform_image_for_sp
 
 
 class SuperGlue(FeatureMatcher):
@@ -75,8 +74,8 @@ def __init__(self, path_to_sg_weights, resize=800, gpu_index: int = 0):
             SuperGlueMatcher(path_to_sg_weights).eval().to(self.device)
         )
 
-    def get_feature(self, image_path: Path):
-        inp = load_image_for_sp(image_path, self.resize).to(self.device)
+    def get_feature(self, image: np.ndarray):
+        inp = transform_image_for_sp(image, self.resize).to(self.device)
         shape = inp.shape[2:]
         with torch.no_grad():
             features = self.super_point({"image": inp})

aero_vloc/geo_referencers/geo_referencer.py

@@ -20,7 +20,7 @@
 class GeoReferencer(ABC):
     @abstractmethod
     def get_lat_lon(
-        self, map_tile: MapTile, pixel: Tuple[int, int], resize: int
+        self, map_tile: MapTile, pixel: Tuple[int, int], resize: int = None
     ) -> Tuple[float, float]:
         """
         Finds geographic coordinates of a given pixel on a satellite image

aero_vloc/geo_referencers/google_maps_referencer.py

@@ -1,11 +1,22 @@
-import cv2
+#  Copyright (c) 2023, Ivan Moskalenko, Anastasiia Kornilova
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 import math
 
 from typing import Tuple
 
 from aero_vloc.geo_referencers.geo_referencer import GeoReferencer
 from aero_vloc.primitives import MapTile
-from aero_vloc.utils import get_new_size
 
 
 class GoogleMapsReferencer(GeoReferencer):
@@ -41,12 +52,13 @@ def __world_to_lat_lon(self, x, y):
         return lat, lon
 
     def get_lat_lon(
-        self, map_tile: MapTile, pixel: Tuple[int, int], resize: int
+        self, map_tile: MapTile, pixel: Tuple[int, int], resize: int = None
     ) -> Tuple[float, float]:
         top_left_x, top_left_y = self.__lat_lon_to_world(
             map_tile.top_left_lat, map_tile.top_left_lon
         )
-
+        if resize is None:
+            resize = max(map_tile.image.shape[:2])
         desired_x = top_left_x + (self.map_size * abs(pixel[0]) / resize) / self.scale
         desired_y = top_left_y + (self.map_size * abs(pixel[1]) / resize) / self.scale
 

aero_vloc/geo_referencers/linear_referencer.py

@@ -1,5 +1,16 @@
-import cv2
-
+#  Copyright (c) 2023, Ivan Moskalenko, Anastasiia Kornilova
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 from typing import Tuple
 
 from aero_vloc.geo_referencers.geo_referencer import GeoReferencer
@@ -9,15 +20,16 @@
 
 class LinearReferencer(GeoReferencer):
     def get_lat_lon(
-        self, map_tile: MapTile, pixel: Tuple[int, int], resize: int
+        self, map_tile: MapTile, pixel: Tuple[int, int], resize: int = None
     ) -> Tuple[float, float]:
-        map_image = cv2.imread(str(map_tile.path))
-        h_new, w_new = get_new_size(*map_image.shape[:2], resize)
+        height, width = map_tile.shape
+        if resize is not None:
+            height, width = get_new_size(height, width, resize)
 
-        lat = map_tile.top_left_lat + (abs(pixel[1]) / h_new) * (
+        lat = map_tile.top_left_lat + (abs(pixel[1]) / height) * (
             map_tile.bottom_right_lat - map_tile.top_left_lat
         )
-        lon = map_tile.top_left_lon + (abs(pixel[0]) / w_new) * (
+        lon = map_tile.top_left_lon + (abs(pixel[0]) / width) * (
             map_tile.bottom_right_lon - map_tile.top_left_lon
         )
         return lat, lon

aero_vloc/homography_estimator/homography_estimator.py

@@ -44,9 +44,7 @@ def __call__(
         if len(matched_kpts_reference) < 4:
             print("Not enough points for homography")
             return None
-        h_new, w_new = get_new_size(
-            *cv2.imread(str(query_image.path)).shape[:2], resize_param
-        )
+        h_new, w_new = get_new_size(*query_image.image.shape[:2], resize_param)
         M, mask = cv2.findHomography(
             matched_kpts_query, matched_kpts_reference, cv2.RANSAC, 5.0
         )

aero_vloc/index_searchers/sequential_searcher.py

@@ -16,7 +16,7 @@
 import numpy as np
 
 from aero_vloc.index_searchers.index_searcher import IndexSearcher
-from aero_vloc.primitives import Map
+from aero_vloc.maps import Map
 
 
 class SequentialSearcher(IndexSearcher):

aero_vloc/localization_pipeline.py

@@ -13,26 +13,22 @@
 #  limitations under the License.
 from typing import Optional, Tuple
 
-from aero_vloc.geo_referencers import GeoReferencer
 from aero_vloc.homography_estimator import HomographyEstimator
 from aero_vloc.primitives import UAVSeq
 from aero_vloc.retrieval_system import RetrievalSystem
 
 
 class LocalizationPipeline:
     """
-    Allows to create a localizator based on the retrieval system,
-    homography estimator and one of the georeference methods.
+    Allows to create a localizator based on the retrieval system and homography estimator.
     """
 
     def __init__(
         self,
         retrieval_system: RetrievalSystem,
-        geo_referencer: GeoReferencer,
         homography_estimator: HomographyEstimator,
     ):
         self.retrieval_system = retrieval_system
-        self.geo_referencer = geo_referencer
         self.homography_estimator = homography_estimator
 
     def __call__(
@@ -41,8 +37,7 @@ def __call__(
         k_closest: int,
     ) -> list[Optional[Tuple[float, float]]]:
         """
-        Calculates UAV locations using the retrieval system,
-        homography estimator and one of the georeference methods.
+        Calculates UAV locations using the retrieval system and homography estimator.
 
         :param query_seq: The sequence of images for which locations should be calculated
         :param k_closest: Specifies how many predictions for each query the global localization should make.
@@ -72,7 +67,10 @@ def __call__(
             if estimator_result is None:
                 localization_results.append(None)
                 continue
-            latitude, longitude = self.geo_referencer.get_lat_lon(
+            (
+                latitude,
+                longitude,
+            ) = self.retrieval_system.sat_map.geo_referencer.get_lat_lon(
                 chosen_sat_image,
                 estimator_result,
                 self.retrieval_system.feature_matcher.resize,

aero_vloc/map_downloader.py

@@ -31,7 +31,6 @@ def __init__(
         south_east_lat: float,
         south_east_lon: float,
         zoom: int,
-        overlap_level: float,
         api_key: str,
         folder_to_save: Path,
     ):
@@ -41,7 +40,6 @@ def __init__(
         :param south_east_lat: Latitude of the southeast point of the map
         :param south_east_lon: Longitude of the southeast point of the map
         :param zoom: Zoom level of the map
-        :param overlap_level: Shows how much neighboring images overlap each other. Float between 0 and 1
         :param api_key: API key for Google Maps API
         :param folder_to_save: Path to save map
         """
@@ -50,7 +48,6 @@ def __init__(
         self.south_east_lat = south_east_lat
         self.south_east_lon = south_east_lon
         self.zoom = zoom
-        self.overlap_level = overlap_level
         self.api_key = api_key
         self.folder_to_save = folder_to_save
 
@@ -174,9 +171,9 @@ def download_map(self):
                     f"{filename} {top_left_lat} {top_left_lon} {bottom_right_lat} {bottom_right_lon}\n"
                 )
 
-                lon = lon + (lon_step * (1 - self.overlap_level))
+                lon = lon + lon_step
                 index += 1
 
             lat_step = self.__get_lat_step(lat, lon)
-            lat = lat + (lat_step * (1 - self.overlap_level))
+            lat = lat + lat_step
         metadata_file.close()

aero_vloc/maps/__init__.py

@@ -0,0 +1,14 @@
+#  Copyright (c) 2023, Ivan Moskalenko, Anastasiia Kornilova
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from aero_vloc.maps.map import Map

aero_vloc/primitives/map.py → aero_vloc/maps/base_map.py

@@ -18,10 +18,10 @@
 from aero_vloc.primitives.map_tile import MapTile
 
 
-class Map:
+class BaseMap:
     """
-    The class represents the satellite map required for UAV localization.
-    It is assumed that the map is divided into tiles.
+    The class represents the base satellite map required for UAV localization.
+    It is assumed that the map is divided into non-overlapping tiles.
     """
 
     def __init__(self, path_to_metadata: Path):
@@ -47,23 +47,39 @@ def __init__(self, path_to_metadata: Path):
                 bottom_right_lon,
             ) = line.split()
             map_tile = MapTile(
-                map_folder / filename,
+                [[map_folder / filename]],
                 float(top_left_lat),
                 float(top_left_lon),
                 float(bottom_right_lat),
                 float(bottom_right_lon),
             )
             tiles.append(map_tile)
+        self.tiles = tiles
+        height, width = self.shape
+        tile_height, tile_width = self.tiles[0].shape
+        self.pixel_shape = height * tile_height, width * tile_width
 
-        self.width = None
-        for i, tile in enumerate(tiles[1:]):
-            if tile.top_left_lat != tiles[i].top_left_lat:
-                self.width = i + 1
+    @property
+    def shape(self) -> tuple[int, int]:
+        """
+        :return: Number of tiles by height and by width
+        """
+        width = None
+        for i, tile in enumerate(self.tiles[1:]):
+            if tile.top_left_lat != self.tiles[i].top_left_lat:
+                width = i + 1
                 break
-        if self.width is None:
-            self.width = len(tiles)
-        self.height = int(len(tiles) / self.width)
-        self.tiles = tiles
+        if width is None:
+            width = len(self.tiles)
+        height = int(len(self.tiles) / width)
+        return height, width
+
+    @property
+    def tiles_2d(self) -> np.ndarray:
+        """
+        :return: Reshaped map based on the number of tiles in height and width
+        """
+        return np.array(self.tiles).reshape(self.shape)
 
     def __iter__(self):
         for map_tile in self.tiles:
@@ -72,14 +88,18 @@ def __iter__(self):
     def __getitem__(self, key):
         return self.tiles[key]
 
+    def __len__(self):
+        return len(self.tiles)
+
     def get_neighboring_tiles(self, query_index: int) -> list[int]:
         """
         Returns the indexes of neighboring tiles
 
         :param query_index: Index of the tile for which you need to find neighbors
         :return: Neighboring tile indices
         """
-        x, y = query_index % self.width, query_index // self.width
+        height, width = self.shape
+        x, y = query_index % width, query_index // width
         potential_neighbors = [
             (x - 1, y - 1),
             (x, y - 1),
@@ -93,8 +113,8 @@ def get_neighboring_tiles(self, query_index: int) -> list[int]:
 
         result_neighbors = []
         for x, y in potential_neighbors:
-            if (0 <= x < self.width) and (0 <= y < self.height):
-                result_neighbors.append(self.width * y + x)
+            if (0 <= x < width) and (0 <= y < height):
+                result_neighbors.append(width * y + x)
         return result_neighbors
 
     def are_neighbors(self, index_1: int, index_2: int) -> bool: