Merge pull request #229 from raphaelrpl/b-0.6

improve check scenes to search by tile

bdc_collection_builder/controller.py

@@ -13,13 +13,12 @@
 from datetime import datetime, timedelta
 
 # 3rdparty
-from bdc_catalog.models import Collection, GridRefSys, Item, Provider
-from bdc_collectors.ext import BaseProvider, CollectorExtension
+from bdc_catalog.models import Collection, GridRefSys, Item
+from bdc_collectors.ext import CollectorExtension
 from celery import chain, group
 from celery.backends.database import Task
 from dateutil.relativedelta import relativedelta
 from flask import current_app
-from redis import Redis
 from sqlalchemy import Date, and_, func, or_
 from werkzeug.exceptions import BadRequest, abort
 
@@ -457,7 +456,7 @@ def get_unsuccessfully_activities(cls):
     @classmethod
     def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime,
                      catalog: str = None, dataset: str = None,
-                     grid: str = None, tiles: list = None, bbox: list = None, catalog_kwargs=None):
+                     grid: str = None, tiles: list = None, bbox: list = None, catalog_kwargs=None, only_tiles=False):
         """Check for the scenes in remote provider and compares with the Collection Builder."""
         bbox_list = []
         if grid and tiles:
@@ -472,9 +471,9 @@ def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime
                 func.ST_Ymax(func.ST_Transform(geom_table.c.geom, 4326)).label('ymax'),
             ).filter(geom_table.c.tile.in_(tiles)).all()
             for row in rows:
-                bbox_list.append((row.xmin, row.ymin, row.xmax, row.ymax))
+                bbox_list.append((row.tile, (row.xmin, row.ymin, row.xmax, row.ymax)))
         else:
-            bbox_list.append(bbox)
+            bbox_list.append(('', bbox))
 
         instance, provider = get_provider(catalog)
 
@@ -504,11 +503,16 @@ def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime
         items = {cid: set() for cid in collection_ids}
         external_scenes = set()
 
-        for _bbox in bbox_list:
+        for tile, _bbox in bbox_list:
             with redis.pipeline() as pipe:
-                options['bbox'] = _bbox
+                if only_tiles:
+                    entry = tile
+                    options['tile'] = tile
+                else:
+                    options['bbox'] = _bbox
+                    entry = _bbox
 
-                periods = _generate_periods(start_date, end_date)
+                periods = _generate_periods(start_date.replace(tzinfo=None), end_date.replace(tzinfo=None))
 
                 for period_start, period_end in periods:
                     _items = db.session.query(Item.name, Item.collection_id).filter(
@@ -532,7 +536,7 @@ def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime
                     options['start_date'] = period_start.strftime('%Y-%m-%d')
                     options['end_date'] = period_end.strftime('%Y-%m-%d')
 
-                    key = f'scenes:{catalog}:{dataset}:{period_start.strftime("%Y%m%d")}_{period_end.strftime("%Y%m%d")}_{_bbox}'
+                    key = f'scenes:{catalog}:{dataset}:{period_start.strftime("%Y%m%d")}_{period_end.strftime("%Y%m%d")}_{entry}'
 
                     pipe.get(key)
                     provider_scenes = []

bdc_collection_builder/forms.py

@@ -171,7 +171,7 @@ class CheckScenesForm(Schema):
     bbox = fields.List(fields.Float, required=False, allow_none=False, many=True)
     start_date = fields.DateTime(required=True, allow_none=False)
     end_date = fields.DateTime(required=True, allow_none=False)
-    verify = fields.Boolean(required=False, allow_none=False, default=False)
+    only_tiles = fields.Boolean(required=False, allow_none=False, default=False)
 
     @validates_schema
     def validate_form_values(self, data, **kwargs):

setup.py

@@ -59,8 +59,8 @@
     'numpy>=1.17.2',
     'numpngw>=0.0.8',
     'scikit-image>=0.16.2',
-    'SQLAlchemy[postgresql_psycopg2binary]>=1.3,<2',
-    'bdc-collectors @ git+git://github.com/brazil-data-cube/bdc-collectors.git@v0.2.1#egg=bdc-collectors',
+    'SQLAlchemy[postgresql_psycopg2binary]>=1.3,<1.4',
+    'bdc-collectors @ git+git://github.com/brazil-data-cube/bdc-collectors.git@master#egg=bdc-collectors',
     'celery[librabbitmq]>=4.3,<4.4.3',
     'Werkzeug>=0.16,<1.0',
     'shapely>=1.7,<2'