From 911e3de4c7603b5cf50171572a1d96e55c84e897 Mon Sep 17 00:00:00 2001 From: ZhuoweiWen Date: Tue, 16 Jul 2024 00:42:20 -0400 Subject: [PATCH] Import previous filings feature Signed-off-by: ZhuoweiWen --- .../celery_controller/celery_tasks.py | 204 +++++++++--------- .../database_controller/file_ops.py | 5 +- .../database_controller/kml_ops.py | 120 +++++------ .../controllers/database_controller/vt_ops.py | 7 +- back-end/database/models.py | 151 +++++++------ back-end/routes.py | 69 +++--- front-end/components/Editmap.js | 1 + front-end/components/Upload.js | 146 +++++++------ front-end/pages/previousexport.js | 2 +- 9 files changed, 369 insertions(+), 336 deletions(-) diff --git a/back-end/controllers/celery_controller/celery_tasks.py b/back-end/controllers/celery_controller/celery_tasks.py index b8d3088..1e8348b 100755 --- a/back-end/controllers/celery_controller/celery_tasks.py +++ b/back-end/controllers/celery_controller/celery_tasks.py @@ -1,4 +1,4 @@ -import logging, subprocess, os, json, uuid, cProfile +import logging, subprocess, os, json, uuid, cProfile, base64 from controllers.celery_controller.celery_config import celery from controllers.database_controller import user_ops, fabric_ops, kml_ops, mbtiles_ops, file_ops, folder_ops, vt_ops, editfile_ops, file_editfile_link_ops from database.models import file, kml_data, editfile @@ -13,115 +13,85 @@ from utils.wireless_form2args import wireless_raster_file_format, wireless_vector_file_format from controllers.signalserver_controller.raster2vector import smooth_edges + @celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) -def process_data(self, file_names, file_data_list, userid, folderid): - print(file_names) +def add_files_to_folder(self, folderid, file_contents): + logger.debug(f"folder id in add files to folder is {folderid}") try: - geojson_array = [] - tasks = [] session = Session() + for filename, content, metadata_json in file_contents: + metadata = json.loads(metadata_json) + content_bytes = base64.b64decode(content) + if (filename.endswith('.csv')): + fileVal = file_ops.create_file(filename=filename, content=content_bytes, folderid=folderid, filetype='fabric', session=session) + elif (filename.endswith('.kml') or filename.endswith('.geojson')): + downloadSpeed = metadata.get('downloadSpeed', '') + uploadSpeed = metadata.get('uploadSpeed', '') + techType = metadata.get('techType', '') + networkType = metadata.get('networkType', '') + latency = metadata.get('latency', '') + category = metadata.get('categoryCode', '') + + fileVal = file_ops.create_file(filename=filename, content=content_bytes, folderid=folderid, filetype=networkType, maxDownloadSpeed=downloadSpeed, maxUploadSpeed=uploadSpeed, techType=techType, latency=latency, category=category, session=session) - csv_file_data = [] - kml_file_data = [] - geojson_file_data = [] - # Separate file data into csv and kml - for file_name, file_data_str in zip(file_names, file_data_list): - if file_name.endswith('.csv'): - csv_file_data.append((file_name, file_data_str)) - elif (file_name.endswith('.kml')): - kml_file_data.append((file_name, file_data_str)) - elif file_name.endswith('.geojson'): - geojson_file_data.append((file_name, file_data_str)) - - for file_name, file_data_str in csv_file_data: - # Check if file name already exists in the database for this user - existing_file = session.query(file).filter(file.name==file_name, file.folder_id==folderid).first() - print(existing_file) - # If file name exists, skip to the next iteration - if existing_file and existing_file.computed: - print("skip") - continue - - # names.append(file_name) - existing_file.computed = True - session.commit() #commit the change - - file_data = json.loads(file_data_str) - - fabricid = existing_file.id - - task_id = str(uuid.uuid4()) - task = fabric_ops.write_to_db(fabricid) - tasks.append(task) + session.commit() + return folderid + except Exception as e: + session.rollback() # Rollback any changes if there's an exception + raise e + finally: + session.close() # Ensure session is closed even if there's an exception - for file_name, file_data_str in kml_file_data: +@celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) +def process_data(self, folderid, recompute): + try: + logger.debug(f"folder id in process data is {folderid}") + logger.debug(f"recompute is {recompute}") + session = Session() - existing_file = session.query(file).filter(file.name==file_name, file.folder_id==folderid).first() - print(existing_file) - # If file name exists, skip to the next iteration - if existing_file and existing_file.computed: - print("skip") - continue + csv_files = file_ops.get_files_with_postfix(folderid, '.csv', session) + + coverage_files = file_ops.get_files_with_postfix(folderid, '.kml', session) + file_ops.get_files_with_postfix(folderid, '.geojson', session) + logger.debug(coverage_files) + for file in csv_files: + if file.computed: + if not recompute: + continue + task = fabric_ops.write_to_db(file.id) + file.computed = True + + for file in coverage_files: + if file.computed: + if not recompute: + continue - # names.append(file_name) - existing_file.computed = True - session.commit() #commit the change - - file_data = json.loads(file_data_str) - - downloadSpeed = file_data.get('downloadSpeed', '') - uploadSpeed = file_data.get('uploadSpeed', '') - techType = file_data.get('techType', '') - networkType = file_data.get('networkType', '') - latency = file_data.get('latency', '') - category = file_data.get('categoryCode', '') + # if recompute: + # # delete old kml_data + # pass + downloadSpeed = file.maxDownloadSpeed + uploadSpeed = file.maxUploadSpeed + techType = file.techType + networkType = file.type + latency = file.latency + category = file.category - task_id = str(uuid.uuid4()) if networkType == "Wired": networkType = 0 else: networkType = 1 - task = kml_ops.add_network_data(folderid, existing_file.id, downloadSpeed, uploadSpeed, techType, networkType, userid, latency, category, session) - tasks.append(task) - - for file_name, file_data_str in geojson_file_data: - - existing_file = session.query(file).filter(file.name==file_name, file.folder_id==folderid).first() - print(existing_file) - # If file name exists, skip to the next iteration - if existing_file and existing_file.computed: - print("skip") - continue - - # names.append(file_name) - existing_file.computed = True - session.commit() #commit the change - - file_data = json.loads(file_data_str) - - downloadSpeed = file_data.get('downloadSpeed', '') - uploadSpeed = file_data.get('uploadSpeed', '') - techType = file_data.get('techType', '') - networkType = file_data.get('networkType', '') - latency = file_data.get('latency', '') - category = file_data.get('categoryCode', '') + task = kml_ops.add_network_data(folderid, file.id, downloadSpeed, uploadSpeed, techType, networkType, latency, category, session) - if networkType == "Wired": - networkType = 0 - else: - networkType = 1 - - task = kml_ops.add_network_data(folderid, existing_file.id, downloadSpeed, uploadSpeed, techType, networkType, userid, latency, category, session) - tasks.append(task) + file.computed = True + geojson_array = [] # This is a temporary solution, we should try optimize to use tile-join - all_kmls = session.query(file).filter(file.folder_id == folderid, file.name.endswith('kml')).all() + all_kmls = file_ops.get_files_with_postfix(folderid, '.kml', session) for kml_f in all_kmls: geojson_array.append(vt_ops.read_kml(kml_f.id, session)) - all_geojsons = session.query(file).filter(file.folder_id == folderid, file.name.endswith('geojson')).all() + all_geojsons = file_ops.get_files_with_postfix(folderid, '.geojson', session) for geojson_f in all_geojsons: geojson_array.append(vt_ops.read_geojson(geojson_f.id, session)) @@ -129,7 +99,7 @@ def process_data(self, file_names, file_data_list, userid, folderid): session.commit() print("finished kml processing, now creating tiles") - vt_ops.create_tiles(geojson_array, userid, folderid, session) + vt_ops.create_tiles(geojson_array, folderid, session) session.close() @@ -198,7 +168,7 @@ def deleteFiles(self, fileid, userid): all_geojsons = file_ops.get_files_with_postfix(folderid, '.geojson', session) for geojson_f in all_geojsons: geojson_array.append(vt_ops.read_geojson(geojson_f.id, session)) - vt_ops.create_tiles(geojson_array, userid, folderid, session) + vt_ops.create_tiles(geojson_array, folderid, session) return {'message': 'mbtiles successfully deleted'} # Returning a dictionary except Exception as e: session.rollback() # Rollback the session in case of error @@ -259,7 +229,7 @@ def toggle_tiles(self, markers, userid, folderid, polygonfeatures): geojson_data.append(vt_ops.read_geojson(geojson_f.id, session)) mbtiles_ops.delete_mbtiles(user_folder.id, session) - vt_ops.create_tiles(geojson_data, userid, user_folder.id, session) + vt_ops.create_tiles(geojson_data, user_folder.id, session) if user_folder.type == 'export': existing_csvs = file_ops.get_files_by_type(folderid=user_folder.id, filetype='export', session=session) for csv_file in existing_csvs: @@ -299,21 +269,42 @@ def toggle_tiles(self, markers, userid, folderid, polygonfeatures): @celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) def async_folder_copy_for_export(self, userid, folderid, serialized_csv): - session = Session() - userVal = user_ops.get_user_with_id(userid) - current_datetime = datetime.now().strftime(DATETIME_FORMAT) - newfolder_name = f"export-{current_datetime}" + try: + session = Session() + userVal = user_ops.get_user_with_id(userid) + deadline = datetime.now() + current_datetime = datetime.strptime(deadline, '%Y-%m-%d').date() + newfolder_name = f"Exported Filing at {current_datetime}" - csv_name = EXPORT_CSV_NAME_TEMPLATE.format(brand_name=userVal.brand_name, current_datetime=current_datetime) + csv_name = EXPORT_CSV_NAME_TEMPLATE.format(brand_name=userVal.brand_name, current_datetime=current_datetime) + + original_folder = folder_ops.get_folder_with_id(userid=userid, folderid=folderid, session=session) + new_folder = original_folder.copy(name=newfolder_name,type='export', deadline=current_datetime, export=True, session=session) + csv_file = file_ops.create_file(filename=csv_name, content=serialized_csv.encode('utf-8'), folderid=new_folder.id, filetype='export', session=session) + session.add(csv_file) + session.commit() + except Exception as e: + session.rollback() # Rollback any changes if there's an exception + raise e + finally: + session.close() # Ensure session is closed even if there's an exception - original_folder = folder_ops.get_folder_with_id(userid=userid, folderid=folderid, session=session) - new_folder = original_folder.copy(name=newfolder_name,type='export', session=session) - csv_file = file_ops.create_file(filename=csv_name, content=serialized_csv.encode('utf-8'), folderid=new_folder.id, filetype='export', session=session) - session.add(csv_file) - session.commit() - session.close() +@celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) +def async_folder_copy_for_import(self, userid, folderid, deadline): + try: + session = Session() + newfolder_name = f"Filing for Deadline {deadline}" + original_folder = folder_ops.get_folder_with_id(userid=userid, folderid=folderid, session=session) + new_folder = original_folder.copy(name=newfolder_name, type='upload', deadline=deadline, export=False, session=session) + session.commit() + return new_folder.id + except Exception as e: + session.rollback() # Rollback any changes if there's an exception + raise e + finally: + session.close() # Ensure session is closed even if there's an exception @celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) def run_signalserver(self, command, outfile_name, tower_id, data): @@ -455,7 +446,6 @@ def raster2vector(self, data, userid, outfile_name): with open(vector_file_name, 'rb') as vector_file: kml_binarydata = vector_file.read() fileVal = file_ops.create_file(vector_file_name, kml_binarydata, folderVal.id, 'wireless', session=session) - fileVal.computed = True session.commit() downloadSpeed = data['downloadSpeed'] uploadSpeed = data['uploadSpeed'] @@ -463,7 +453,7 @@ def raster2vector(self, data, userid, outfile_name): latency = data['latency'] category = data['categoryCode'] - kml_ops.compute_wireless_locations(fileVal.folder_id, fileVal.id, downloadSpeed, uploadSpeed, techType, userid, latency, category, session) + kml_ops.compute_wireless_locations(fileVal.folder_id, fileVal.id, downloadSpeed, uploadSpeed, techType, latency, category, session) geojson_array = [] all_kmls = file_ops.get_files_with_postfix(fileVal.folder_id, '.kml', session) for kml_f in all_kmls: @@ -475,7 +465,7 @@ def raster2vector(self, data, userid, outfile_name): logger.info("Creating Vector Tiles") mbtiles_ops.delete_mbtiles(fileVal.folder_id, session) session.commit() - vt_ops.create_tiles(geojson_array, userid, fileVal.folder_id, session) + vt_ops.create_tiles(geojson_array, fileVal.folder_id, session) for f_extension in wireless_vector_file_format: os.remove(outfile_name + f_extension) diff --git a/back-end/controllers/database_controller/file_ops.py b/back-end/controllers/database_controller/file_ops.py index 5b8fba0..1172586 100755 --- a/back-end/controllers/database_controller/file_ops.py +++ b/back-end/controllers/database_controller/file_ops.py @@ -137,14 +137,14 @@ def get_file_with_name(filename, folderid, session=None): session.close() -def create_file(filename, content, folderid, filetype=None, session=None): +def create_file(filename, content, folderid, filetype=None, maxDownloadSpeed=None, maxUploadSpeed=None, techType=None, latency=None, category=None, session=None): owns_session = False if session is None: session = Session() owns_session = True try: - new_file = file(name=filename, data=content, folder_id=folderid, timestamp=datetime.now(), type=filetype) + new_file = file(name=filename, data=content, folder_id=folderid, timestamp=datetime.now(), type=filetype, maxDownloadSpeed=maxDownloadSpeed, maxUploadSpeed=maxUploadSpeed, techType=techType, latency=latency, category=category) session.add(new_file) if owns_session: session.commit() @@ -198,7 +198,6 @@ def get_filesinfo_in_folder(folderid, session=None): 'timestamp': file.timestamp, 'folder_id': file.folder_id, 'type': file.type, - 'computed': file.computed, 'kml_data': None } diff --git a/back-end/controllers/database_controller/kml_ops.py b/back-end/controllers/database_controller/kml_ops.py index 0ac6a3c..e6eb2da 100755 --- a/back-end/controllers/database_controller/kml_ops.py +++ b/back-end/controllers/database_controller/kml_ops.py @@ -6,22 +6,23 @@ from database.sessions import ScopedSession, Session from datetime import datetime import logging, uuid, psycopg2, io, pandas, geopandas, shapely -from shapely.geometry import Point +from shapely.geometry import Point, shape import fiona from io import StringIO, BytesIO from .user_ops import get_user_with_id from .file_ops import get_files_with_postfix, get_file_with_id, get_files_with_postfix, create_file, get_files_by_type from .folder_ops import create_folder, get_upload_folder +from .file_editfile_link_ops import get_editfiles_for_file from utils.logger_config import logger +import json db_lock = Lock() -def get_kml_data(userid, folderid, session=None): +def get_kml_data(folderid, session=None): owns_session = False if session is None: session = Session() owns_session = True - userVal = get_user_with_id(userid, session) try: # Query the File records related to the folder_id @@ -49,7 +50,6 @@ def get_kml_data(userid, folderid, session=None): 'served': False, 'wireless': False, 'lte': False, - 'username': userVal.username, 'coveredLocations': "", 'maxDownloadNetwork': -1, 'maxDownloadSpeed': -1 @@ -62,7 +62,6 @@ def get_kml_data(userid, folderid, session=None): kml_data.served, kml_data.wireless, kml_data.lte, - kml_data.username, kml_data.coveredLocations, kml_data.maxDownloadNetwork, kml_data.maxDownloadSpeed @@ -79,10 +78,9 @@ def get_kml_data(userid, folderid, session=None): 'served': r[1], 'wireless': r[2] or existing_data.get('wireless', False), 'lte': r[3] or existing_data.get('lte', False), - 'username': r[4], - 'coveredLocations': ', '.join(filter(None, [r[5], existing_data.get('coveredLocations', '')])), - 'maxDownloadNetwork': r[6] if r[7] > existing_data.get('maxDownloadSpeed', -1) else existing_data.get('maxDownloadNetwork', ''), - 'maxDownloadSpeed': max(r[7], existing_data.get('maxDownloadSpeed', -1)) + 'coveredLocations': ', '.join(filter(None, [r[4], existing_data.get('coveredLocations', '')])), + 'maxDownloadNetwork': r[5] if r[6] > existing_data.get('maxDownloadSpeed', -1) else existing_data.get('maxDownloadNetwork', ''), + 'maxDownloadSpeed': max(r[6], existing_data.get('maxDownloadSpeed', -1)) } # Merge the existing and new data all_data[r[0]].update(new_data) @@ -104,7 +102,6 @@ def get_kml_data(userid, folderid, session=None): kml_data.served, kml_data.wireless, kml_data.lte, - kml_data.username, kml_data.coveredLocations, kml_data.maxDownloadNetwork, kml_data.maxDownloadSpeed, @@ -124,13 +121,12 @@ def get_kml_data(userid, folderid, session=None): 'served': r[1], 'wireless': r[2] or existing_data.get('wireless', False), 'lte': r[3] or existing_data.get('lte', False), - 'username': r[4], - 'coveredLocations': ', '.join(filter(None, [r[5], existing_data.get('coveredLocations', '')])), - 'maxDownloadNetwork': r[6] if r[7] > existing_data.get('maxDownloadSpeed', -1) else existing_data.get('maxDownloadNetwork', ''), - 'maxDownloadSpeed': max(r[7], existing_data.get('maxDownloadSpeed', -1)), - 'address': r[8], - 'latitude': r[9], - 'longitude': r[10], + 'coveredLocations': ', '.join(filter(None, [r[4], existing_data.get('coveredLocations', '')])), + 'maxDownloadNetwork': r[5] if r[6] > existing_data.get('maxDownloadSpeed', -1) else existing_data.get('maxDownloadNetwork', ''), + 'maxDownloadSpeed': max(r[6], existing_data.get('maxDownloadSpeed', -1)), + 'address': r[7], + 'latitude': r[8], + 'longitude': r[9], 'bsl': 'False', } # Merge the existing and new data @@ -160,10 +156,9 @@ def get_kml_data_by_file(fileid, session=None): if owns_session: session.close() -def add_to_db(pandaDF, kmlid, download, upload, tech, wireless, userid, latency, category, session): +def add_to_db(pandaDF, kmlid, download, upload, tech, wireless, latency, category, session): batch = [] - userVal = get_user_with_id(userid) fileVal = get_file_with_id(kmlid) for _, row in pandaDF.iterrows(): @@ -179,7 +174,6 @@ def add_to_db(pandaDF, kmlid, download, upload, tech, wireless, userid, latency, served = True, wireless = wireless, lte = False, - username = userVal.username, coveredLocations = fileVal.name, maxDownloadNetwork = fileVal.name, maxDownloadSpeed = int(download), @@ -253,44 +247,6 @@ def export(userid, folderid, providerid, brandname, session): return output -def compute_lte(folderid, geojsonid, download, upload, tech, userid, latency, category): - kml_files = get_files_with_postfix(folderid, '.kml') - kml_file_ids = [file.id for file in kml_files] - - geojson_file = get_file_with_id(geojsonid) - geojson_content = BytesIO(geojson_file.data) - - session = ScopedSession() - - # Load the required points from the kml_data table - kml_data_points = session.query(kml_data).filter(kml_data.file_id.in_(kml_file_ids), kml_data.wireless == True).all() - - # Convert those points to a GeoDataFrame - df = pandas.DataFrame([(point.id, point.longitude, point.latitude) for point in kml_data_points], columns=['id', 'longitude', 'latitude']) - gdf_points = geopandas.GeoDataFrame(df, crs="EPSG:4326", geometry=geopandas.points_from_xy(df.longitude, df.latitude)) - - # Load geoJSON polygon data - lte_coverage = geopandas.read_file(geojson_content) - lte_coverage = lte_coverage.to_crs("EPSG:4326") - - # Perform a spatial join to determine which points are within the polygon - points_in_coverage = geopandas.sjoin(gdf_points, lte_coverage, predicate='within') - - # Update the lte column based on results - covered_ids = points_in_coverage['id'].to_list() - for point in kml_data_points: - if point.id in covered_ids: - point.lte = True - point.file_id = geojsonid - point.maxDownloadNetwork = download - point.maxUploadSpeed = upload - point.techType = tech - point.coveredLocations = geojson_file.name - point.latency = latency - point.category = category - - session.commit() - session.close() def rename_conflicting_columns(gdf): """Rename 'index_left' and 'index_right' columns if they exist in a GeoDataFrame.""" @@ -301,8 +257,34 @@ def rename_conflicting_columns(gdf): rename_dict['index_right'] = 'index_right_original' return gdf.rename(columns=rename_dict) -#might need to add lte data in the future -def compute_wireless_locations(folderid, kmlid, download, upload, tech, userid, latency, category, session): +def filter_points_within_editfile_polygons(points_gdf, coverage_file, session): + all_polygons = [] + + editfiles = get_editfiles_for_file(coverage_file.id, session) + for editfile in editfiles: + try: + geojson_feature = json.loads(editfile.data.decode('utf-8')) + if geojson_feature['geometry']['type'] == 'Polygon': + polygon = shape(geojson_feature['geometry']) + all_polygons.append(polygon) + except json.JSONDecodeError: + logger.error("Failed to decode JSON data") + continue + + if all_polygons: + polygons_gdf = geopandas.GeoDataFrame(geometry=all_polygons, crs="EPSG:4326") + # Rename conflicting columns before the spatial join + points_gdf = rename_conflicting_columns(points_gdf) + polygons_gdf = rename_conflicting_columns(polygons_gdf) + + # Perform spatial join + points_gdf = geopandas.sjoin(points_gdf, polygons_gdf, how="left", predicate="within") + # Keep only points that are NOT within any polygon + points_gdf = points_gdf[points_gdf.index_right.isna()] + + return points_gdf + +def compute_wireless_locations(folderid, kmlid, download, upload, tech, latency, category, session): fabric_files = get_files_with_postfix(folderid, '.csv') coverage_file = get_file_with_id(kmlid) @@ -337,8 +319,11 @@ def compute_wireless_locations(folderid, kmlid, download, upload, tech, userid, bsl_fabric_in_wireless = fabric_in_wireless[fabric_in_wireless['bsl_flag']] bsl_fabric_in_wireless = bsl_fabric_in_wireless.drop_duplicates(subset='location_id', keep='first') + bsl_fabric_in_wireless = filter_points_within_editfile_polygons(bsl_fabric_in_wireless, coverage_file, session) + bsl_fabric_in_wireless = bsl_fabric_in_wireless.drop_duplicates(subset='location_id', keep='first') + logger.debug(f'number of points computed: {len(bsl_fabric_in_wireless)}') - res = add_to_db(bsl_fabric_in_wireless, kmlid, download, upload, tech, True, userid, latency, category, session) + res = add_to_db(bsl_fabric_in_wireless, kmlid, download, upload, tech, True, latency, category, session) return res def preview_wireless_locations(folderid, kml_filename): @@ -379,7 +364,7 @@ def preview_wireless_locations(folderid, kml_filename): logger.debug(f'number of points computed: {len(bsl_fabric_in_wireless)}') return bsl_fabric_in_wireless -def compute_wired_locations(folderid, kmlid, download, upload, tech, userid, latency, category, session): +def compute_wired_locations(folderid, kmlid, download, upload, tech, latency, category, session): # Fetch Fabric file from database @@ -428,17 +413,18 @@ def compute_wired_locations(folderid, kmlid, download, upload, tech, userid, lat bsl_fabric_near_fiber = fabric_near_fiber[fabric_near_fiber['bsl_flag']] bsl_fabric_near_fiber = bsl_fabric_near_fiber.drop_duplicates(subset='location_id', keep='first') - # bsl_fabric_near_fiber = bsl_fabric_near_fiber.drop_duplicates() + bsl_fabric_near_fiber = filter_points_within_editfile_polygons(bsl_fabric_near_fiber, fiber_file_record, session) + bsl_fabric_near_fiber = bsl_fabric_near_fiber.drop_duplicates(subset='location_id', keep='first') - res = add_to_db(bsl_fabric_near_fiber, kmlid, download, upload, tech, False, userid, latency, category, session) + res = add_to_db(bsl_fabric_near_fiber, kmlid, download, upload, tech, False, latency, category, session) return res -def add_network_data(folderid, kmlid ,download, upload, tech, type, userid, latency, category, session): +def add_network_data(folderid, kmlid ,download, upload, tech, type, latency, category, session): res = False if type == 0: - res = compute_wired_locations(folderid, kmlid, download, upload, tech, userid, latency, category, session) + res = compute_wired_locations(folderid, kmlid, download, upload, tech, latency, category, session) elif type == 1: - res = compute_wireless_locations(folderid, kmlid, download, upload, tech, userid, latency, category, session) + res = compute_wireless_locations(folderid, kmlid, download, upload, tech, latency, category, session) return res diff --git a/back-end/controllers/database_controller/vt_ops.py b/back-end/controllers/database_controller/vt_ops.py index e04ae50..9da67ff 100755 --- a/back-end/controllers/database_controller/vt_ops.py +++ b/back-end/controllers/database_controller/vt_ops.py @@ -204,9 +204,9 @@ def tiles_join(geojson_data, folderid, session): -def create_tiles(geojson_array, userid, folderid, session): +def create_tiles(geojson_array, folderid, session): from controllers.celery_controller.celery_tasks import run_tippecanoe - network_data = get_kml_data(userid, folderid, session) + network_data = get_kml_data(folderid, session) if network_data: point_geojson = { "type": "FeatureCollection", @@ -219,7 +219,6 @@ def create_tiles(geojson_array, userid, folderid, session): "address": point['address'], "wireless": point['wireless'], 'lte': point['lte'], - 'username': point['username'], 'network_coverages': point['coveredLocations'], 'maxDownloadNetwork': point['maxDownloadNetwork'], 'maxDownloadSpeed': point['maxDownloadSpeed'], @@ -243,7 +242,7 @@ def create_tiles(geojson_array, userid, folderid, session): with open(unique_geojson_filename, 'w') as f: json.dump(point_geojson, f) - outputFile = "output" + str(userid) + ".mbtiles" + outputFile = "output" + str(folderid) + ".mbtiles" command = f"tippecanoe -o {outputFile} --base-zoom=7 -P --maximum-tile-bytes=3000000 -z 16 --drop-densest-as-needed {unique_geojson_filename} --force --use-attribute-for-id=location_id --layer=data" run_tippecanoe(command, folderid, outputFile) diff --git a/back-end/database/models.py b/back-end/database/models.py index 45709b6..37ea20f 100755 --- a/back-end/database/models.py +++ b/back-end/database/models.py @@ -66,28 +66,47 @@ class folder(Base): #filing, will change the name later for less confusion when id = Column(Integer, primary_key=True) name = Column(String, nullable=False) type = Column(String, default='upload') # Currently upload or export + deadline = Column(Date) #This deadline makes it a filing for the current period user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE')) user = relationship('user', back_populates='folders') - deadline = Column(Date) #This deadline makes it a filing for the current period files = relationship('file', back_populates='folder', cascade='all, delete') mbtiles = relationship('mbtiles', back_populates='folder', cascade='all, delete') editfiles = relationship('editfile', back_populates='folder', cascade='all,delete') - def copy(self, session, name=None, type=None): + def copy(self, session, export=True, name=None, type=None, deadline=None): name = name if name is not None else self.name type = type if type is not None else self.type - new_folder = folder(name=name, type=type, user_id=self.user_id) + new_folder = folder(name=name, type=type, user_id=self.user_id, deadline = deadline) session.add(new_folder) session.flush() # To generate an ID for the new folder + + new_file_mapping = {} + new_editfile_mapping = {} + # Copy related files, mbtiles for file in self.files: - file.copy(session=session, new_folder_id=new_folder.id) - for edit_file in self.edit_files: - edit_file.copy(session=session, new_folder_id=new_folder.id) - for mbtile in self.mbtiles: - mbtile.copy(session=session, new_folder_id=new_folder.id) + if not export and file.name.endswith('.csv'): + continue + new_file = file.copy(session=session, new_folder_id=new_folder.id, export=export) + new_file_mapping[file.id] = new_file.id + for edit_file in self.editfiles: + new_edit_file = edit_file.copy(session=session, new_folder_id=new_folder.id) + new_editfile_mapping[edit_file.id] = new_edit_file.id + + # Only copy + if export: + for mbtile in self.mbtiles: + mbtile.copy(session=session, new_folder_id=new_folder.id) + for file in self.files: + for link in file.editfile_links: + if link.editfile_id in new_editfile_mapping: # Ensure the editfile was copied + new_link = file_editfile_link( + file_id=new_file_mapping[file.id], + editfile_id=new_editfile_mapping[link.editfile_id] + ) + session.add(new_link) return new_folder class file_editfile_link(Base): @@ -109,70 +128,81 @@ class file(Base): folder_id = Column(Integer, ForeignKey('folder.id', ondelete='CASCADE')) timestamp = Column(DateTime) type = Column(String) + maxDownloadSpeed = Column(Integer) + maxUploadSpeed = Column(Integer) + techType = Column(String) + latency = Column(String) + category = Column(String) computed = Column(Boolean, default=False) folder = relationship('folder', back_populates='files') fabric_data = relationship('fabric_data', back_populates='file', cascade='all, delete') # Use fabric_data instead of data_entries kml_data = relationship('kml_data', back_populates='file', cascade='all, delete') editfile_links = relationship("file_editfile_link", back_populates="file") - def copy(self, session, new_folder_id=None): + + def copy(self, session, export, new_folder_id=None): new_file = file(name=self.name, data=self.data, folder_id=new_folder_id, timestamp=datetime.now(), - type=self.type, - computed=self.computed) + type=self.type, + maxDownloadSpeed=self.maxDownloadSpeed, + maxUploadSpeed=self.maxUploadSpeed, + techType=self.techType, + latency=self.latency, + category=self.category, + ) session.add(new_file) session.flush() - fabric_data_copies = [] - for fabric_entry in self.fabric_data: - fabric_data_copy = { - "location_id":fabric_entry.location_id, - "address_primary" : fabric_entry.address_primary, - "city" : fabric_entry.city, - "state" : fabric_entry.state, - "zip_code" : fabric_entry.zip_code, - "zip_suffix" : fabric_entry.zip_suffix, - "unit_count" : fabric_entry.unit_count, - "bsl_flag" : fabric_entry.bsl_flag, - "building_type_code" : fabric_entry.building_type_code, - "land_use_code" : fabric_entry.land_use_code, - "address_confidence_code" : fabric_entry.address_confidence_code, - "country_geoid" : fabric_entry.country_geoid, - "block_geoid" : fabric_entry.block_geoid, - "h3_9" : fabric_entry.h3_9, - "latitude" : fabric_entry.latitude, - "longitude" : fabric_entry.longitude, - "fcc_rel" : fabric_entry.fcc_rel, - "file_id" : new_file.id - } - fabric_data_copies.append(fabric_data_copy) - - kml_data_copies = [] - for kml_entry in self.kml_data: - kml_data_copy = { - "location_id":kml_entry.location_id, - "served" : kml_entry.served, - "wireless" : kml_entry.wireless, - "lte" : kml_entry.lte, - "username" : kml_entry.username, - "coveredLocations" : kml_entry.coveredLocations, - "maxDownloadNetwork" : kml_entry.maxDownloadNetwork, - "maxDownloadSpeed" : kml_entry.maxDownloadSpeed, - "maxUploadSpeed" : kml_entry.maxUploadSpeed, - "techType" : kml_entry.techType, - "address_primary" : kml_entry.address_primary, - "longitude" : kml_entry.longitude, - "latitude" : kml_entry.latitude, - "latency" : kml_entry.latency, - "category" : kml_entry.category, - "file_id": new_file.id - } - kml_data_copies.append(kml_data_copy) - - session.bulk_insert_mappings(fabric_data, fabric_data_copies) - session.bulk_insert_mappings(kml_data, kml_data_copies) + if export: + fabric_data_copies = [] + for fabric_entry in self.fabric_data: + fabric_data_copy = { + "location_id":fabric_entry.location_id, + "address_primary" : fabric_entry.address_primary, + "city" : fabric_entry.city, + "state" : fabric_entry.state, + "zip_code" : fabric_entry.zip_code, + "zip_suffix" : fabric_entry.zip_suffix, + "unit_count" : fabric_entry.unit_count, + "bsl_flag" : fabric_entry.bsl_flag, + "building_type_code" : fabric_entry.building_type_code, + "land_use_code" : fabric_entry.land_use_code, + "address_confidence_code" : fabric_entry.address_confidence_code, + "country_geoid" : fabric_entry.country_geoid, + "block_geoid" : fabric_entry.block_geoid, + "h3_9" : fabric_entry.h3_9, + "latitude" : fabric_entry.latitude, + "longitude" : fabric_entry.longitude, + "fcc_rel" : fabric_entry.fcc_rel, + "file_id" : new_file.id + } + fabric_data_copies.append(fabric_data_copy) + + kml_data_copies = [] + for kml_entry in self.kml_data: + kml_data_copy = { + "location_id":kml_entry.location_id, + "served" : kml_entry.served, + "wireless" : kml_entry.wireless, + "lte" : kml_entry.lte, + "coveredLocations" : kml_entry.coveredLocations, + "maxDownloadNetwork" : kml_entry.maxDownloadNetwork, + "maxDownloadSpeed" : kml_entry.maxDownloadSpeed, + "maxUploadSpeed" : kml_entry.maxUploadSpeed, + "techType" : kml_entry.techType, + "address_primary" : kml_entry.address_primary, + "longitude" : kml_entry.longitude, + "latitude" : kml_entry.latitude, + "latency" : kml_entry.latency, + "category" : kml_entry.category, + "file_id": new_file.id + } + kml_data_copies.append(kml_data_copy) + + session.bulk_insert_mappings(fabric_data, fabric_data_copies) + session.bulk_insert_mappings(kml_data, kml_data_copies) return new_file @@ -196,6 +226,8 @@ def copy(self, session, new_folder_id=None): session.add(new_edit_file) session.flush() + return new_edit_file + class fabric_data(Base): __tablename__ = 'fabric_data' location_id = Column(Integer) @@ -247,7 +279,6 @@ class kml_data(Base): served = Column(Boolean) wireless = Column(Boolean) lte = Column(Boolean) - username = Column(String) coveredLocations = Column(String) maxDownloadNetwork = Column(String) maxDownloadSpeed = Column(Integer) diff --git a/back-end/routes.py b/back-end/routes.py index 12cfc78..ca629ae 100755 --- a/back-end/routes.py +++ b/back-end/routes.py @@ -15,12 +15,13 @@ from datetime import datetime import shortuuid from celery.result import AsyncResult +from celery import chain from utils.settings import DATABASE_URL, COOKIE_EXP_TIME, backend_port from database.sessions import Session from controllers.database_controller import fabric_ops, kml_ops, user_ops, vt_ops, file_ops, folder_ops, mbtiles_ops, challenge_ops, editfile_ops from utils.flask_app import app, jwt from controllers.celery_controller.celery_config import celery -from controllers.celery_controller.celery_tasks import process_data, deleteFiles, toggle_tiles, run_signalserver, raster2vector, preview_fabric_locaiton_coverage +from controllers.celery_controller.celery_tasks import process_data, deleteFiles, toggle_tiles, run_signalserver, raster2vector, preview_fabric_locaiton_coverage, async_folder_copy_for_import, add_files_to_folder from utils.namingschemes import DATETIME_FORMAT, EXPORT_CSV_NAME_TEMPLATE, SIGNALSERVER_RASTER_DATA_NAME_TEMPLATE from controllers.signalserver_controller.signalserver_command_builder import runsig_command_builder from controllers.database_controller.tower_ops import create_tower, get_tower_with_towername @@ -63,7 +64,7 @@ def get_number_records(folderid): return jsonify({'error': 'Folder ID is invalid'}), 400 else: folderVal = folder_ops.get_folder_with_id(userid=identity['id'], folderid=folderid, session=session) - return jsonify(kml_ops.get_kml_data(userid=identity['id'], folderid=folderVal.id, session=session)), 200 + return jsonify(kml_ops.get_kml_data(folderid=folderVal.id, session=session)), 200 except NoAuthorizationError: return jsonify({'error': 'Token is invalid or expired'}), 401 @@ -85,7 +86,10 @@ def submit_data(folderid): file_data_list = request.form.getlist('fileData') userVal = user_ops.get_user_with_id(identity['id'], session=session) - + import_folder_id = int(request.form.get('importFolder')) + logger.debug(f"import folder id is {import_folder_id}") + # Prepare data for the task + file_contents = [(f.filename, base64.b64encode(f.read()).decode('utf-8'), data) for f, data in zip(files, file_data_list)] if folderid == -1: deadline = request.form.get('deadline') if not deadline: @@ -96,47 +100,50 @@ def submit_data(folderid): deadline_date = datetime.strptime(deadline, '%Y-%m-%d').date() except ValueError: return jsonify({'Status': "Failed, invalid deadline format"}), 400 - folder_name = f"Filing for Deadline {deadline_date}" - folderVal = folder_ops.create_folder(folder_name, userVal.id, deadline_date, 'upload', session=session) - session.commit() + + if import_folder_id != -1: + # Asynchronous copy and create new folder with deadline + logger.debug("here 1") + task_chain = chain( + async_folder_copy_for_import.s(userVal.id, import_folder_id, deadline), + add_files_to_folder.s(file_contents=file_contents), + process_data.s(recompute=True) + ) + + else: + # Create new folder with deadline + logger.debug("here 2") + new_folder_name = f"Filing for Deadline {deadline_date}" + folderVal = folder_ops.create_folder(new_folder_name, userVal.id, deadline_date, 'upload', session) + session.commit() + + task_chain = chain( + add_files_to_folder.s(folderVal.id, file_contents), + process_data.s(recompute=False) + ) + else: + logger.debug("here 3") folderVal = folder_ops.get_folder_with_id(userid=userVal.id, folderid=folderid, session=session) - file_names = [] - matching_file_data_list = [] - - for index, f in enumerate(files): - existing_file = file_ops.get_file_with_name(f.filename, folderVal.id, session=session) - if existing_file is not None: - # If file already exists, append its name to file_names and skip to next file - continue - - - data = f.read() - if (f.filename.endswith('.csv')): - file_ops.create_file(f.filename, data, folderVal.id, 'fabric', session=session) - file_names.append(f.filename) - matching_file_data_list.append(file_data_list[index]) - elif (f.filename.endswith('.kml') or f.filename.endswith('.geojson')): - file_ops.create_file(f.filename, data, folderVal.id,None, session=session) - file_names.append(f.filename) - matching_file_data_list.append(file_data_list[index]) - - - - session.commit() + task_chain = chain( + add_files_to_folder.s(folderVal.id, file_contents), + process_data.s(recompute=False) + ) - task = process_data.apply_async(args=[file_names, matching_file_data_list, userVal.id, folderVal.id]) # store the AsyncResult instance - return jsonify({'Status': "OK", 'task_id': task.id}), 200 # return task id to the client + result = task_chain.apply_async() + return jsonify({'Status': "OK", 'task_id': result.id}), 200 # return task id to the client except NoAuthorizationError: return jsonify({'error': 'Token is invalid or expired'}), 401 except ValueError as ve: # Handle specific errors, e.g., value errors + logger.debug(ve) return jsonify({'Status': "Failed due to bad value", 'error': str(ve)}), 400 except Exception as e: # General catch-all for other exceptions + logger.debug(e) session.rollback() # Rollback the session in case of error return jsonify({'Status': "Failed, server failed", 'error': str(e)}), 500 finally: diff --git a/front-end/components/Editmap.js b/front-end/components/Editmap.js index 79a5c7a..53a31e1 100755 --- a/front-end/components/Editmap.js +++ b/front-end/components/Editmap.js @@ -267,6 +267,7 @@ function Editmap() { map.current.on("draw.create", (event) => { const polygon = event.features[0]; + console.log(polygon); // Convert drawn polygon to turf polygon const turfPolygon = turf.polygon(polygon.geometry.coordinates); diff --git a/front-end/components/Upload.js b/front-end/components/Upload.js index 168353a..45f9a5b 100755 --- a/front-end/components/Upload.js +++ b/front-end/components/Upload.js @@ -10,7 +10,7 @@ import Tooltip from "@mui/material/Tooltip"; import Input from "@mui/material/Input"; import ExportButton from "./SubmitButton"; import { DataGrid } from "@mui/x-data-grid"; -import { FormControl, InputLabel, MenuItem, Select, Button, Dialog, DialogActions, DialogTitle, Box } from '@mui/material'; +import { FormControl, InputLabel, MenuItem, Select, Button, Dialog, DialogActions, DialogTitle, DialogContent, Box } from '@mui/material'; import Grid from "@mui/material/Grid"; import LoadingEffect from "./LoadingEffect"; import Swal from "sweetalert2"; @@ -118,7 +118,8 @@ export default function Upload() { const [folders, setFolders] = React.useState([]); const [newDeadline, setNewDeadline] = React.useState({ month: '', year: '' }); - const [openConfirmDialog, setOpenConfirmDialog] = React.useState(false); + + const [importFolderID, setImportFolderID] = React.useState(-1); const months = Array.from({ length: 12 }, (_, i) => i + 1); // Months 1-12 const currentYear = new Date().getFullYear(); @@ -160,6 +161,7 @@ export default function Upload() { const formData = new FormData(); formData.append("deadline", `${newDeadline.year}-${newDeadline.month}-03`); + formData.append("importFolder", importFolderID); files.forEach((fileDetails) => { const allowedExtensions = ["kml", "geojson", "csv"]; const fileExtension = fileDetails.file.name @@ -311,32 +313,31 @@ export default function Upload() { } else { const selectedFolder = folders.find(folder => folder.folder_id === newFolderID); - if (selectedFolder) { - // Parse the deadline date of the selected folder - const date = new Date(selectedFolder.deadline); - const month = date.getMonth() + 1; // getMonth() is zero-indexed, so add 1 - const year = date.getFullYear(); - // Set newDeadline to reflect the selected folder's deadline - setNewDeadline({ month: month.toString(), year: year.toString() }); - } + if (selectedFolder) { + // Parse the deadline date of the selected folder + const date = new Date(selectedFolder.deadline); + const month = date.getMonth() + 1; // getMonth() is zero-indexed, so add 1 + const year = date.getFullYear(); + // Set newDeadline to reflect the selected folder's deadline + setNewDeadline({ month: month.toString(), year: year.toString() }); + } } setFolderID(newFolderID); }; + const handleImportChange = (event) => { + const selectedFolderID = event.target.value; + setImportFolderID(selectedFolderID); + + }; const handleNewDeadlineChange = ({ month, year }) => { - - + setNewDeadline({ month, year }); - - }; - const confirmCreation = () => { - // Logic to add the new folder - // setFolders([...folders, { folder_id: folders.length, deadline: `${newDeadline.month}/${newDeadline.year}` }]); - setOpenConfirmDialog(false); }; + const columns = [ { field: "id", headerName: "ID" }, { @@ -395,6 +396,7 @@ export default function Upload() { ]; + return ( @@ -417,56 +419,74 @@ export default function Upload() { }} >
- - You are working on filing for deadline: - - - {folderID === -1 && ( - <> - - Month - - - - Year + + You are working on filing for deadline: - - )} -
+ {folderID === -1 && ( + <> + + Month + + + + Year + + + + Import Data from Previous Filing + + + + )} + { - Your Previous Reports + Your Previous Exported Filings