From 4d43cf171ae044d52d96f86ee2babd901a7421c9 Mon Sep 17 00:00:00 2001 From: William Flores Date: Sun, 2 Jun 2024 12:56:56 -0700 Subject: [PATCH 01/16] feat: support login workflow with ecoinvent_interface --- .../ui/wizards/db_import_wizard.py | 73 +++++++++++++++---- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 2ac185050..00c5605c5 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -5,7 +5,7 @@ import zipfile from pathlib import Path -import eidl +import ecoinvent_interface import requests from bw2io import BW2Package, SingleOutputEcospold2Importer from bw2io.extractors import Ecospold2DataExtractor @@ -344,7 +344,10 @@ def __init__(self, parent=None): self.setLayout(layout) def initializePage(self): - self.stored_dbs = eidl.eidlstorage.stored_dbs + # TODO: get this from eco_invent + # previous stored_dbs was list just listing out all the database + # locally available + self.stored_dbs = ecoinvent_interface.eidlstorage.stored_dbs self.stored_combobox.clear() self.stored_combobox.addItems(sorted(self.stored_dbs.keys())) @@ -1039,7 +1042,7 @@ def nextId(self): class LoginThread(QtCore.QThread): - def __init__(self, downloader, parent=None): + def __init__(self, downloader: "ABEcoinventDownloader", parent=None): super().__init__(parent) self.downloader = downloader @@ -1048,7 +1051,12 @@ def update(self, username: str, password: str) -> None: self.downloader.password = password def run(self): - self.downloader.login() + try: + self.downloader.login() + except Exception as e: + log.error(str(e), exc_info=True) + import_signals.login_success.emit(False) + import_signals.connection_problem.emit(("Unexpected error", str(e))) class EcoinventVersionPage(QtWidgets.QWizardPage): @@ -1316,14 +1324,50 @@ class ImportSignals(QtCore.QObject): import_signals = ImportSignals() -class ABEcoinventDownloader(eidl.EcoinventDownloader): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) +# TODO: reimplement downloader using ecoinvent_interface +class ABEcoinventDownloader(object): + def __init__(self, version=None, system_model=None): + self.version = version + self.system_model = system_model self.extraction_process = None + self._settings = ecoinvent_interface.Settings() + + @property + def username(self): + return self._settings.username + + @username.setter + def username(self, value): + self._settings.username = value - def login_success(self, success): - import_signals.login_success.emit(success) + @property + def password(self): + return self._settings.password + + @password.setter + def password(self, value): + self._settings.password = value + def login(self): + release = ecoinvent_interface.EcoinventRelease(self._settings) + try: + release.login() + login_success = True + except (requests.ConnectTimeout, requests.ReadTimeout, requests.ConnectionError) as e: + login_success = False + self.handle_connection_timeout() + except requests.exceptions.HTTPError as e: + login_success = False + if e.response.status_code != 401: + log.error( + "Unexpected status code (%d) received when trying to list ecoinvent_versions, response: %s", + e.response.status_code, + e.response.text + ) + + import_signals.login_success.emit(login_success) + + # TODO: don't think we need this anymore def extract(self, target_dir): """Override extract method to redirect the stdout to dev null.""" code = super().extract(target_dir=target_dir, stdout=subprocess.DEVNULL) @@ -1334,9 +1378,8 @@ def extract(self, target_dir): def handle_connection_timeout(self): msg = "The request timed out, please check your internet connection!" - if eidl.eidlstorage.stored_dbs: - msg += ( - "\n\nIf you work offline you can use your previously downloaded databases" - + " via the archive option of the import wizard." - ) - import_signals.connection_problem.emit(("Connection problem", msg)) + # TODO: get this out of ecoinvent cache + # if eidl.eidlstorage.stored_dbs: + # msg += ("\n\nIf you work offline you can use your previously downloaded databases" + + # " via the archive option of the import wizard.") + import_signals.connection_problem.emit(('Connection problem', msg)) From 61fb2fb225f8c0ded429eb0ef070f2c87bfed31d Mon Sep 17 00:00:00 2001 From: William Flores Date: Sun, 2 Jun 2024 13:32:28 -0700 Subject: [PATCH 02/16] feat: update ecoinvent version page --- .../ui/wizards/db_import_wizard.py | 55 +++++++++++-------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 00c5605c5..a749d5e91 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import io +from functools import lru_cache import subprocess import tempfile import zipfile @@ -1062,7 +1063,7 @@ def run(self): class EcoinventVersionPage(QtWidgets.QWizardPage): def __init__(self, parent=None): super().__init__(parent) - self.wizard = self.parent() + self.wizard: "DatabaseImportWizard" = self.parent() self.description_label = QtWidgets.QLabel( "Choose ecoinvent version and system model:" ) @@ -1083,27 +1084,17 @@ def __init__(self, parent=None): self.setLayout(layout) def initializePage(self): - if self.db_dict is None: - self.wizard.downloader.db_dict = ( - self.wizard.downloader.get_available_files() - ) - self.db_dict = self.wizard.downloader.db_dict - self.system_models = { - version: sorted( - {k[1] for k in self.db_dict.keys() if k[0] == version}, reverse=True - ) - for version in sorted( - {k[0] for k in self.db_dict.keys() if k[0] in __ei_versions__}, - reverse=True, - ) - } + available_versions = self.wizard.downloader.list_versions() + shown_versions = set( + [version for version in available_versions if version in __ei_versions__] + ) # Catch for incorrect 'universal' key presence # (introduced in version 3.6 of ecoinvent) - if "universal" in self.system_models: - del self.system_models["universal"] + if "universal" in shown_versions: + shown_versions.remove("universal") self.version_combobox.clear() self.system_model_combobox.clear() - versions = sort_semantic_versions(self.system_models.keys()) + versions = sort_semantic_versions(shown_versions) self.version_combobox.addItems(versions) if bool(self.version_combobox.count()): # Adding the items will cause system_model_combobox to update @@ -1128,7 +1119,9 @@ def update_system_model_combobox(self, version: str) -> None: different ecoinvent version. """ self.system_model_combobox.clear() - self.system_model_combobox.addItems(self.system_models[version]) + items = self.wizard.downloader.list_system_models(version) + items = sorted(items, reverse=True) + self.system_model_combobox.addItems(items) class LocalDatabaseImportPage(QtWidgets.QWizardPage): @@ -1331,6 +1324,10 @@ def __init__(self, version=None, system_model=None): self.system_model = system_model self.extraction_process = None self._settings = ecoinvent_interface.Settings() + self._release = ecoinvent_interface.EcoinventRelease(self._settings) + + def update_ecoinvent_release(self): + self._release = ecoinvent_interface.EcoinventRelease(self._settings) @property def username(self): @@ -1339,6 +1336,7 @@ def username(self): @username.setter def username(self, value): self._settings.username = value + self.update_ecoinvent_release() @property def password(self): @@ -1347,13 +1345,18 @@ def password(self): @password.setter def password(self, value): self._settings.password = value + self.update_ecoinvent_release() def login(self): release = ecoinvent_interface.EcoinventRelease(self._settings) try: release.login() login_success = True - except (requests.ConnectTimeout, requests.ReadTimeout, requests.ConnectionError) as e: + except ( + requests.ConnectTimeout, + requests.ReadTimeout, + requests.ConnectionError, + ) as e: login_success = False self.handle_connection_timeout() except requests.exceptions.HTTPError as e: @@ -1362,11 +1365,19 @@ def login(self): log.error( "Unexpected status code (%d) received when trying to list ecoinvent_versions, response: %s", e.response.status_code, - e.response.text + e.response.text, ) import_signals.login_success.emit(login_success) + @lru_cache(maxsize=1) + def list_versions(self): + return self._release.list_versions() + + @lru_cache(maxsize=100) + def list_system_models(self, version: str): + return self._release.list_system_models(version) + # TODO: don't think we need this anymore def extract(self, target_dir): """Override extract method to redirect the stdout to dev null.""" @@ -1382,4 +1393,4 @@ def handle_connection_timeout(self): # if eidl.eidlstorage.stored_dbs: # msg += ("\n\nIf you work offline you can use your previously downloaded databases" + # " via the archive option of the import wizard.") - import_signals.connection_problem.emit(('Connection problem', msg)) + import_signals.connection_problem.emit(("Connection problem", msg)) From 723a1e6794c7e09643df5fa26022f809c11d6b64 Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 00:54:09 -0700 Subject: [PATCH 03/16] feat: implement downloading and extract file also cleanup the login signal and move them out of the downloader --- .../ui/wizards/db_import_wizard.py | 156 +++++++++++++----- 1 file changed, 119 insertions(+), 37 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index a749d5e91..a9d438a7f 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- import io +import shutil +import typing from functools import lru_cache import subprocess import tempfile @@ -12,6 +14,7 @@ from bw2io.extractors import Ecospold2DataExtractor from PySide2 import QtCore, QtWidgets from PySide2.QtCore import Signal, Slot +from py7zr import py7zr from activity_browser import log from activity_browser.bwutils import errors @@ -94,9 +97,14 @@ def version(self): def system_model(self): return self.ecoinvent_version_page.system_model_combobox.currentText() + @property + def release_type(self): + return self.ecoinvent_version_page.release_type_combobox.currentText() + def update_downloader(self): self.downloader.version = self.version self.downloader.system_model = self.system_model + self.downloader.release_type = self.release_type def done(self, result: int): """ @@ -744,7 +752,7 @@ def report_failed_unarchive(self, file: str) -> None: class MainWorkerThread(ABThread): def __init__(self, downloader, parent=None): super().__init__(parent) - self.downloader = downloader + self.downloader: "ABEcoinventDownloader" = downloader self.forwast_url = ( "https://lca-net.com/wp-content/uploads/forwast.bw2package.zip" ) @@ -788,16 +796,12 @@ def run_safely(self): def run_ecoinvent(self) -> None: """Run the ecoinvent downloader from start to finish.""" - self.downloader.outdir = eidl.eidlstorage.eidl_dir - if self.downloader.check_stored(): - import_signals.download_complete.emit() - else: - self.run_download() + archive_file = self.run_download() with tempfile.TemporaryDirectory() as tempdir: temp_dir = Path(tempdir) if not import_signals.cancel_sentinel: - self.run_extract(temp_dir) + self.run_extract(archive_file, temp_dir) if not import_signals.cancel_sentinel: dataset_dir = temp_dir.joinpath("datasets") self.run_import(dataset_dir) @@ -827,17 +831,23 @@ def run_forwast(self) -> None: else: self.delete_canceled_db() - def run_download(self) -> None: + def run_download(self) -> Path: """Use the connected ecoinvent downloader.""" - self.downloader.download() + filepath = self.downloader.download() import_signals.download_complete.emit() + return filepath - def run_extract(self, temp_dir: Path) -> None: + def run_extract(self, archive_file: Path, temp_dir: Path) -> None: """Use the connected ecoinvent downloader to extract the downloaded 7zip file. """ - self.downloader.extract(target_dir=temp_dir) - import_signals.unarchive_finished.emit() + try: + self.downloader.extract(archive_file, temp_dir) + except Exception: + import_signals.cancel_sentinel = True + import_signals.unarchive_failed.emit(temp_dir) + else: + import_signals.unarchive_finished.emit() def run_extract_import(self) -> None: """Combine the extract and import steps when beginning from a selected @@ -974,11 +984,20 @@ def __init__(self, parent=None): super().__init__(parent) self.wizard = parent self.complete = False + eco_settings = ecoinvent_interface.Settings() self.username_edit = QtWidgets.QLineEdit() - self.username_edit.setPlaceholderText("ecoinvent username") + if eco_settings.username: + self.username_edit.setText(eco_settings.username) + else: + self.username_edit.setPlaceholderText("ecoinvent username") self.password_edit = QtWidgets.QLineEdit() - self.password_edit.setPlaceholderText("ecoinvent password"), self.password_edit.setEchoMode(QtWidgets.QLineEdit.Password) + if eco_settings.password: + self.password_edit.setText(eco_settings.password) + else: + self.password_edit.setPlaceholderText("ecoinvent password") + self.save_creds = QtWidgets.QPushButton("Save Credentials") + self.save_creds.clicked.connect(self.save_credentials) self.login_button = QtWidgets.QPushButton("login") self.login_button.clicked.connect(self.login) self.password_edit.returnPressed.connect(self.login_button.click) @@ -993,6 +1012,7 @@ def __init__(self, parent=None): box_layout.addWidget(self.password_edit) hlay = QtWidgets.QHBoxLayout() hlay.addWidget(self.login_button) + hlay.addWidget(self.save_creds) hlay.addStretch(1) box_layout.addLayout(hlay) box_layout.addWidget(self.success_label) @@ -1022,6 +1042,13 @@ def login(self) -> None: self.login_thread.update(self.username, self.password) self.login_thread.start() + @Slot(name="SaveEiCredentials") + def save_credentials(self): + self.success_label.setText("Saving Credentials") + ecoinvent_interface.permanent_setting("username", self.username) + ecoinvent_interface.permanent_setting("password", self.password) + self.success_label.setText("Saved Credentials") + @Slot(bool, name="handleLoginResponse") def login_response(self, success: bool) -> None: if not success: @@ -1052,12 +1079,25 @@ def update(self, username: str, password: str) -> None: self.downloader.password = password def run(self): + error_message = None try: - self.downloader.login() + login_success, error_message = self.downloader.login() except Exception as e: log.error(str(e), exc_info=True) import_signals.login_success.emit(False) - import_signals.connection_problem.emit(("Unexpected error", str(e))) + msg = str(e) + cs = ecoinvent_interface.CachedStorage() + if len(cs.catalogue) > 0: + msg += ( + "\n\nIf you work offline you can use your previously downloaded databases" + + " via the archive option of the import wizard." + ) + import_signals.connection_problem.emit(("Unexpected error", msg)) + else: + import_signals.login_success.emit(login_success) + finally: + if error_message: + import_signals.connection_problem.emit(error_message) class EcoinventVersionPage(QtWidgets.QWizardPage): @@ -1074,6 +1114,10 @@ def __init__(self, parent=None): self.update_system_model_combobox ) self.system_model_combobox = QtWidgets.QComboBox() + self.release_type_combobox = QtWidgets.QComboBox() + self.release_type_combobox.addItems( + [x.name for x in list(ecoinvent_interface.ReleaseType)] + ) layout = QtWidgets.QGridLayout() layout.addWidget(self.description_label, 0, 0, 1, 3) @@ -1081,6 +1125,8 @@ def __init__(self, parent=None): layout.addWidget(self.version_combobox, 1, 1, 1, 2) layout.addWidget(QtWidgets.QLabel("System model: "), 2, 0) layout.addWidget(self.system_model_combobox, 2, 1, 1, 2) + layout.addWidget(QtWidgets.QLabel("Release Type: "), 3, 0) + layout.addWidget(self.release_type_combobox, 3, 1, 1, 2) self.setLayout(layout) def initializePage(self): @@ -1319,10 +1365,15 @@ class ImportSignals(QtCore.QObject): # TODO: reimplement downloader using ecoinvent_interface class ABEcoinventDownloader(object): - def __init__(self, version=None, system_model=None): + def __init__( + self, + version=None, + system_model=None, + release_type: typing.Optional[ecoinvent_interface.ReleaseType] = None, + ): self.version = version self.system_model = system_model - self.extraction_process = None + self._release_type = release_type self._settings = ecoinvent_interface.Settings() self._release = ecoinvent_interface.EcoinventRelease(self._settings) @@ -1347,8 +1398,25 @@ def password(self, value): self._settings.password = value self.update_ecoinvent_release() - def login(self): + @property + def release_type(self): + return self._release_type + + @release_type.setter + def release_type(self, value: typing.Union[str, ecoinvent_interface.ReleaseType]): + if isinstance(value, ecoinvent_interface.ReleaseType): + self._release_type = value + return + + if isinstance(value, str): + self._release_type = ecoinvent_interface.ReleaseType[value] + return + + raise ValueError("invalid value provided for release_type") + + def login(self) -> (bool, typing.Optional[typing.Tuple[str, str]]): release = ecoinvent_interface.EcoinventRelease(self._settings) + error_message = None try: release.login() login_success = True @@ -1358,17 +1426,26 @@ def login(self): requests.ConnectionError, ) as e: login_success = False - self.handle_connection_timeout() + error_message = ( + "Connection Problem", + "The request timed out, please check your internet connection!", + ) except requests.exceptions.HTTPError as e: login_success = False + error_message = None if e.response.status_code != 401: log.error( "Unexpected status code (%d) received when trying to list ecoinvent_versions, response: %s", e.response.status_code, e.response.text, ) + error_message = ( + "Unexpected Problem", + "An unexpected error occurred, please try again status code %d" + % e.response.status_code, + ) - import_signals.login_success.emit(login_success) + return login_success, error_message @lru_cache(maxsize=1) def list_versions(self): @@ -1378,19 +1455,24 @@ def list_versions(self): def list_system_models(self, version: str): return self._release.list_system_models(version) - # TODO: don't think we need this anymore - def extract(self, target_dir): - """Override extract method to redirect the stdout to dev null.""" - code = super().extract(target_dir=target_dir, stdout=subprocess.DEVNULL) - if code != 0: - # The archive was corrupted in some way. - import_signals.cancel_sentinel = True - import_signals.unarchive_failed.emit(self.out_path) - - def handle_connection_timeout(self): - msg = "The request timed out, please check your internet connection!" - # TODO: get this out of ecoinvent cache - # if eidl.eidlstorage.stored_dbs: - # msg += ("\n\nIf you work offline you can use your previously downloaded databases" + - # " via the archive option of the import wizard.") - import_signals.connection_problem.emit(("Connection problem", msg)) + def download(self) -> Path: + return self._release.get_release( + version=self.version, + system_model=self.system_model, + release_type=self.release_type, + extract=False, + ) + + @staticmethod + def extract(filepath: Path, out_dir: Path = None): + """ + Extract archive + """ + if filepath.suffix.lower() == ".7z": + with py7zr.SevenZipFile(filepath, "r") as archive: + directory = out_dir or (filepath.parent / filepath.stem) + if directory.exists(): + shutil.rmtree(directory) + archive.extractall(path=directory) + else: + raise ValueError("Unsupported archive format") From db7135224b7c5a799d03bcbd29d571e0600dbeed Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 01:06:41 -0700 Subject: [PATCH 04/16] fix: import errors --- activity_browser/ui/wizards/db_import_wizard.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index a9d438a7f..7cf7eefac 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -8,6 +8,7 @@ import zipfile from pathlib import Path +import bw2data.errors import ecoinvent_interface import requests from bw2io import BW2Package, SingleOutputEcospold2Importer @@ -880,14 +881,14 @@ def run_import(self, import_dir: Path) -> None: signal=import_signals.strategy_progress, ) importer.apply_strategies() - importer.write_database(backend="activitybrowser") + importer.write_database(backend="sqlite") if not import_signals.cancel_sentinel: import_signals.finished.emit() else: self.delete_canceled_db() except errors.ImportCanceledError: self.delete_canceled_db() - except errors.InvalidExchange: + except bw2data.errors.InvalidExchange: # Likely caused by new version of ecoinvent not finding required # biosphere flows. self.delete_canceled_db() From 6796052a83eed466d602d356a073257e69826161 Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 01:07:08 -0700 Subject: [PATCH 05/16] fix: always download ecospold file --- activity_browser/ui/wizards/db_import_wizard.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 7cf7eefac..03046630d 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -100,7 +100,7 @@ def system_model(self): @property def release_type(self): - return self.ecoinvent_version_page.release_type_combobox.currentText() + return ecoinvent_interface.ReleaseType.ecospold def update_downloader(self): self.downloader.version = self.version @@ -1115,10 +1115,6 @@ def __init__(self, parent=None): self.update_system_model_combobox ) self.system_model_combobox = QtWidgets.QComboBox() - self.release_type_combobox = QtWidgets.QComboBox() - self.release_type_combobox.addItems( - [x.name for x in list(ecoinvent_interface.ReleaseType)] - ) layout = QtWidgets.QGridLayout() layout.addWidget(self.description_label, 0, 0, 1, 3) @@ -1126,8 +1122,6 @@ def __init__(self, parent=None): layout.addWidget(self.version_combobox, 1, 1, 1, 2) layout.addWidget(QtWidgets.QLabel("System model: "), 2, 0) layout.addWidget(self.system_model_combobox, 2, 1, 1, 2) - layout.addWidget(QtWidgets.QLabel("Release Type: "), 3, 0) - layout.addWidget(self.release_type_combobox, 3, 1, 1, 2) self.setLayout(layout) def initializePage(self): From efb1c94d26bea7ac51bdd4e5f0e4d3fc31d87a16 Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 02:04:01 -0700 Subject: [PATCH 06/16] fix: activity browser sqlite3 superclass --- .../ui/wizards/db_import_wizard.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 03046630d..2074c6985 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -11,6 +11,7 @@ import bw2data.errors import ecoinvent_interface import requests +from bw2data.subclass_mapping import DATABASE_BACKEND_MAPPING from bw2io import BW2Package, SingleOutputEcospold2Importer from bw2io.extractors import Ecospold2DataExtractor from PySide2 import QtCore, QtWidgets @@ -28,6 +29,7 @@ from ..threading import ABThread from ..widgets import DatabaseLinkingDialog + # TODO: Rework the entire import wizard, the amount of different classes # and interwoven connections makes the entire thing nearly incomprehensible. @@ -881,7 +883,8 @@ def run_import(self, import_dir: Path) -> None: signal=import_signals.strategy_progress, ) importer.apply_strategies() - importer.write_database(backend="sqlite") + # backend is a custom implementation that wraps sqlite database + importer.write_database(backend="activitybrowser") if not import_signals.cancel_sentinel: import_signals.finished.emit() else: @@ -1318,22 +1321,28 @@ def extract(cls, dirpath: str, db_name: str, *args, **kwargs): class ActivityBrowserBackend(bd.backends.SQLiteBackend): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self._ab_current_index = 0 + self._ab_total = 0 def _efficient_write_many_data(self, *args, **kwargs): data = args[0] - self.total = len(data) + self._ab_total = len(data) super()._efficient_write_many_data(*args, **kwargs) def _efficient_write_dataset(self, *args, **kwargs): - index = args[0] if import_signals.cancel_sentinel: - log.info(f"\nWriting canceled at position {index}!") + log.info(f"\nWriting canceled at position {self._ab_current_index}!") raise errors.ImportCanceledError - import_signals.db_progress.emit(index + 1, self.total) + self._ab_current_index += 1 + import_signals.db_progress.emit(self._ab_current_index, self._ab_total) return super()._efficient_write_dataset(*args, **kwargs) bd.config.backends["activitybrowser"] = ActivityBrowserBackend +# config is no longer enough to provide an additional backend +# database chooser, specifically looks at DATABASE_BACKEND_MAPPING +# to get the class implementation +DATABASE_BACKEND_MAPPING.update({"activitybrowser": ActivityBrowserBackend}) class ImportSignals(QtCore.QObject): From 41cb2c9cc3ea2da90c19b3edc68a1dffed4d154f Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 02:32:01 -0700 Subject: [PATCH 07/16] feat: support listing cached downloads --- activity_browser/ui/wizards/db_import_wizard.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 2074c6985..30f54507c 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -356,16 +356,13 @@ def __init__(self, parent=None): self.setLayout(layout) def initializePage(self): - # TODO: get this from eco_invent - # previous stored_dbs was list just listing out all the database - # locally available - self.stored_dbs = ecoinvent_interface.eidlstorage.stored_dbs + self.stored_dbs = ecoinvent_interface.CachedStorage() self.stored_combobox.clear() - self.stored_combobox.addItems(sorted(self.stored_dbs.keys())) + self.stored_combobox.addItems(sorted(self.stored_dbs.catalogue.keys())) @Slot(int, name="updateSelectedIndex") def update_stored(self, index: int) -> None: - self.path_edit.setText(self.stored_dbs[self.stored_combobox.currentText()]) + self.path_edit.setText(self.stored_dbs.catalogue[self.stored_combobox.currentText()]["path"]) @Slot(name="getArchiveFile") def get_archive(self) -> None: @@ -1367,7 +1364,6 @@ class ImportSignals(QtCore.QObject): import_signals = ImportSignals() -# TODO: reimplement downloader using ecoinvent_interface class ABEcoinventDownloader(object): def __init__( self, From f12ec3b5d2f8dbbbf77d1c427bc11dfceedc2e58 Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 23:32:07 -0700 Subject: [PATCH 08/16] chore: update typings --- .../ui/wizards/db_import_wizard.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 30f54507c..205db15f2 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -30,10 +30,6 @@ from ..widgets import DatabaseLinkingDialog -# TODO: Rework the entire import wizard, the amount of different classes -# and interwoven connections makes the entire thing nearly incomprehensible. - - class DatabaseImportWizard(QtWidgets.QWizard): IMPORT_TYPE = 1 REMOTE_TYPE = 2 @@ -750,9 +746,9 @@ def report_failed_unarchive(self, file: str) -> None: class MainWorkerThread(ABThread): - def __init__(self, downloader, parent=None): + def __init__(self, downloader: "ABEcoinventDownloader", parent=None): super().__init__(parent) - self.downloader: "ABEcoinventDownloader" = downloader + self.downloader = downloader self.forwast_url = ( "https://lca-net.com/wp-content/uploads/forwast.bw2package.zip" ) @@ -1367,8 +1363,8 @@ class ImportSignals(QtCore.QObject): class ABEcoinventDownloader(object): def __init__( self, - version=None, - system_model=None, + version: typing.Optional[str] = None, + system_model: typing.Optional[str] = None, release_type: typing.Optional[ecoinvent_interface.ReleaseType] = None, ): self.version = version @@ -1381,20 +1377,20 @@ def update_ecoinvent_release(self): self._release = ecoinvent_interface.EcoinventRelease(self._settings) @property - def username(self): + def username(self) -> typing.Optional[str]: return self._settings.username @username.setter - def username(self, value): + def username(self, value: str): self._settings.username = value self.update_ecoinvent_release() @property - def password(self): + def password(self) -> typing.Optional[str]: return self._settings.password @password.setter - def password(self, value): + def password(self, value: str): self._settings.password = value self.update_ecoinvent_release() From 429f7406c3a6a8fcfd9e063105244fc9b30c327d Mon Sep 17 00:00:00 2001 From: William Flores Date: Thu, 6 Jun 2024 23:35:45 -0700 Subject: [PATCH 09/16] chore: remove explict inheritance for downloader --- activity_browser/ui/wizards/db_import_wizard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 205db15f2..01ffdccd2 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -1360,7 +1360,7 @@ class ImportSignals(QtCore.QObject): import_signals = ImportSignals() -class ABEcoinventDownloader(object): +class ABEcoinventDownloader: def __init__( self, version: typing.Optional[str] = None, From 4eaec6dfef1baac51a73852984152bcccaa295ea Mon Sep 17 00:00:00 2001 From: William Flores Date: Fri, 7 Jun 2024 00:53:30 -0700 Subject: [PATCH 10/16] feat: skip login page if user has valid credentials --- .../ui/wizards/db_import_wizard.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 01ffdccd2..f5bccec21 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -165,6 +165,12 @@ def cleanup(self): self.import_page.complete = False self.reject() + def has_existing_remote_credentials(self) -> bool: + return ( + self.downloader.username is not None + and self.downloader.password is not None + ) + @Slot(tuple, name="showMessage") def show_info(self, info: tuple) -> None: title, message = info @@ -222,6 +228,7 @@ def __init__(self, parent=None): self.wizard = parent self.radio_buttons = [QtWidgets.QRadioButton(o[0]) for o in self.OPTIONS] self.radio_buttons[0].setChecked(True) + self.has_valid_remote_creds = False layout = QtWidgets.QVBoxLayout() box = QtWidgets.QGroupBox("Data source:") @@ -233,10 +240,21 @@ def __init__(self, parent=None): layout.addWidget(box) self.setLayout(layout) + def validatePage(self): + if ( + self.wizard.has_existing_remote_credentials() + and self.radio_buttons[0].isChecked() + ): + self.has_valid_remote_creds, _ = self.wizard.downloader.login() + return True + def nextId(self): option_id = [b.isChecked() for b in self.radio_buttons].index(True) self.wizard.import_type = self.OPTIONS[option_id][1] - return self.OPTIONS[option_id][2] + next_id = self.OPTIONS[option_id][2] + if next_id == DatabaseImportWizard.EI_LOGIN and self.has_valid_remote_creds: + return DatabaseImportWizard.EI_VERSION + return next_id class LocalImportPage(QtWidgets.QWizardPage): From 3245bca52fbda31a7b5427ee9e03e9685d8f033d Mon Sep 17 00:00:00 2001 From: William Flores Date: Fri, 7 Jun 2024 00:55:36 -0700 Subject: [PATCH 11/16] fix: handle case when version is empty going back and forth between the wizard pages can set this to an empty string and cause an exception --- activity_browser/ui/wizards/db_import_wizard.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index f5bccec21..ac81cd2c3 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -376,7 +376,9 @@ def initializePage(self): @Slot(int, name="updateSelectedIndex") def update_stored(self, index: int) -> None: - self.path_edit.setText(self.stored_dbs.catalogue[self.stored_combobox.currentText()]["path"]) + self.path_edit.setText( + self.stored_dbs.catalogue[self.stored_combobox.currentText()]["path"] + ) @Slot(name="getArchiveFile") def get_archive(self) -> None: @@ -1467,6 +1469,8 @@ def list_versions(self): @lru_cache(maxsize=100) def list_system_models(self, version: str): + if version == "": + return [] return self._release.list_system_models(version) def download(self) -> Path: From c9c93ecbc209c93fa7127367a37af13751725de4 Mon Sep 17 00:00:00 2001 From: William Flores Date: Fri, 7 Jun 2024 01:09:46 -0700 Subject: [PATCH 12/16] fix: filter catalog items for ecoSpold02 files only --- activity_browser/ui/wizards/db_import_wizard.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index ac81cd2c3..40803ca0a 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -372,7 +372,17 @@ def __init__(self, parent=None): def initializePage(self): self.stored_dbs = ecoinvent_interface.CachedStorage() self.stored_combobox.clear() - self.stored_combobox.addItems(sorted(self.stored_dbs.catalogue.keys())) + self.stored_combobox.addItems( + sorted( + [ + key + for key, value in self.stored_dbs.catalogue.items() + if value["extracted"] == False + and value["kind"] == "release" + and key.partition(value["system_model"])[2] == "_ecoSpold02.7z" + ] + ) + ) @Slot(int, name="updateSelectedIndex") def update_stored(self, index: int) -> None: From 65ac1dc71cb3e6147f39747a01947caa5e2c2f7d Mon Sep 17 00:00:00 2001 From: William Flores Date: Fri, 7 Jun 2024 01:21:06 -0700 Subject: [PATCH 13/16] feat: update biosphere_database preference on import --- activity_browser/ui/wizards/db_import_wizard.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 40803ca0a..954347061 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -1177,6 +1177,12 @@ def initializePage(self): ) self.wizard.back() + def validatePage(self): + version = self.version_combobox.currentText() + bd.preferences["biosphere_database"] = "ecoinvent-{}-biosphere".format(version) + bd.preferences.flush() + return True + def nextId(self): return DatabaseImportWizard.DB_NAME From bd5c3559bbb07a3acfa9637935f5548d74521ecb Mon Sep 17 00:00:00 2001 From: William Flores Date: Sat, 8 Jun 2024 21:10:48 -0700 Subject: [PATCH 14/16] fix: handle case when no ecoinvent credentials are pre-configured --- activity_browser/ui/wizards/db_import_wizard.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index 954347061..e1d025ec7 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -1407,10 +1407,19 @@ def __init__( self.system_model = system_model self._release_type = release_type self._settings = ecoinvent_interface.Settings() - self._release = ecoinvent_interface.EcoinventRelease(self._settings) + self.update_ecoinvent_release() def update_ecoinvent_release(self): - self._release = ecoinvent_interface.EcoinventRelease(self._settings) + try: + self._release = ecoinvent_interface.EcoinventRelease(self._settings) + except ValueError: + self._release = None + + @property + def release(self) -> ecoinvent_interface.EcoinventRelease: + if self._release is None: + raise ValueError("ecoinvent release has not been initialized properly") + return self._release @property def username(self) -> typing.Optional[str]: @@ -1490,7 +1499,7 @@ def list_system_models(self, version: str): return self._release.list_system_models(version) def download(self) -> Path: - return self._release.get_release( + return self.release.get_release( version=self.version, system_model=self.system_model, release_type=self.release_type, From e6fa005a2c13d0c534feb3a0aaf1fa91c482be79 Mon Sep 17 00:00:00 2001 From: William Flores Date: Sat, 8 Jun 2024 21:51:37 -0700 Subject: [PATCH 15/16] refactor: use extract of ecoinvent still have a fallback if the provided file is 7z --- .../ui/wizards/db_import_wizard.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index e1d025ec7..e1fa92d2c 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import io +import os.path import shutil import typing from functools import lru_cache @@ -824,13 +825,17 @@ def run_ecoinvent(self) -> None: """Run the ecoinvent downloader from start to finish.""" archive_file = self.run_download() - with tempfile.TemporaryDirectory() as tempdir: - temp_dir = Path(tempdir) - if not import_signals.cancel_sentinel: - self.run_extract(archive_file, temp_dir) - if not import_signals.cancel_sentinel: - dataset_dir = temp_dir.joinpath("datasets") - self.run_import(dataset_dir) + if os.path.isdir(archive_file): + import_signals.unarchive_finished.emit() + self.run_import(archive_file.joinpath("datasets")) + else: + with tempfile.TemporaryDirectory() as tempdir: + temp_dir = Path(tempdir) + if not import_signals.cancel_sentinel: + self.run_extract(archive_file, temp_dir) + if not import_signals.cancel_sentinel: + dataset_dir = temp_dir.joinpath("datasets") + self.run_import(dataset_dir) def run_forwast(self) -> None: """Adapted from pjamesjoyce/lcopt.""" @@ -1503,7 +1508,7 @@ def download(self) -> Path: version=self.version, system_model=self.system_model, release_type=self.release_type, - extract=False, + extract=True, ) @staticmethod From a7d87e16af4e2e85e73941ecb8ff38ab9c42e8dd Mon Sep 17 00:00:00 2001 From: William Flores Date: Sat, 8 Jun 2024 23:37:19 -0700 Subject: [PATCH 16/16] feat: add page to create versioned biosphere database --- .../ui/wizards/db_import_wizard.py | 133 ++++++++++++++++-- 1 file changed, 122 insertions(+), 11 deletions(-) diff --git a/activity_browser/ui/wizards/db_import_wizard.py b/activity_browser/ui/wizards/db_import_wizard.py index e1fa92d2c..78ea9510a 100644 --- a/activity_browser/ui/wizards/db_import_wizard.py +++ b/activity_browser/ui/wizards/db_import_wizard.py @@ -17,11 +17,16 @@ from bw2io.extractors import Ecospold2DataExtractor from PySide2 import QtCore, QtWidgets from PySide2.QtCore import Signal, Slot +from bw2io.importers import Ecospold2BiosphereImporter from py7zr import py7zr -from activity_browser import log +from activity_browser import log, project_settings from activity_browser.bwutils import errors from activity_browser.mod import bw2data as bd +from activity_browser.mod.bw2data import databases +from activity_browser.bwutils.ecoinvent_biosphere_versions.ecospold2biosphereimporter import ( + ABEcospold2BiosphereImporter, +) from ...bwutils.importers import ABExcelImporter, ABPackage from ...info import __ei_versions__ @@ -29,6 +34,7 @@ from ..style import style_group_box from ..threading import ABThread from ..widgets import DatabaseLinkingDialog +from ..widgets.biosphere_update import UpdateBiosphereThread class DatabaseImportWizard(QtWidgets.QWizard): @@ -37,13 +43,14 @@ class DatabaseImportWizard(QtWidgets.QWizard): LOCAL_TYPE = 3 EI_LOGIN = 4 EI_VERSION = 5 - ARCHIVE = 6 - DIR = 7 - LOCAL = 8 - EXCEL = 9 - DB_NAME = 10 - CONFIRM = 11 - IMPORT = 12 + DB_BIOSPHERE_CREATION = 6 + ARCHIVE = 7 + DIR = 8 + LOCAL = 9 + EXCEL = 10 + DB_NAME = 11 + CONFIRM = 12 + IMPORT = 13 def __init__(self, parent=None): super().__init__(parent) @@ -60,6 +67,7 @@ def __init__(self, parent=None): self.local_page = LocalImportPage(self) self.ecoinvent_login_page = EcoinventLoginPage(self) self.ecoinvent_version_page = EcoinventVersionPage(self) + self.biosphere_database_setup = BiosphereDatabaseSetup(self) self.archive_page = Choose7zArchivePage(self) self.choose_dir_page = ChooseDirPage(self) self.local_import_page = LocalDatabaseImportPage(self) @@ -72,6 +80,7 @@ def __init__(self, parent=None): self.setPage(self.LOCAL_TYPE, self.local_page) self.setPage(self.EI_LOGIN, self.ecoinvent_login_page) self.setPage(self.EI_VERSION, self.ecoinvent_version_page) + self.setPage(self.DB_BIOSPHERE_CREATION, self.biosphere_database_setup) self.setPage(self.ARCHIVE, self.archive_page) self.setPage(self.DIR, self.choose_dir_page) self.setPage(self.LOCAL, self.local_import_page) @@ -515,8 +524,12 @@ def initializePage(self): ) else: self.path_label.setText( - "Ecoinvent version: {}
Ecoinvent system model: {}".format( - self.wizard.version, self.wizard.system_model + "Ecoinvent version: {}
" + "Ecoinvent system model: {}
" + "Dependent Database: {}".format( + self.wizard.version, + self.wizard.system_model, + bd.preferences["biosphere_database"], ) ) @@ -1140,7 +1153,7 @@ def __init__(self, parent=None): "Choose ecoinvent version and system model:" ) self.db_dict = None - self.system_models = {} + self.requires_database_creation = False self.version_combobox = QtWidgets.QComboBox() self.version_combobox.currentTextChanged.connect( self.update_system_model_combobox @@ -1186,9 +1199,13 @@ def validatePage(self): version = self.version_combobox.currentText() bd.preferences["biosphere_database"] = "ecoinvent-{}-biosphere".format(version) bd.preferences.flush() + if bd.preferences["biosphere_database"] not in databases: + self.requires_database_creation = True return True def nextId(self): + if self.requires_database_creation: + return DatabaseImportWizard.DB_BIOSPHERE_CREATION return DatabaseImportWizard.DB_NAME @Slot(str) @@ -1202,6 +1219,100 @@ def update_system_model_combobox(self, version: str) -> None: self.system_model_combobox.addItems(items) +class VersionedBiosphereThread(UpdateBiosphereThread): + update = Signal(int, str) + + def __init__(self, version, parent=None): + # reduce biosphere update list up to the selected version + sorted_versions = sort_semantic_versions( + __ei_versions__, highest_to_lowest=False + ) + ei_versions = sorted_versions[: sorted_versions.index(version) + 1] + super().__init__(ei_versions, parent=parent) + self.version = version + + def run_safely(self): + project = f"{bd.projects.current}" + if bd.preferences["biosphere_database"] not in bd.databases: + self.update.emit( + 0, + "Creating {} database for {}".format( + bd.preferences["biosphere_database"], project + ), + ) + self.create_biosphere3_database() + project_settings.add_db(bd.preferences["biosphere_database"]) + + self.update.emit( + 1, + "Updating biosphere database", + ) + super().run_safely() + + def create_biosphere3_database(self): + if self.version == sort_semantic_versions(__ei_versions__)[0][:3]: + eb = Ecospold2BiosphereImporter( + name=bd.preferences["biosphere_database"], version=self.version + ) + else: + eb = ABEcospold2BiosphereImporter( + name=bd.preferences["biosphere_database"], version=self.version + ) + eb.apply_strategies() + eb.write_database() + + +class BiosphereDatabaseSetup(QtWidgets.QWizardPage): + def __init__(self, parent=None): + super().__init__(parent=parent) + self.wizard: "DatabaseImportWizard" = self.parent() + self.update_label = QtWidgets.QLabel() + self.progressbar = QtWidgets.QProgressBar() + self.progressbar.setRange(0, 2) + self.complete = False + + box = QtWidgets.QGroupBox("Creating biosphere database") + box_layout = QtWidgets.QVBoxLayout() + box_layout.addWidget(self.progressbar) + box_layout.addWidget(self.update_label) + box.setLayout(box_layout) + box.setStyleSheet(style_group_box.border_title) + layout = QtWidgets.QVBoxLayout() + layout.addWidget(box) + self.setLayout(layout) + + def isComplete(self): + return self.complete + + def initializePage(self): + self.biosphere_thread = VersionedBiosphereThread(self.wizard.version, self) + self.biosphere_thread.update.connect(self.update_progress) + self.biosphere_thread.finished.connect(self.thread_finished) + self.biosphere_thread.start() + + def validatePage(self): + return self.biosphere_thread.isFinished() + + @Slot(int, str, name="updateThread") + def update_progress(self, current: int, text: str) -> None: + self.progressbar.setValue(current) + self.update_label.setText(text) + + @Slot(int, name="threadFinished") + def thread_finished(self, result: int = None) -> None: + self.progressbar.setMaximum(1) + self.progressbar.setValue(1) + if result and result != 0: + self.update_label.setText("Something went wrong...") + else: + self.update_label.setText("All Done") + self.complete = True + self.completeChanged.emit() + + def nextId(self): + return DatabaseImportWizard.DB_NAME + + class LocalDatabaseImportPage(QtWidgets.QWizardPage): def __init__(self, parent=None): super().__init__(parent=parent)