From 00807b866ae745184de8efd6b340ea3c949fa734 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Wed, 13 Sep 2023 16:04:30 +0200 Subject: [PATCH 01/11] Initial commit WFH. --- scripts/import-users.py | 164 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 scripts/import-users.py diff --git a/scripts/import-users.py b/scripts/import-users.py new file mode 100644 index 000000000..293ec0a7d --- /dev/null +++ b/scripts/import-users.py @@ -0,0 +1,164 @@ +import os +import base64 +import argparse +import warnings +from pathlib import Path +from getpass import getpass + +import requests +import pandas as pd + + +def main(): + server, data_directory = CommandLineParser().parse() + api = API(server) + processor = ProjectProcessor(api) + + +class ProjectDataParser: + def parse(self, directory): + return_dict = {} + files = os.listdir(directory) + for file in files: + if not file.endswith((".tsv", ".tsv.gz")): + continue + full_path = os.path.join(directory, file) + project_name = self._parse_filename(file) + if project_name == "uncategorized users": + pass + + + @staticmethod + def _parse_filename(filename): + filename = filename.replace("_or_", "_/_") + filename = filename.replace("_", " ") + filename = filename.replace() + return filename + + +class ProjectProcessor: + def __init__(self, api): + self.api = api + + def get_users(self): + self.api.get_users() + + def put_user(self, user_data): + self.api.put_user(user_data) + + def put_project(self, project_data): + self.api.put_project(project_data) + + +class API: + def __init__(self, server): + self.target = server + self.header = self._create_authorazation_header() + + @staticmethod + def _create_authorazation_header(): + token = getpass("Armadillo 3 token:") + if token == "": + warnings.warn("Token not supplied, requiring basic auth password!") + password = getpass("Basic auth password:") + auth_header = "Basic " + base64.b64encode(("admin:" + password).encode('ascii')).decode("UTF-8") + else: + auth_header = "Bearer " + token + return {"Content-Type": "application/json", "Authorization": auth_header} + + def put_user(self, data): + response = requests.put(self.target + "access/users", json=data, headers=self.header) + response.raise_for_status() + + def get_users(self): + response = requests.get(self.target + "access/users", headers=self.header).json() + return response + + def put_project(self, project): + response = requests.put(self.target + "access/projects", json=project, headers=self.header) + response.raise_for_status() + + def get_projects(self): + response = requests.get(self.target + "access/projects", headers=self.header).json() + return response + + +class CommandLineParser: + def __init__(self): + self.parser = CommandLineInterface() + + def parse(self): + server = self._parse_server() + directory = self._parse_data_directory() + return server, directory + + def _parse_server(self): + server_argument = self.parser.get_argument("server") + if not server_argument.startswith("http"): + warnings.warn("CLI supplied argument did not contain scheme, adding https://") + server_argument = "http://" + server_argument + if not server_argument.endswith("/"): + server_argument = server_argument + "/" + return server_argument + + def _parse_data_directory(self): + directory = Path(self.parser.get_argument("user_data")) + if not os.path.isdir(directory): + raise IOError("Given user data argument is not a directory") + contains_tsv = False + files = os.listdir(directory) + for file in files: + if file.endswith((".tsv", ".tsv.gz")): + contains_tsv = True + break + if not contains_tsv: + raise IOError("Given user data directory does not contain the user export TSVs") + return directory + + +class CommandLineInterface: + def __init__(self): + parser = self._create_argument_parser() + self.arguments = parser.parse_args() + + @staticmethod + def _create_argument_parser(): + """ + Creation of the Command Line Arguments, and whenever they should be displayed as + "required" or not. + """ + parser = argparse.ArgumentParser( + prog="Import cohort users", + description="Utilitarian script of the Armadillo migration " + "process to import users and their rights into the correct cohorts." + ) + required = parser.add_argument_group("Required arguments") + required.add_argument( + "-s", + "--server", + type=str, + required=True, + help="The server into which the users and their rights should be imported to." + ) + required.add_argument( + "-d", + "--user-data", + type=str, + required=True, + help="The folder in which all export TSV's are located for each cohort as they are obtained from " + "export-users.py." + ) + return parser + + def get_argument(self, argument_key): + """ + Small getter function for CLI, including some error handling. + """ + if argument_key in self.arguments: + return getattr(self.arguments, argument_key) + else: + raise KeyError(f"Argument {argument_key} invalid, not found in CLI arguments.") + + +if __name__ == "__main__": + main() From bd8dfddc6add9869da551b715ca113f4705dc450 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Thu, 14 Sep 2023 09:43:58 +0200 Subject: [PATCH 02/11] Finalized script. Untested. --- scripts/import-users.py | 87 +++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/scripts/import-users.py b/scripts/import-users.py index 293ec0a7d..c9613deb5 100644 --- a/scripts/import-users.py +++ b/scripts/import-users.py @@ -12,7 +12,9 @@ def main(): server, data_directory = CommandLineParser().parse() api = API(server) + cohorts = ProjectDataParser().parse(data_directory) processor = ProjectProcessor(api) + processor.parse(cohorts) class ProjectDataParser: @@ -25,8 +27,10 @@ def parse(self, directory): full_path = os.path.join(directory, file) project_name = self._parse_filename(file) if project_name == "uncategorized users": - pass - + return_dict["NaN"] = self._load_tsv(full_path) + else: + return_dict[project_name] = self._load_tsv(full_path) + return return_dict @staticmethod def _parse_filename(filename): @@ -35,19 +39,76 @@ def _parse_filename(filename): filename = filename.replace() return filename + @staticmethod + def _load_tsv(path): + try: + data = pd.read_csv( + path, + sep='\t', + dtype={ + "active": bool, + "email": str, + "firstName": str, + "fullName": str, + "lastName": str, + "roles": str + } + ).fillna("") + except pd.errors.EmptyDataError: + warnings.warn(f"Obtaining {path} resulted in empty dataframe!") + data = pd.DataFrame( + columns=["active", "email", "firstName", "fullName", "lastName", "roles"] + ) + return data + class ProjectProcessor: def __init__(self, api): self.api = api + self.su = [] - def get_users(self): - self.api.get_users() - - def put_user(self, user_data): - self.api.put_user(user_data) + def parse(self, cohort_data): + for cohort_name, cohort_users in cohort_data.items(): + if cohort_name == "NaN": + continue + self._obtain_su(cohort_users) + self._put_project(name=cohort_name, users=cohort_users['email'].values.tolist()) + self._process_uncategorized_users(cohort_data['NaN']) + self._add_admin_rights() + + def _put_user(self, email, firstName, lastName): + self.api.put_user({ + "email": email, + "firstName": firstName, + "lastName": lastName, + "admin": False, + }) + + def _put_project(self, name, users): + self.api.put_project({"name": name, "users": users}) + + def _obtain_su(self, project_data): + su = project_data.loc[ + project_data[project_data['roles'].isin(("SU", "ADMIN", "admin"))].index, + "email" + ] + for user in su: + if user not in self.su: + self.su.append(user) + + def _process_uncategorized_users(self, uncategorized_users): + self._obtain_su(uncategorized_users) + uncategorized_users.apply( + lambda x: self._put_user( + email=x['email'], + firstName=x['firstName'], + lastName=x['lastName'] + ) + ) - def put_project(self, project_data): - self.api.put_project(project_data) + def _add_admin_rights(self): + for user in self.su: + self.api.put_user({"email": user, "admin": True}) class API: @@ -70,18 +131,10 @@ def put_user(self, data): response = requests.put(self.target + "access/users", json=data, headers=self.header) response.raise_for_status() - def get_users(self): - response = requests.get(self.target + "access/users", headers=self.header).json() - return response - def put_project(self, project): response = requests.put(self.target + "access/projects", json=project, headers=self.header) response.raise_for_status() - def get_projects(self): - response = requests.get(self.target + "access/projects", headers=self.header).json() - return response - class CommandLineParser: def __init__(self): From 291cf2484cabeb1c3096f8fca2f367ff078499a8 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Thu, 14 Sep 2023 16:12:37 +0200 Subject: [PATCH 03/11] Finalizing script. Documentation WIP. --- scripts/import-users.py | 218 +++++++++++++++++++++++++++++++++++----- 1 file changed, 193 insertions(+), 25 deletions(-) diff --git a/scripts/import-users.py b/scripts/import-users.py index c9613deb5..0f7dd8039 100644 --- a/scripts/import-users.py +++ b/scripts/import-users.py @@ -1,4 +1,5 @@ import os +import re import base64 import argparse import warnings @@ -13,12 +14,25 @@ def main(): server, data_directory = CommandLineParser().parse() api = API(server) cohorts = ProjectDataParser().parse(data_directory) - processor = ProjectProcessor(api) - processor.parse(cohorts) + ProjectProcessor(api).parse(cohorts) class ProjectDataParser: def parse(self, directory): + """ + Method to parse all TSVs in a user supplied directory, assuming all those TSVs are export + TSVs from the export script. Returns a dictionary with a (parsed and sanitized) cohort name + as key and it's user data as value. + + Args: + directory: + Absolute path to the directory containing the user cohort export TSVs. + + Returns: + dict: + Returns a dictionary containing (key) cohort name, parsed to adhere to AWS + bucket standards and (value) cohort user data. + """ return_dict = {} files = os.listdir(directory) for file in files: @@ -26,21 +40,45 @@ def parse(self, directory): continue full_path = os.path.join(directory, file) project_name = self._parse_filename(file) - if project_name == "uncategorized users": - return_dict["NaN"] = self._load_tsv(full_path) - else: - return_dict[project_name] = self._load_tsv(full_path) + return_dict[project_name] = self._load_tsv(full_path) return return_dict @staticmethod def _parse_filename(filename): - filename = filename.replace("_or_", "_/_") - filename = filename.replace("_", " ") - filename = filename.replace() - return filename + """ + Function to parse the filename to the project name. + + Args: + filename: + The string of the filename of the TSV. + Returns: + string: + Returns a string that is AWS bucket naming compliant. + """ + # First, break off file extension + filename = filename.replace(".tsv.gz", "") + filename = filename.replace(".tsv", "") + # Then create an intermediate with only valid characters + name = "".join( + [c for c in filename.replace("-", "") if c.isalnum() or c == "_"] + ) + # Then clean up starting spaces + name = re.sub(r"^_*", "", name) + name = re.sub(r"_+", "-", name) + return name.lower() @staticmethod def _load_tsv(path): + """ + Method to load in the pandas TSV, even if the TSV is just headers or not even headers. + Args: + path: + Absolute path to the (gzipped) TSV file. + Returns: + dataframe: + Returns a pandas.DataFrame containing the columns of a loaded in pandas TSV or + empty dataframe containing the correct columns. + """ try: data = pd.read_csv( path, @@ -55,7 +93,6 @@ def _load_tsv(path): } ).fillna("") except pd.errors.EmptyDataError: - warnings.warn(f"Obtaining {path} resulted in empty dataframe!") data = pd.DataFrame( columns=["active", "email", "firstName", "fullName", "lastName", "roles"] ) @@ -66,25 +103,80 @@ class ProjectProcessor: def __init__(self, api): self.api = api self.su = [] + self.current_su = self._obtain_current_su(api) def parse(self, cohort_data): + """ + Bread and butter of the script. This function parses the cohort data dictionary into the + API, creating new projects and users alike. Also sets users admin rights accordingly, + as they were according to the export TSVs. + + Args: + cohort_data: + Dictionary containing (key) the cohort name and (value) its user data as a pandas + dataframe. + """ for cohort_name, cohort_users in cohort_data.items(): - if cohort_name == "NaN": + # Skip users that are not categorized into a cohort + if cohort_name == "uncategorized-users": continue self._obtain_su(cohort_users) self._put_project(name=cohort_name, users=cohort_users['email'].values.tolist()) - self._process_uncategorized_users(cohort_data['NaN']) + self._process_uncategorized_users(cohort_data['uncategorized-users']) self._add_admin_rights() - def _put_user(self, email, firstName, lastName): + @staticmethod + def _obtain_current_su(api): + """ + Method to obtain the already present superusers / admins from the target URL, to make sure + that current admins do not lose their privilege. + + Args: + api: + Initiated class of the API present in this file. + Returns: + list: + Returns a list containing the email address of all users currently marked as admin + in the target armadillo URL. + """ + su_users = [] + users = api.get_users() + for user in users: + if user["admin"]: + su_users.append(user['email']) + return su_users + + def _put_user(self, email, admin, firstname="", lastname=""): + """ + Method primarily designed to update a given user their email. + + Args: + email: + String of the users email address. + admin: + Boolean whenever the user should become an admin or not. + firstname: + [Optional] String of the user's first name. + lastname: + [Optional] String of the user's last name. + """ self.api.put_user({ "email": email, - "firstName": firstName, - "lastName": lastName, - "admin": False, + "firstName": firstname, + "lastName": lastname, + "admin": admin }) def _put_project(self, name, users): + """ + Method to call the put command + Args: + name: + users: + + Returns: + + """ self.api.put_project({"name": name, "users": users}) def _obtain_su(self, project_data): @@ -101,14 +193,15 @@ def _process_uncategorized_users(self, uncategorized_users): uncategorized_users.apply( lambda x: self._put_user( email=x['email'], - firstName=x['firstName'], - lastName=x['lastName'] - ) + admin=x['email'] in self.current_su, + firstname=x['firstName'], + lastname=x['lastName'] + ), axis=1 ) def _add_admin_rights(self): for user in self.su: - self.api.put_user({"email": user, "admin": True}) + self._put_user(email=user, admin=True) class API: @@ -118,34 +211,98 @@ def __init__(self, server): @staticmethod def _create_authorazation_header(): + """ + Method to create the authentication header and content-type header, based on whenever the + user supplies a Token (from a user that is admin ofcourse) or basic auth. + + Returns: + json: + JSON containing the 'Content-Type': 'application/json' and the 'Authorization' + based on user input. + """ token = getpass("Armadillo 3 token:") if token == "": warnings.warn("Token not supplied, requiring basic auth password!") password = getpass("Basic auth password:") - auth_header = "Basic " + base64.b64encode(("admin:" + password).encode('ascii')).decode("UTF-8") + auth_header = "Basic " + base64.b64encode(("admin:" + password).encode('UTF-8')).decode("UTF-8") else: auth_header = "Bearer " + token return {"Content-Type": "application/json", "Authorization": auth_header} def put_user(self, data): + """ + Requests function to execute the curl 'PUT' command to access/users. + + Args: + data: + JSON of the user that needs to be added or updated. + See swagger UI for format. + (URL: /swagger-ui/index.html) + Raises: + HTTPError: + HTTPError is raised when HTTP error codes 400+ are obtained as a response. + """ response = requests.put(self.target + "access/users", json=data, headers=self.header) response.raise_for_status() def put_project(self, project): + """ + Requests function to execute the curl 'PUT' command to access/projects. + + Args: + project: + JSON of the project that needs to be added or updated (very likely added). + See swagger UI for format. + (URL: /swagger-ui/index.html) + Raises: + HTTPError: + HTTPError is raised when HTTP error codes 400+ are obtained as a response. + """ response = requests.put(self.target + "access/projects", json=project, headers=self.header) response.raise_for_status() + def get_users(self): + """ + Requests function to execute the curl 'GET' command to obtain all users from target. + + Returns: + json: + Returns a json containing the users response. + See swagger UI for format. + (URL: /swagger-ui/index.html) + """ + response = requests.get(self.target + "access/users", headers=self.header) + response.raise_for_status() + return response.json() + class CommandLineParser: def __init__(self): self.parser = CommandLineInterface() def parse(self): + """ + Parser function of CommandLineParser. Will obtain the CLA, checks their validity and + mutates them to be compliant in the rest of the code. + + Returns: + tuple: + Returns a tuple containing [0] the server url and [1] the absolute path to the TSV + directory. + """ server = self._parse_server() directory = self._parse_data_directory() return server, directory def _parse_server(self): + """ + Function to parse the user CLA supplied target server. Adds http:// if missing and a / at + the end if missing. + + Returns: + url: + Fully API compliant url including scheme and a slash at the end. + """ server_argument = self.parser.get_argument("server") if not server_argument.startswith("http"): warnings.warn("CLI supplied argument did not contain scheme, adding https://") @@ -155,7 +312,18 @@ def _parse_server(self): return server_argument def _parse_data_directory(self): - directory = Path(self.parser.get_argument("user_data")) + """ + Function to check if TSVs are present in the user supplied directory + Returns: + path: + Returns the absolute path of the user supplied command line argument if (gzipped) + TSVs are present. + Raises: + IOError: + IOError is raised when the user supplied CLA is not a directory or does not contain + any TSVs. + """ + directory = Path(self.parser.get_argument("user_data")).absolute() if not os.path.isdir(directory): raise IOError("Given user data argument is not a directory") contains_tsv = False @@ -198,8 +366,8 @@ def _create_argument_parser(): "--user-data", type=str, required=True, - help="The folder in which all export TSV's are located for each cohort as they are obtained from " - "export-users.py." + help="The folder in which all export TSV's are located for " + "each cohort as they are obtained from export-users.py." ) return parser From af74202982063114446e04d0f3ddff2d01c51c74 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Fri, 15 Sep 2023 10:09:55 +0200 Subject: [PATCH 04/11] added utilitairy script to reset a localhost to just project lifecycle and user that is supplied through CLA --- scripts/reset-localhost-projects.py | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 scripts/reset-localhost-projects.py diff --git a/scripts/reset-localhost-projects.py b/scripts/reset-localhost-projects.py new file mode 100644 index 000000000..6828aa214 --- /dev/null +++ b/scripts/reset-localhost-projects.py @@ -0,0 +1,114 @@ +import os +import argparse +from getpass import getpass + +import requests + + +def main(): + user = CommandLineParser().parse() + API(user).parse() + + +class API: + def __init__(self, user): + self.target = "http://localhost:8080/" + self.user = user + self.header = self._create_header() + + def parse(self): + projects = self._parse_projects() + for project in projects: + if project != "lifecycle": + self._delete_project(project) + users = self._parse_users() + for user in users: + if user != self.user: + self._delete_user(user) + + def _parse_projects(self): + projects = [] + localhost_project_dict = self._get_projects() + for project in localhost_project_dict: + projects.append(project["name"]) + return projects + + def _parse_users(self): + users = [] + localhost_user_dict = self._get_users() + for user in localhost_user_dict: + users.append(user["email"]) + return users + + @staticmethod + def _create_header(): + token = getpass("Token: ") + return {"Authorization": "Bearer " + token} + + def _delete_project(self, project_name): + response = requests.delete(f"{self.target}access/projects/{project_name}", headers=self.header) + response.raise_for_status() + + def _delete_user(self, user_email): + response = requests.delete(f"{self.target}access/users/{user_email}", headers=self.header) + response.raise_for_status() + + def _get_projects(self): + response = requests.get(f"{self.target}access/projects", headers=self.header).json() + return response + + def _get_users(self): + response = requests.get(f"{self.target}access/users", headers=self.header).json() + return response + + +class CommandLineParser: + def __init__(self): + self.parser = CommandLineInterface() + + def parse(self): + su = self._parse_su() + return su + + def _parse_su(self): + return self.parser.get_argument("user") + + +class CommandLineInterface: + def __init__(self): + parser = self._create_argument_parser() + self.arguments = parser.parse_args() + + @staticmethod + def _create_argument_parser(): + """ + Creation of the Command Line Arguments, and whenever they should be displayed as + "required" or not. + """ + parser = argparse.ArgumentParser( + prog="Import cohort users", + description="Utilitarian script of the Armadillo migration " + "process to import users and their rights into the correct cohorts." + ) + required = parser.add_argument_group("Required arguments") + required.add_argument( + "-u", + "--user", + type=str, + required=True, + help="The single user that is registered on the localhost that needs to remain." + ) + return parser + + def get_argument(self, argument_key): + """ + Small getter function for CLI, including some error handling. + """ + if argument_key in self.arguments: + return getattr(self.arguments, argument_key) + else: + raise KeyError(f"Argument {argument_key} invalid, not found in CLI arguments.") + + +if __name__ == "__main__": + main() From d125a819576070d7a56c27e54680177cdb91f464 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Fri, 15 Sep 2023 10:11:39 +0200 Subject: [PATCH 05/11] Finalized documentation --- scripts/import-users.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/import-users.py b/scripts/import-users.py index 0f7dd8039..439b02e11 100644 --- a/scripts/import-users.py +++ b/scripts/import-users.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import os import re import base64 @@ -179,9 +181,20 @@ def _put_project(self, name, users): """ self.api.put_project({"name": name, "users": users}) - def _obtain_su(self, project_data): - su = project_data.loc[ - project_data[project_data['roles'].isin(("SU", "ADMIN", "admin"))].index, + def _obtain_su(self, cohort_users): + """ + Method to obtain all marked superusers / admins from a cohort and store them, to at the end of parsing re-add + admin rights to the users that used to have it. According to the roles "ADMIN", "admin" and "SU", the user will + be marked to become admin in armadillo 3. The reason it adds to an init variable instead of return the list is + because of the uncategorized users, which have to be processed separately, but still added to the admin list + to make sure they don't lose or not obtain their admin rights. + + Args: + cohort_users: + Pandas.dataframe object of the users of the cohort_data. Can be empty. + """ + su = cohort_users.loc[ + cohort_users[cohort_users['roles'].isin(("SU", "ADMIN", "admin"))].index, "email" ] for user in su: @@ -189,6 +202,14 @@ def _obtain_su(self, project_data): self.su.append(user) def _process_uncategorized_users(self, uncategorized_users): + """ + Method to add the "uncategorized_users.tsv" users that are registered, but are not assigned to any cohort + to the armadillo 3 server. + + Args: + uncategorized_users: + Pandas.dataframe object of the users of key "uncategorized-users" of the cohort_data dictionary. + """ self._obtain_su(uncategorized_users) uncategorized_users.apply( lambda x: self._put_user( @@ -200,6 +221,10 @@ def _process_uncategorized_users(self, uncategorized_users): ) def _add_admin_rights(self): + """ + Method to be used last after all users and cohorts have been added, to give back admin rights to the users + that used to have it. + """ for user in self.su: self._put_user(email=user, admin=True) From c6a2bc696ba92bafde2aa427cc2c68c2f1bdf229 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Fri, 15 Sep 2023 10:19:30 +0200 Subject: [PATCH 06/11] Added pipfile and pipfile.lock originating from the branch feat/extract_users_cohorts_script --- scripts/Pipfile | 2 + scripts/Pipfile.lock | 366 ++++++++++++++++++++++++++++++++----------- 2 files changed, 275 insertions(+), 93 deletions(-) diff --git a/scripts/Pipfile b/scripts/Pipfile index 484a59704..806697d21 100644 --- a/scripts/Pipfile +++ b/scripts/Pipfile @@ -8,6 +8,8 @@ minio = "*" fusionauth-client = "*" requests = "*" simple-term-menu = "*" +pandas = "2.0.3" +numpy = "1.24.4" [dev-packages] diff --git a/scripts/Pipfile.lock b/scripts/Pipfile.lock index 8eb287e93..e561c112c 100644 --- a/scripts/Pipfile.lock +++ b/scripts/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8e7f8396820223772761b875f738ae7d42ff64b9629575c0c4e470903767489b" + "sha256": "d868023798e14c6c70e7cf83969271ebf52b63179770d0de09468adc9ae4f025" }, "pipfile-spec": 6, "requires": { @@ -18,35 +18,108 @@ "default": { "certifi": { "hashes": [ - "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", - "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" ], - "index": "pypi", - "version": "==2022.12.7" + "markers": "python_version >= '3.6'", + "version": "==2023.7.22" }, "charset-normalizer": { "hashes": [ - "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", - "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" + "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96", + "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c", + "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710", + "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706", + "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020", + "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252", + "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad", + "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329", + "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a", + "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f", + "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6", + "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4", + "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a", + "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46", + "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2", + "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23", + "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace", + "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd", + "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982", + "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10", + "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2", + "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea", + "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09", + "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5", + "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149", + "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489", + "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9", + "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80", + "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592", + "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3", + "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6", + "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed", + "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c", + "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200", + "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a", + "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e", + "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d", + "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6", + "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623", + "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669", + "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3", + "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa", + "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9", + "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2", + "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f", + "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1", + "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4", + "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a", + "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8", + "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3", + "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029", + "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f", + "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959", + "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22", + "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7", + "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952", + "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346", + "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e", + "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d", + "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299", + "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd", + "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a", + "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3", + "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037", + "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94", + "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c", + "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858", + "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a", + "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449", + "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c", + "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918", + "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1", + "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c", + "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac", + "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa" ], - "markers": "python_version >= '3.6'", - "version": "==2.1.1" + "markers": "python_full_version >= '3.7.0'", + "version": "==3.2.0" }, "deprecated": { "hashes": [ - "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d", - "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d" + "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c", + "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.13" + "version": "==1.2.14" }, "fusionauth-client": { "hashes": [ - "sha256:7f93164ce90237752ae7d9528b8ac6b0af49d8c20d752d2e0694794afadfbe88", - "sha256:a4017c11982a3c7636f703fa373a4b8835edb142e6d326a11b57e5c6c348508a" + "sha256:0abfbe4ced2e4b14ce045d3aa87c6ae94c59cd818684d2b6f4570ae6ce955de2", + "sha256:a6d4265eb6fbe270fa8f96d9161d43384638b35b10c32a4d99c78920fd0242e7" ], "index": "pypi", - "version": "==1.36.0" + "version": "==1.47.0" }, "idna": { "hashes": [ @@ -58,105 +131,212 @@ }, "minio": { "hashes": [ - "sha256:12ac2d1d4fd3cea159d625847445e1bfceba3fbc2f4ab692c2d2bf716f82246c", - "sha256:1cab424275749b8b5b8bb0c6cc856d667305ef549796ae56f3237fe55306a1fc" + "sha256:56ecb1e7e0103d2dc212fb460fdb70ab2abb7fa5685db378429325d96d95587a", + "sha256:8073bed2b4b1853f3d69ab2f01a0de86264071083032985921201cfbb0950b15" + ], + "index": "pypi", + "version": "==7.1.16" + }, + "numpy": { + "hashes": [ + "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f", + "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61", + "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7", + "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400", + "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef", + "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2", + "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d", + "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc", + "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835", + "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706", + "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5", + "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4", + "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6", + "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463", + "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a", + "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f", + "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e", + "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e", + "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694", + "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8", + "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64", + "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d", + "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc", + "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254", + "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2", + "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1", + "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810", + "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9" ], "index": "pypi", - "version": "==7.1.11" + "version": "==1.24.4" + }, + "pandas": { + "hashes": [ + "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682", + "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc", + "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b", + "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089", + "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5", + "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26", + "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210", + "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b", + "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641", + "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd", + "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78", + "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b", + "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e", + "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061", + "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0", + "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e", + "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8", + "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d", + "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0", + "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c", + "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183", + "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df", + "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8", + "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f", + "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02" + ], + "index": "pypi", + "version": "==2.0.3" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", + "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" + ], + "version": "==2023.3" }, "requests": { "hashes": [ - "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", - "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" ], "index": "pypi", - "version": "==2.28.1" + "version": "==2.31.0" }, "simple-term-menu": { "hashes": [ - "sha256:3b8b1b155716c2490547836893ad480126cfc2bc84303eae5e22a7b65b746b40", - "sha256:bd019c4e2e54ac9a6d1abef32910b45f72257519736497a5a7975aca3c406554" + "sha256:368b4158d1749b868552fb6c054b8301785086c71a7253dac8404cc3cb2d30e8", + "sha256:f12945d5c6998088e86a228e0aff12ff655f5bfad786c86677f23faa1d2afa50" ], "index": "pypi", - "version": "==1.5.0" + "version": "==1.6.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "tzdata": { + "hashes": [ + "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a", + "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda" + ], + "markers": "python_version >= '2'", + "version": "==2023.3" }, "urllib3": { "hashes": [ - "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", - "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" + "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11", + "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.13" + "markers": "python_version >= '3.7'", + "version": "==2.0.4" }, "wrapt": { "hashes": [ - "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3", - "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b", - "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4", - "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2", - "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656", - "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3", - "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff", - "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310", - "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a", - "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57", - "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069", - "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383", - "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe", - "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87", - "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d", - "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b", - "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907", - "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f", - "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0", - "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28", - "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1", - "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853", - "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc", - "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3", - "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3", - "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164", - "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1", - "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c", - "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1", - "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7", - "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1", - "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320", - "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed", - "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1", - "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248", - "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c", - "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456", - "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77", - "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef", - "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1", - "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7", - "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86", - "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4", - "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d", - "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d", - "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8", - "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5", - "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471", - "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00", - "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68", - "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3", - "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d", - "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735", - "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d", - "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569", - "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7", - "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59", - "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5", - "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb", - "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b", - "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f", - "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462", - "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015", - "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af" + "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0", + "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420", + "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a", + "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c", + "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079", + "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923", + "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f", + "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1", + "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8", + "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86", + "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0", + "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364", + "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e", + "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c", + "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e", + "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c", + "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727", + "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff", + "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e", + "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29", + "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7", + "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72", + "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475", + "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a", + "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317", + "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2", + "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd", + "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640", + "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98", + "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248", + "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e", + "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d", + "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec", + "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1", + "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e", + "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9", + "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92", + "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb", + "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094", + "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46", + "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29", + "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd", + "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705", + "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8", + "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975", + "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb", + "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e", + "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b", + "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418", + "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019", + "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1", + "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba", + "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6", + "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2", + "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3", + "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7", + "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752", + "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416", + "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f", + "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1", + "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc", + "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145", + "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee", + "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a", + "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7", + "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b", + "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653", + "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0", + "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90", + "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29", + "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6", + "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034", + "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09", + "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559", + "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.14.1" + "version": "==1.15.0" } }, "develop": {} From ef516aaf3c8ca27c32013a6f9cf83deb082344fd Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Wed, 20 Sep 2023 14:58:05 +0200 Subject: [PATCH 07/11] Processed PR feedback --- scripts/import-users.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/import-users.py b/scripts/import-users.py index 439b02e11..3dbcaf25f 100644 --- a/scripts/import-users.py +++ b/scripts/import-users.py @@ -63,9 +63,10 @@ def _parse_filename(filename): # Then create an intermediate with only valid characters name = "".join( [c for c in filename.replace("-", "") if c.isalnum() or c == "_"] - ) - # Then clean up starting spaces + ).strip() + # Then clean up starting underscores name = re.sub(r"^_*", "", name) + # Replacing one or more underscores with - name = re.sub(r"_+", "-", name) return name.lower() @@ -104,8 +105,8 @@ def _load_tsv(path): class ProjectProcessor: def __init__(self, api): self.api = api - self.su = [] - self.current_su = self._obtain_current_su(api) + self.superusers = [] + self.current_superusers = self._obtain_current_superusers(api) def parse(self, cohort_data): """ @@ -128,7 +129,7 @@ def parse(self, cohort_data): self._add_admin_rights() @staticmethod - def _obtain_current_su(api): + def _obtain_current_superusers(api): """ Method to obtain the already present superusers / admins from the target URL, to make sure that current admins do not lose their privilege. @@ -141,12 +142,12 @@ def _obtain_current_su(api): Returns a list containing the email address of all users currently marked as admin in the target armadillo URL. """ - su_users = [] + super_users = [] users = api.get_users() for user in users: if user["admin"]: - su_users.append(user['email']) - return su_users + super_users.append(user['email']) + return super_users def _put_user(self, email, admin, firstname="", lastname=""): """ @@ -198,8 +199,8 @@ def _obtain_su(self, cohort_users): "email" ] for user in su: - if user not in self.su: - self.su.append(user) + if user not in self.superusers: + self.superusers.append(user) def _process_uncategorized_users(self, uncategorized_users): """ @@ -214,7 +215,7 @@ def _process_uncategorized_users(self, uncategorized_users): uncategorized_users.apply( lambda x: self._put_user( email=x['email'], - admin=x['email'] in self.current_su, + admin=x['email'] in self.current_superusers, firstname=x['firstName'], lastname=x['lastName'] ), axis=1 @@ -225,7 +226,7 @@ def _add_admin_rights(self): Method to be used last after all users and cohorts have been added, to give back admin rights to the users that used to have it. """ - for user in self.su: + for user in self.superusers: self._put_user(email=user, admin=True) From a5007c852b1a3b501276d442bcac0a70f8121bc0 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Fri, 3 Nov 2023 10:14:51 +0100 Subject: [PATCH 08/11] Processed PR feedback --- README.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 191507b3a..b90170c8a 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,76 @@ Yes, you can run in 'offline' profile ``` ### How to import data from Armadillo 2? -To export data from and Armadillo 2 server take the following steps: +For data managers, it can be very useful to export projects (and their belonging users with their user rights) from Armadillo 2 +and then later import those projects with their corresponding users into Armadillo 3. **Please note that this does not transfer the data stored in these projects!.** To do this, take the following steps: + +#### .1 Install required system packages +The user export and the user import scripts both depend on Python 3.8 and a number of libraries. +This can be easily installed through `Pipenv` (for additionally required PIP libraries) and `Pyenv` (for Python3.8). +_Please note that if you already have Python3.8 installed, you may skip `Pyenv`._ + +``` +apt install pyenv +apt install pipenv +``` +If you do not wish to install pyenv and pipenv, there is an alternative way of installing the required python libraries described in step 2. + +#### .2 Install Python required libraries + +Using `pipenv`: +``` +cd /scrips +pipenv install +pipenv shell +``` + +Alternatively, manually installing all libraries is also possible: +``` +cd /scrips +python3 -m venv venv +source ./venv/bin/activate +pip install minio fusionauth-client requests simple-term-menu pandas=2.0.3 numpy=1.24.4 +``` +You may encounter an error from the installation of pandas that numpy is not installed yet, +in that case install numpy (version 1.24.4) first and then after numpy is installed, install pandas (version 2.0.3). + +Also note that you do not have to install all required packages again if you wish to rerun the export at a later date, +simply activating the virtual environment by `source /scrips/venv/bin/activate` will be enough. + + +#### .3: Export projects and their users from an Armadillo 2 server + +`export-users.py` can be run by supplying the following arguments: + +- -f / --fusion-auth **(required)**: The URL of the Armadillo 2 instance of which the projects and its users should be exported from. Please note that `export-users.py` will prompt to supply the API key of this URL once started. +- -o / --output **(required)**: The path to the output folder. Please note that the parent of this folder HAS to exist. `export-users.py` will place a folder inside this user supplied folder named `YYYY-MM-DD`, where YYYY is the year, MM is the month in numbers 1-12 and DD is the day. +Within this folder is where all unzipped TSVs will be placed, named according to the project containing its users. + +Example: + +``` +python3 export-users.py -f https://some-armadillo2-server.org/ -o ./some_export_folder +``` +_Once started and the arguments are valid, `export-users.py` will prompt the user to supply an API key with which it can call the API to get all projects and their users exported._ + + +#### .4: Import projects and their users to an Armadillo 3 server from generated TSVs + +`import-users.py` can be run by supplying the following arguments: + +- -s / --server **(required)**: The URL of the Armadillo 3 instance of which the projects and its users should be imported to. Please note that `import-users.py` will prompt to supplie the API key of this URL once started. +- -d / --user-data **(required)**: The local directory in which all the project TSVs from step 3 are stored, including the folder named after a date (YYYY-MM-DD). + +Example: + +``` +python3 import-users.py -s https://some-armadillo3-server.org/ -d ./some_export_folder/2023-11-03/ +``` +_Once started and the arguments are valid, `import-users.py` will prompt the user to supply an API key with which it can call the API to get all the projects and their users imported._ + + +### How to migrate from Armadillo 2 to Armadillo 3? +To migrate your local Armadillo 2 to Armadillo 3 take the following steps: #### 1. Check if there's enough space left on the server ``` From 4198cec52c10d1a9f108316ff23bfe882d067be5 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Wed, 8 Nov 2023 15:49:54 +0100 Subject: [PATCH 09/11] - Processed PR feedback to add documentation on how to use import and export scrips in the upgrade-2-3.md - Fixed typo in reset-localhost-projects.py --- docs/upgrade-2-3.md | 72 ++++++++++++++++++++++++++++- scripts/reset-localhost-projects.py | 6 +-- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/upgrade-2-3.md b/docs/upgrade-2-3.md index 66810469c..7d48e2b4f 100644 --- a/docs/upgrade-2-3.md +++ b/docs/upgrade-2-3.md @@ -1,6 +1,41 @@ ### Migrate Armadillo 2 to Armadillo 3 -To export data from and Armadillo 2 server take the following steps: +Migrating from Armadillo 2 to Armadillo 3 can be done in 2 variants, a full migration including [projects, users and data](#migrate-projects-users-and-data) or [just projects and their users](#migrate-projects-and-their-users). +Both options require Python (version 3.8) and additional python libraries, described in [Getting started](#getting-started). + +### Getting started + +To start the migration, python 3.8 is advised together with a number of utilitarian python libraries. Other python versions might work, but performance has only been tested with python 3.8. +We recommend the use of `pyenv` to get multiple python versions running and the use of `pipenv` to install the additional libraries. Alternatively, if `pipenv` is not an option, one may install the required python libraries through a [python virtual environment](https://docs.python.org/3/library/venv.html). +See [install with pipenv](#install-with-pipenv) to see the installation of the python libraries with `pipenv`, alternatively see [install with Python virtual environment](#install-with-python-virtual-environment) if `pipenv` is not an option. + +#### Install with pipenv + +The following code assumes you have some sort of super user rights (either through the use of `sudo` or `su`) and the usage of Ubuntu. Change `apt` to your package manager. +The code also assumes you are already in the [scrips](https://github.com/molgenis/molgenis-service-armadillo/tree/master/scripts) directory. + +```bash +apt update +apt install pyenv pipenv +pipenv install +pipenv shell +``` + +If you wish to exit, you can type `exit` in the terminal. To re-enter, change directory to `scrips` and execute `pipenv shell`. + +#### Install with Python virtual environment + +The following code does **NOT** require super user rights. The code does assume you are already in the [scrips](https://github.com/molgenis/molgenis-service-armadillo/tree/master/scripts) directory. + +```bash +python3 -m venv venv +source ./venv/bin/activate +pip install minio fusionauth-client requests simple-term-menu numpy=1.24.4 pandas=2.0.3 +``` + +If the installation of one (or more libraries) fails, try to install the libraries one by one. + +### Migrate Projects, users and data #### 1. Check if there's enough space left on the server ``` @@ -173,3 +208,38 @@ docker rm containername -f ``` After that remove the data: ```rm -Rf /var/lib/minio/ ``` + +### Migrate Projects and their users + +Migration of just the projects and their users (with their corresponding rights) can be done by using [export-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/export-users.py) and [import-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/import-users.py). +**This options does not migrate the data!** + +#### 1. Export Projects and users from Armadillo 2 + +To export users from an Armadillo 2 server, one must use the [export-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/export-users.py) script. `export-users.py` can be used by using the following arguments: + +- -f / --fusion-auth **(required)**: The full URL (including http) of the Armadillo 2 server of which you wish to export the Projects and their users from. **Please note that `export-users.py` will prompt to supply the API key for this server once all arguments are valid!** +- -o / --output **(required)**: The output directory in which (unzipped) TSVs will be placed of all projects and their users, with the project name being the TSV name. `export-users.py` will create a new folder in the supplied output folder named: `YYYY-MM-DD`, where `YYYY` is the current year, `MM` is the current month and `DD` is the current day. + +**Again, note that `export-users.py` will prompt to supply the API key for the `-f / --fusion-auth` server once all arguments are valid!** + +Empty projects (without users) will also be exported as empty TSV (containing only the header). This is a feature that `import-users.py`, the next step, is able to function with. + +Also note that some projects might change in name, as Armadillo 3 is stricter with naming projects. + +Example: +```bash +pipenv shell +python3 export-users.py -f https://armadillo2-server.org -o ./armadillo_2_exports +``` + +#### 2. Import Projects and users TSVs into Armadillo 3 + +To import users into an Armadillo 3 server, one must use the [import-users.py](https://github.com/molgenis/molgenis-service-armadillo/blob/master/scripts/import-users.py) script. `import-users` can be used by using the following arguments: + +- -s / --server **(required)**: The full URL (including http) of the Armadillo 3 server of which you wish to import the Projects and their users TSVs in [step 1](#1-export-projects-and-users-from-armadillo-2). **Please note that `import-users.py` will prompt to supply the API key for this server once all arguments are valid!** +- -d / --user-data **(required)**: The directory, including the folder named after the year-month-day combination, where the export TSVs from [step 1](#1-export-projects-and-users-from-armadillo-2) are stored. + +**Again, note that `import-users.py` will prompt to supply the API key for the `-s / --server` server once all arguments are valid!** + +Empty TSVs from [step 1](#1-export-projects-and-users-from-armadillo-2) will be imported as empty projects with no users. diff --git a/scripts/reset-localhost-projects.py b/scripts/reset-localhost-projects.py index 6828aa214..940ee500b 100644 --- a/scripts/reset-localhost-projects.py +++ b/scripts/reset-localhost-projects.py @@ -86,9 +86,9 @@ def _create_argument_parser(): "required" or not. """ parser = argparse.ArgumentParser( - prog="Import cohort users", - description="Utilitarian script of the Armadillo migration " - "process to import users and their rights into the correct cohorts." + prog="Reset localhost projects and users", + description="Utilitarian script for developers that want to reset their localhost instance to just Project " + "'Lifecycle' and a singular user if they supply -u. Does not affect Profiles." ) required = parser.add_argument_group("Required arguments") required.add_argument( From 29bc32dc59c3ae2ea111efd0c9590d4a37d04e15 Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Thu, 9 Nov 2023 16:04:16 +0100 Subject: [PATCH 10/11] Added example for the import script --- docs/upgrade-2-3.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/upgrade-2-3.md b/docs/upgrade-2-3.md index d85765f34..26388e045 100644 --- a/docs/upgrade-2-3.md +++ b/docs/upgrade-2-3.md @@ -246,3 +246,9 @@ To import users into an Armadillo 3 server, one must use the [import-users.py](h **Again, note that `import-users.py` will prompt to supply the API key for the `-s / --server` server once all arguments are valid!** Empty TSVs from [step 1](#1-export-projects-and-users-from-armadillo-2) will be imported as empty projects with no users. + +Example: +```bash +pipenv shell +python3 import-users.py -s https://armadillo3-server.org -d ./armadillo_2_exports/2023-11-09 +``` From f9e2633abd6d2024f2ed2679b4a123ae10b2b50a Mon Sep 17 00:00:00 2001 From: SietsmaRJ Date: Thu, 30 Nov 2023 11:21:31 +0100 Subject: [PATCH 11/11] Processed PR feedback --- docs/upgrade-2-3.md | 2 +- scripts/requirements.txt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 scripts/requirements.txt diff --git a/docs/upgrade-2-3.md b/docs/upgrade-2-3.md index 26388e045..0fe290d0d 100644 --- a/docs/upgrade-2-3.md +++ b/docs/upgrade-2-3.md @@ -30,7 +30,7 @@ The following code does **NOT** require super user rights. The code does assume ```bash python3 -m venv venv source ./venv/bin/activate -pip install minio fusionauth-client requests simple-term-menu numpy=1.24.4 pandas=2.0.3 +pip install -r requirements.txt ``` If the installation of one (or more libraries) fails, try to install the libraries one by one. diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 000000000..6b5ad5b93 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,17 @@ +-i https://pypi.org/simple +certifi==2023.7.22; python_version >= '3.6' +charset-normalizer==3.2.0; python_full_version >= '3.7.0' +deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +fusionauth-client==1.47.0 +idna==3.4; python_version >= '3.5' +minio==7.1.16 +numpy==1.24.4 +pandas==2.0.3 +python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +pytz==2023.3 +requests==2.31.0 +simple-term-menu==1.6.1 +six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +tzdata==2023.3; python_version >= '2' +urllib3==2.0.4; python_version >= '3.7' +wrapt==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'