diff --git a/.gitignore b/.gitignore index b46497b..1932f84 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,9 @@ test-logs test-logs/* coverage.tmp.out coverage.out + +# Python3 +*__pycache__* +*.log +*token.json +*creds.json diff --git a/googleimport/achievement.py b/googleimport/achievement.py index c0ce0b0..12d6d72 100644 --- a/googleimport/achievement.py +++ b/googleimport/achievement.py @@ -4,5 +4,11 @@ class Achievement: def __init__(self, count : str, description : str): self.count = count self.description = description + + def ToDict(self): + return { + 'count': self.count, + 'description': self.description + } \ No newline at end of file diff --git a/googleimport/export.py b/googleimport/export.py new file mode 100644 index 0000000..bc387f8 --- /dev/null +++ b/googleimport/export.py @@ -0,0 +1,262 @@ +from organisation import Organization +from member import Member +from achievement import Achievement +import requests as re +import os +import utils +import random as r + +class AuthorizationError(re.HTTPError): + pass + +class UpdatePhotoError(re.HTTPError): + pass + +class Exporter: + + def autoLogin(func): + def wrapper(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except AuthorizationError as e: + self.login() + return func(self, *args, **kwargs) + except: + raise + + return wrapper + + def __init__(self, ip, port, adminLogin, adminPassword): + self.ip = ip + self.port = port + self.adminLogin = adminLogin + self.adminPassword = adminPassword + self.loginSession = re.Session() + + def login(self): + response = self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/guard/login/", + json={ + "login": self.adminLogin, + "password": self.adminPassword + } + ) + + if response.status_code != 200: + raise re.HTTPError("Can't login:", response=response) + + def logout(self): + self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/guard/logout/" + ) + + @autoLogin + def GetRandomDefaultMedia(self) -> int: + response = self.loginSession.get( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/media/default/" + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 200: + raise re.HTTPError("Can't get random default media:", response=response) + return r.choice(response.json()['media'])['id'] + + @autoLogin + def ExportPhotoPrivate(self, filepath) -> int: + if not os.path.exists(filepath): + raise FileNotFoundError(f"File '{filepath}' not found.") + + with open(filepath, 'rb') as file: + b = file.read() + name = os.path.basename(filepath) + + data = utils.BytesToIntList(b) + response = self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/media/private/", + json={ + "name": name, + "data" : data + } + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code != 200: + raise re.HTTPError(f"Can't export photo: {response.status_code}", response=response) + return response.json()['id'] + + @autoLogin + def GetClubIdByName(self, clubName): + response = self.loginSession.get( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/clubs/", + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 200: + raise re.HTTPError("Can't get club id by name:", response=response) + + clubs = [club for club in response.json()['clubs']] + for c in clubs: + if c['name'] == clubName: + return c['id'] + else: + raise ValueError(f"Club '{clubName}' not found.") + + @autoLogin + def ExportOrganisationOnly(self, club: Organization, photoId: int) -> int: + response = self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/clubs/", + json={ + "name" : club.Name, + "short_name" : club.ShortName, + "description" : club.Description, + "short_description": club.ShortDescription, + "type" : club.ClubType, + "logo_id" : photoId, + "vk_url" : club.Vk, + "tg_url" : club.Telegram, + "parent_id" : 0, + "orgs" : [] + } + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 200: + raise re.HTTPError("Can't organisation:", response=response) + + + return self.GetClubIdByName(club.Name) + + @autoLogin + def GetMemberIdByName(self, name): + response = self.loginSession.get( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/members/search/{name}", + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 200: + raise re.HTTPError("Can't get member id by name:", response=response) + + return response.json()['members'][0]['id'] + + # default password: qwerty12345678 + # Транслитом Фамилия и инициалы. + # Если коллизия, то к фамилиии инициалам добавить цифру. + # На наличие в базе смотреть по имени. + def ExportMember(self, member: Member, photoId: int) -> int: + self.logout() + response = self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/guard/register/", + json={ + "login": member.GetLogin(), + "password" : member.GetPassword(), + "name" : member.GetName(), + "telegram": member.GetTelegram(), + "vk": member.GetVk(), + } + ) + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 201: + raise re.HTTPError("Can't export member:", response=response) + + self.logout() + + id = self.GetMemberIdByName(member.GetName()) + response = self.loginSession.put( + url = f"{self.ip}:{self.port}/bmstu-stud-web/api/members/{id}", + json={ + "is_admin": False, + "login": member.GetLogin(), + "media_id": photoId, + "name": member.GetName(), + "telegram": member.GetTelegram(), + "vk": member.GetVk() + } + ) + + if response.status_code != 200: + raise UpdatePhotoError(f"Can't update photo for member id = {id}", response=response) + + return id + + + @autoLogin + def AddOrgsToClub(self, clubID: int, members: dict[int, Member]): + response = self.loginSession.get( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/clubs/{clubID}" + ) + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 200: + raise re.HTTPError("Can't get club info:", response=clubInfo) + + clubInfo = response.json() + orgs = [] + for memberID, member in members.items(): + orgs.append({ + "member_id": memberID, + "role_name": member.GetRoleName(), + "role_spec": member.GetRoleSpec() + }) + + response = self.loginSession.put( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/clubs/{clubID}", + json={ + "description": clubInfo["description"], + "logo_id": clubInfo["logo"]['id'], + "name": clubInfo["name"], + "orgs": orgs, + "parent_id": clubInfo["parent_id"], + "short_description": clubInfo["short_description"], + "short_name": clubInfo["short_name"], + "tg_url": clubInfo["tg_url"], + "type": clubInfo["type"], + "vk_url": clubInfo["vk_url"] + } + ) + if response.status_code != 200: + raise re.HTTPError(f"Can't update club info for club id = {clubID}", response=response) + + + + @autoLogin + def ExportClubPhoto(self, photoId: int, clubId: int, refNumber: int): + response = self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/clubs/media/{clubId}", + json={ + "photos": [ + { + "ref_number": refNumber, + "media_id" : photoId + } + ] + } + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code != 201: + raise re.HTTPError("Can't export club photo:", response=response) + + + @autoLogin + def ExportAchievement(self, achievement: Achievement, clubId: int): + response = self.loginSession.post( + url=f"{self.ip}:{self.port}/bmstu-stud-web/api/feed/encounters/", + json={ + "club_id": clubId, + "count" : achievement.count, + "description" : achievement.description + } + ) + + if response.status_code == 401: + raise AuthorizationError("Not authorized.") + if response.status_code!= 201: + raise re.HTTPError("Can't export achievement:", response=response) + diff --git a/googleimport/main.py b/googleimport/main.py new file mode 100644 index 0000000..4a55f3a --- /dev/null +++ b/googleimport/main.py @@ -0,0 +1,47 @@ +from organisationparser import OrganizationParser, SCOPES +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from export import Exporter +from organizationexport import OrganizationExporter +import os +import logging + +SAMPLE_SPREADSHEET_ID = "1lfzZuoui21E78wYmF6fBPXHimgREkx81iSSFF45dOw8" +LOST_DATA_JSON = 'BAS.json' +LOG_FILE = 'BAS.log' +creds = None +# The file token.json stores the user's access and refresh tokens, and is +# created automatically when the authorization flow completes for the first +# time. +if os.path.exists("token.json"): + creds = Credentials.from_authorized_user_file("token.json", SCOPES) +# If there are no (valid) credentials available, let the user log in. +if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + "creds.json", SCOPES) + creds = flow.run_local_server(port=0) +# Save the credentials for the next run + with open("token.json", "w") as token: + token.write(creds.to_json()) + + +l = logging.getLogger("OrganizationParser") +l.setLevel(logging.INFO) +logging.basicConfig(level=logging.DEBUG, filename=LOG_FILE, filemode="w") +# logging.basicConfig(level=logging.WARNING, filename="warning.log", filemode="w") + +org = OrganizationParser(SAMPLE_SPREADSHEET_ID, creds, l) +org.ParseAndDownload() +orgas = org.GetOrganisation() + +e = Exporter("http://localhost", 5000, "TestHeadMaster", "12345678") + +oe = OrganizationExporter(l, orgas, e, LOST_DATA_JSON) +oe.StartExport() + + + diff --git a/googleimport/member.py b/googleimport/member.py index eb6d1a4..73e961b 100644 --- a/googleimport/member.py +++ b/googleimport/member.py @@ -12,6 +12,8 @@ def __init__(self, name, photoUrl, telegram, vk, roleName, roleSpec, roleField): self.roleName = roleName self.roleSpec = roleSpec self.roleField = roleField + self.login = None + self.password = None def GetPhotoGoogleId(self): return self.photoGoogleId @@ -19,5 +21,49 @@ def GetPhotoGoogleId(self): def SetOsPhotoPath(self, path): self.osPhotoPath = path + def GetOsPhotoPath(self): + return self.osPhotoPath + def GetName(self): - return self.name \ No newline at end of file + return self.name + + def GetTelegram(self): + return self.telegram + + def GetVk(self): + return self.vk + + def GetRoleName(self): + return self.roleName + + def GetRoleSpec(self): + return self.roleSpec + + def GetRoleField(self): + return self.roleField + + def GetLogin(self): + return self.login + + def GetPassword(self): + return self.password + + def SetLogin(self, login): + self.login = login + + def SetPassword(self, password): + self.password = password + + def ToDict(self): + return { + 'name': self.name, + 'password': self.password, + 'role_name' : self.roleName, + 'role_spec' : self.roleSpec, + 'role_field' : self.roleField, + 'login': self.login, + 'photo_path': self.osPhotoPath, + 'photo_id': self.photoGoogleId, + 'telegram': self.telegram, + 'vk': self.vk + } \ No newline at end of file diff --git a/googleimport/organisationparser.py b/googleimport/organisationparser.py index b8e8e00..da3a20d 100644 --- a/googleimport/organisationparser.py +++ b/googleimport/organisationparser.py @@ -1,15 +1,12 @@ import json -from googleapiclient.discovery import build import os.path import utils from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow -from googleapiclient.discovery import build -from googleapiclient.errors import HttpError from member import Member from achievement import Achievement -from googlesheet import GoogleSheetRange, GoogleSpreadsheet +from googlesheet import GoogleSpreadsheet from googleloader import GoogleLoader from organisation import Organization import googleloader @@ -17,7 +14,6 @@ import logging SCOPES = googlesheet.SCOPES + googleloader.SCOPES -SAMPLE_SPREADSHEET_ID = "1lfzZuoui21E78wYmF6fBPXHimgREkx81iSSFF45dOw8" Settings = json.load(open("settings.json")) class OrganizationParser: @@ -29,6 +25,16 @@ def __init__(self, spreadsheetID: str, creds, logger: logging.Logger): self.organisation = Organization() self.logger = logger + def GetOrganisation(self): + return self.organisation + + def ParseAndDownload(self): + self.ParseSpreadsheet() + self.ParseMembers() + self.ParseAchievements() + self.DownloadLogo() + self.DownloadOrganizationPhotos() + self.DownloadMemberPhotos() def ParseSpreadsheet(self): self.logger.info(f"Start loading spreadsheet id={self.spreadsheetGoogleID}.") @@ -59,9 +65,6 @@ def ParseSpreadsheet(self): self.organisation.Vk = urlRange[f"{Settings["urls_column"]}{Settings["vk_url_row"]}"] self.logger.debug(f"Parsed organisation {self.organisation.__dict__}.") - - self.ParseMembers() - self.ParseAchievements() def ParseMembers(self): self.logger.info(f"Parsing members of organisation {self.organisation.Name}, id={self.spreadsheetGoogleID}.") @@ -200,36 +203,3 @@ def GetOrganisationDir(self): def GetOrganisationSubdir(self, subdir): return os.path.join(os.path.join(Settings["data_dir"], self.organisation.Name, subdir)) - - -if __name__ == "__main__": - creds = None - # The file token.json stores the user's access and refresh tokens, and is - # created automatically when the authorization flow completes for the first - # time. - if os.path.exists("token.json"): - creds = Credentials.from_authorized_user_file("token.json", SCOPES) - # If there are no (valid) credentials available, let the user log in. - if not creds or not creds.valid: - if creds and creds.expired and creds.refresh_token: - creds.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file( - "creds.json", SCOPES) - creds = flow.run_local_server(port=0) - # Save the credentials for the next run - with open("token.json", "w") as token: - token.write(creds.to_json()) - - - l = logging.getLogger("OrganizationParser") - l.setLevel(logging.INFO) - logging.basicConfig(level=logging.DEBUG, filename="info.log", filemode="w") - # logging.basicConfig(level=logging.WARNING, filename="warning.log", filemode="w") - - org = OrganizationParser(SAMPLE_SPREADSHEET_ID, creds, l) - org.ParseSpreadsheet() - # print(org.organisation.__dict__) - org.DownloadLogo() - org.DownloadOrganizationPhotos() - org.DownloadMemberPhotos() \ No newline at end of file diff --git a/googleimport/organizationexport.py b/googleimport/organizationexport.py new file mode 100644 index 0000000..92aef51 --- /dev/null +++ b/googleimport/organizationexport.py @@ -0,0 +1,203 @@ +import logging +from organisation import Organization +from member import Member +from export import Exporter +import os +import json +import utils + +DEFAULT_PASSWORD = 'qwerty12345678' + +class OrganizationExporter: + def __init__(self, logger: logging.Logger, organization : Organization, exporter: Exporter, lostDataFileName: str): + self.logger = logger + self.organization = organization + self.exporter = exporter + self.lostData = dict() + self.lostDataFileName = lostDataFileName + + def StartExport(self): + clubId = self.exportOrganisation() + memberMap = self.exportMembers() + self.exportClubOrgs(memberMap, clubId) + self.exportAchievements(clubId) + self.exportClubPhoto(clubId) + self.saveLostData() + + + + def exportOrganisation(self): + self.logger.info(f"Start export club '{self.organization.Name}'") + logo = None + if self.organization.OsLogoPath != None: + try: + print(self.organization.OsLogoPath) + logo = self.exporter.ExportPhotoPrivate(self.organization.OsLogoPath) + except Exception as e: + self.logger.error(f"Error exporting logo '{self.organization.Name}': {e}") + + + if logo == None: + if self.organization.OsLogoPath == None: + self.logger.info("No logo path specified for club '{self.organization.Name}. Using default.") + else: + self.logger.info(f"Using default Media for club '{self.organization.Name}") + try: + logo = self.exporter.GetRandomDefaultMedia() + except BaseException as e: + self.logger.error(f"Error getting default media: {str(e)}") + raise + + + + self.logger.info(f"Got logo for '{self.organization.Name}' with ID: {logo}") + try: + id = self.exporter.ExportOrganisationOnly(self.organization, logo) + except Exception as e: + self.logger.error(f"Error posting club: {str(e)}") + raise + + return id + + def exportAchievements(self, clubId): + self.logger.info(f"Exporting achievements for '{self.organization.Name}'") + self.lostData['achievements'] = [] + countSuccess = 0 + countFailed = 0 + for a in self.organization.Achievements: + try: + self.exporter.ExportAchievement(a, clubId) + countSuccess += 1 + except BaseException as e: + self.logger.error(f"Error exporting achievement: {a.__dict__}.") + countFailed += 1 + self.lostData['achievments'].append(a.ToDict()) + + self.logger.info(f"Done exporting achievements: success: {countSuccess}, failed: {countFailed}") + if countFailed != 0: + self.logger.error(f"Lost some achievement. Search in lostdata json") + + def exportMembers(self) -> dict[int, Member]: + self.logger.info(f"Exporting members for '{self.organization.Name}'") + self.lostData['members'] = [] + self.lostData['member_photos'] = [] + memberMap = dict() + countSuccess = 0 + countFailed = 0 + for m in self.organization.Members: + m.SetLogin(utils.CreateLoginFromName(m.GetName())) + m.SetPassword(DEFAULT_PASSWORD) + try: + memberId = self.exporter.GetMemberIdByName(m.GetName()) + memberMap[memberId] = m + countSuccess += 1 + self.logger.info(f"Found member {m.GetName()} in database.") + continue + + except Exception as e: + self.logger.info(f"Not found member for '{m.GetName()}': {str(e)}") + photoPath = m.GetOsPhotoPath() + photoId = None + if photoPath != None: + try: + photoId = self.exporter.ExportPhotoPrivate(photoPath) + except Exception as e: + self.logger.error(f"Error exporting photo for member '{m.GetName()}': {str(e)}") + self.lostData['member_photos'].append(photoPath) + + if photoId == None: + self.logger.info(f"No photo found for member '{m.GetName()}'. Using default photo") + try: + photoId = self.exporter.GetRandomDefaultMedia() + except BaseException as e: + self.logger.error(f"Error getting default media: {str(e)}") + self.lostData['members'].append(m.ToDict()) + countFailed += 1 + continue + + try: + mid = self.exporter.ExportMember(m, photoId) + self.logger.info(f"Member '{m.GetName()}' exported") + memberMap[mid] = m + except Exception as e: + self.logger.error(f"Error exporting member: {m.ToDict()}.") + self.lostData['members'].append(m.ToDict()) + countFailed += 1 + continue + + self.logger.info(f"Done exporting members: success: {countSuccess}, failed: {countFailed}") + if countFailed!= 0: + self.logger.error(f"Lost some members. Search in lostdata json") + return memberMap + + def exportClubOrgs(self, memberMap: dict[int, Member], clubId): + self.logger.info(f"Exporting club organizators for '{self.organization.Name}'") + try: + self.exporter.AddOrgsToClub(clubId, memberMap) + self.logger.info(f"Club organizations exported for '{self.organization.Name}'") + except Exception as e: + self.logger.error(f"Error adding club organizators: {str(e)}") + self.lostData['club_orgs'] = { + "club_id" : clubId, + "members" : [ + { + "member_id":mid, + "role_name": member.GetRoleName(), + "role_spec": member.GetRoleSpec() + } + for mid, member in memberMap.items() + ] + } + + def exportClubPhoto(self, clubId): + countSuccess = 0 + countFailed = 0 + + self.lostData['club_photos'] = [] + self.logger.info(f"Exporting club photo for '{self.organization.Name}'") + photoPath = self.organization.OsPhotosPath + if photoPath == None or not os.path.isdir(photoPath): + self.logger.error(f"No photo path specified for club '{self.organization.Name}'.") + return + photoPaths = os.listdir(photoPath) + if len(photoPaths) == 0: + self.logger.error(f"No photos found in '{photoPath}'.") + return + ref_num = 1 + for photo in photoPaths: + curPath = os.path.join(photoPath, photo) + try: + photoId = self.exporter.ExportPhotoPrivate(curPath) + self.exporter.ExportClubPhoto(photoId, clubId, ref_num) + self.logger.info(f"Photo '{photo}' exported for '{self.organization.Name}'") + ref_num +=1 + except Exception as e: + self.logger.error(f"Error exporting photo '{photo}': {str(e)}") + self.lostData['club_photos'].append(photo) + self.logger.info(f"Done exporting club photo for '{self.organization.Name}'") + self.logger.info(f"Photo '{self.organization.Name}: success: {countSuccess}, failed: {countFailed}") + if countFailed!= 0: + self.logger.error(f"Lost some photos. Search in lostdata json") + + def saveLostData(self): + try: + with open(self.lostDataFileName, 'w') as outfile: + json.dump(self.lostData, outfile, indent=4) + except Exception as e: + self.logger.error(f"Error saving lost data: {str(e)}. Printing in log:") + self.logger.error(self.lostData) + + + + + + + + + + + + + + + diff --git a/googleimport/utils.py b/googleimport/utils.py index cbb3168..0e75883 100644 --- a/googleimport/utils.py +++ b/googleimport/utils.py @@ -1,3 +1,5 @@ +import transliterate as tr + def ParseSharedFolderID(url: str) -> str: """Parses the Shared Folder ID from the Google Drive URL.""" @@ -19,3 +21,17 @@ def ParseSharedFileID(url: str) -> str: fileIndex += 1 raise ValueError('Could not parse Shared File ID: no files found') +def BytesToIntList(b : bytes) -> list[int]: + """Converts a byte array to a list of integers(every byte to one integer).""" + return [int(el) for el in b] + +def CreateLoginFromName(name : str) -> str: + nameParts = name.split() + login = "" + loginParts = [] + + loginParts.append(tr.translit(nameParts[0], 'ru', reversed=True)) + for part in nameParts[1:]: + loginParts.append(tr.translit(part, 'ru', reversed=True)[0]) + login = '_'.join(loginParts) + return login.lower() \ No newline at end of file diff --git a/internal/app/mapper/club.go b/internal/app/mapper/club.go index ef6ea20..aeedca4 100644 --- a/internal/app/mapper/club.go +++ b/internal/app/mapper/club.go @@ -56,6 +56,7 @@ func MakeResponseClub(club *domain.Club, mainOrgs []domain.ClubOrg, subOrgs []do Type: club.Type, VkUrl: club.VkUrl, TgUrl: club.TgUrl, + ParentID: club.ParentID, MainOrgs: make([]responses.MainOrg, 0), SubOrgs: make([]responses.SubClubOrg, 0), } diff --git a/internal/domain/responses/get-club.go b/internal/domain/responses/get-club.go index 7bf24d2..1ac6c37 100644 --- a/internal/domain/responses/get-club.go +++ b/internal/domain/responses/get-club.go @@ -14,6 +14,7 @@ type GetClub struct { Logo domain.MediaFile `json:"logo"` VkUrl string `json:"vk_url"` TgUrl string `json:"tg_url"` + ParentID int `json:"parent_id"` MainOrgs []MainOrg `json:"main_orgs"` SubOrgs []SubClubOrg `json:"sub_orgs"` } diff --git a/internal/ports/clubs.go b/internal/ports/clubs.go index aa4f5de..4b55c85 100644 --- a/internal/ports/clubs.go +++ b/internal/ports/clubs.go @@ -745,7 +745,7 @@ func (h *ClubsHandler) PostClubMedia(w http.ResponseWriter, req *http.Request) h return handler.InternalServerErrorResponse() } - return handler.OkResponse(nil) + return handler.CreatedResponse(nil) } // DeleteClubPhoto