From 85b1ee5df79c3fb6aaec4d62dd3f6df71be43d66 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Sat, 26 Aug 2023 00:15:04 +0200 Subject: [PATCH] Improve postprocessing of new machines (#121) * feat: postprocess title/area * feat: postprocess address * feat: postprocess address * change pull request routing to commit everything to the data branch * fix import * debug fuzzy search and gmaps api to newest version * rename branch to final name * bug fixes in app after testing * make frontend display http responses * refactor: simplify ID computation * chore: improve user messages * refactor: add labels to PR * change flask url and add response text for connection failures * version update message * show loading wheel and then alert with correct message --------- Co-authored-by: NinaWie --- PennyMe.xcodeproj/project.pbxproj | 4 +- PennyMe/NewMachineRequest.swift | 99 ++++++---- PennyMe/PinViewController.swift | 4 +- PennyMe/VersionManager.swift | 2 +- PennyMe/ViewController.swift | 2 +- backend/app.py | 192 ++++++++++++------ backend/pennyme/github_update.py | 201 +++++++++++++++++++ backend/pennyme/locations.py | 311 ++++++++++++++++++++++++++++++ backend/pennyme/utils.py | 31 +++ backend/pull_request.py | 128 ------------ backend/requirements.txt | 4 +- 11 files changed, 741 insertions(+), 237 deletions(-) create mode 100644 backend/pennyme/github_update.py create mode 100644 backend/pennyme/utils.py delete mode 100644 backend/pull_request.py diff --git a/PennyMe.xcodeproj/project.pbxproj b/PennyMe.xcodeproj/project.pbxproj index 5030702b..ab1c1343 100644 --- a/PennyMe.xcodeproj/project.pbxproj +++ b/PennyMe.xcodeproj/project.pbxproj @@ -501,7 +501,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.8; + MARKETING_VERSION = 1.9; PRODUCT_BUNDLE_IDENTIFIER = "PennyMe--com.de.pennyme"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 4.2; @@ -521,7 +521,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.8; + MARKETING_VERSION = 1.9; PRODUCT_BUNDLE_IDENTIFIER = "PennyMe--com.de.pennyme"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 4.2; diff --git a/PennyMe/NewMachineRequest.swift b/PennyMe/NewMachineRequest.swift index 9003f673..b081c5f0 100644 --- a/PennyMe/NewMachineRequest.swift +++ b/PennyMe/NewMachineRequest.swift @@ -78,7 +78,7 @@ struct ConfirmationMessageView: View { } } -@available(iOS 13.0, *) +@available(iOS 14.0, *) struct RequestFormView: View { let coords: CLLocationCoordinate2D // Properties to hold user input @@ -88,13 +88,12 @@ struct RequestFormView: View { @State private var paywall: Bool = false @State private var multimachine: String = "" @State private var showFinishedAlert = false - @State private var submittedName: String = "" + @State private var displayResponse: String = "" @Environment(\.presentationMode) private var presentationMode // Access the presentationMode environment variable @State private var selectedImage: UIImage? = nil @State private var isImagePickerPresented: Bool = false - @State private var isSubmitting = false @State private var showAlert = false - @State private var submitted = false + @State private var isLoading = false @State private var keyboardHeight: CGFloat = 0 private var keyboardObserver: AnyCancellable? @@ -159,31 +158,28 @@ struct RequestFormView: View { } .padding() - // Submit button - Button(action: { - submitRequest() - }) { - Text("Submit") + if isLoading { + ProgressView("Loading...") .padding() - .foregroundColor(Color.white) - .frame(maxWidth: .infinity) - .background(Color.blue) - .cornerRadius(10) - }.padding().disabled(isSubmitting) - - // Enter all info - Text("\(submittedName)").foregroundColor(Color.red) + } else { + Button(action: { + submitRequest() + }) { + Text("Submit") + .padding() + .foregroundColor(Color.white) + .frame(maxWidth: .infinity) + .background(Color.blue) + .cornerRadius(10) + }.padding().disabled(isLoading) + } - AlertPresenter(showAlert: $showFinishedAlert, title: "Finished", message: "Thanks for adding this machine. We will review this request and the machine will be added shortly.") + AlertPresenter(showAlert: $showFinishedAlert, title: "Finished", message: "Thanks for suggesting this machine. We will review this request shortly. Note that it can take up to a few days until the machine becomes visible.") .padding() } .alert(isPresented: $showAlert) { - Alert(title: Text("Processing"), message: Text("Please wait..."), dismissButton: .default(Text("Dismiss"))) - } - .onAppear { - // Call the private function to regulate the machine - checkRequest() + Alert(title: Text("Attention!"), message: Text(displayResponse), dismissButton: .default(Text("Dismiss"))) } .padding() .navigationBarTitle("Add new machine") @@ -194,37 +190,35 @@ struct RequestFormView: View { .padding(.bottom, keyboardHeight) } - private func checkRequest() { - if submitted{ - showAlert = true - } + private func finishLoading(message: String) { + displayResponse = message + showAlert = true + isLoading = false } // Function to handle the submission of the request private func submitRequest() { - submitted = true + isLoading = true if name == "" || address == "" || area == "" || selectedImage == nil { - submittedName = "Please enter all information & upload image" + finishLoading(message: "Please enter all information & upload image") } else { - showAlert = true // correct multimachine information if multimachine == "" { multimachine = "1" } - isSubmitting = true // upload image and make request if let image = selectedImage! as UIImage ?? nil { // Convert the image to a data object guard let imageData = image.jpegData(compressionQuality: 1.0) else { print("Failed to convert image to data") - submittedName = "Something went wrong with your image" + finishLoading(message: "Something went wrong with your image") return } // call flask method called create_machine let urlString = flaskURL+"/create_machine?title=\(name)&address=\(address)&lat_coord=\(coords.latitude)&lon_coord=\(coords.longitude)&multimachine=\(multimachine)&paywall=\(paywall)&area=\(area)" guard let url = URL(string: urlString.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? "None" ) else { - submittedName = "Something went wrong. Please try to re-enter the information" + finishLoading(message: "Something went wrong. Please try to re-enter the information") return } var request = URLRequest(url: url) @@ -244,13 +238,42 @@ struct RequestFormView: View { // Create a URLSessionDataTask to send the request let task = URLSession.shared.dataTask(with: request) { (data, response, error) in if let error = error { - print("Error: \(error)") + finishLoading(message: "Something went wrong. Please check your internet connection and try again") return } - DispatchQueue.main.async { - self.showFinishedAlert = true - self.presentationMode.wrappedValue.dismiss() - isSubmitting = false + // Check if a valid HTTP response was received + guard let httpResponse = response as? HTTPURLResponse else { + finishLoading(message: "Something went wrong. Please check your internet connection and try again") + return + } + // Extract the status code from the HTTP response + let statusCode = httpResponse.statusCode + + // Check if the status code indicates success (e.g., 200 OK) + if 200 ..< 300 ~= statusCode { + // everything worked, finish + DispatchQueue.main.async { + self.showFinishedAlert = true + self.presentationMode.wrappedValue.dismiss() + isLoading = false + } + } + else { + if let responseData = data { + do { + // Parse the JSON response + if let json = try JSONSerialization.jsonObject(with: responseData, options: []) as? [String: Any] { + // Handle the JSON data here + if let answerString = json["error"] as? String { + finishLoading(message: answerString) + return + } + } + } catch { + print("JSON parsing error: \(error)") + finishLoading(message: "Something went wrong. Please check your internet connection and try again") + } + } } } task.resume() diff --git a/PennyMe/PinViewController.swift b/PennyMe/PinViewController.swift index 859c8e08..f1dcc0e2 100644 --- a/PennyMe/PinViewController.swift +++ b/PennyMe/PinViewController.swift @@ -11,7 +11,7 @@ import MapKit var FOUNDIMAGE : Bool = false -let flaskURL = "http://37.120.179.15:5000/" +let flaskURL = "http://37.120.179.15:6006/" let imageURL = "http://37.120.179.15:8000/" class PinViewController: UITableViewController, UIImagePickerControllerDelegate, UINavigationControllerDelegate { @@ -289,7 +289,7 @@ class PinViewController: UITableViewController, UIImagePickerControllerDelegate, func chooseImage() { if UIImagePickerController.isSourceTypeAvailable(.savedPhotosAlbum){ // Create the alert controller - let alertController = UIAlertController(title: "Attention!", message: "Your image will be shown to all users of the app! Please be considerate. Upload only images that are strictly related to penny machines. With the upload, you grant the PennyMe team the unrestricted right to process, alter, share, distribute and publicly expose this image.", preferredStyle: .alert) + let alertController = UIAlertController(title: "Attention!", message: "Your image will be shown to all users of the app! Please be considerate. Upload an image of the penny machine, not just an image of a coin. With the upload, you grant the PennyMe team the unrestricted right to process, alter, share, distribute and publicly expose this image.", preferredStyle: .alert) // Create the OK action let okAction = UIAlertAction(title: "OK", style: .default) { (_) in diff --git a/PennyMe/VersionManager.swift b/PennyMe/VersionManager.swift index b2ecbafb..ca84c9be 100644 --- a/PennyMe/VersionManager.swift +++ b/PennyMe/VersionManager.swift @@ -27,7 +27,7 @@ class VersionManager { func showVersionInfoAlertIfNeeded() { if shouldShowVersionInfo() { - let alert = UIAlertController(title: "PennyMe v\(currentVersion ?? "")", message: "Add new machine (BUGFIX)! \n This version allows you to submit a request to add a new machine to the map. Simply press longer on the map, input some information, upload a picture as a “proof” and complete the PennyMe database with your contributions!\n In addition, search results are now colored based on machine status.", preferredStyle: .alert) + let alert = UIAlertController(title: "PennyMe v\(currentVersion ?? "")", message: "This version includes various small usability improvements. Pop-ups should be more clear now. When you create a new machine (via long-tap on the map) the pop-ups are more informative.", preferredStyle: .alert) alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil)) UIApplication.shared.windows.first?.rootViewController?.present(alert, animated: true, completion: nil) } diff --git a/PennyMe/ViewController.swift b/PennyMe/ViewController.swift index 2851770f..3df0ffe7 100644 --- a/PennyMe/ViewController.swift +++ b/PennyMe/ViewController.swift @@ -499,7 +499,7 @@ extension ViewController: MKMapViewDelegate { guard let annotation = (sender.view as? MKAnnotationView)?.annotation else {return} // first option: it's a new machine pin - present form if let newmachine = annotation as? NewMachine { - if #available(iOS 13.0, *) { + if #available(iOS 14.0, *) { let swiftUIViewController = UIHostingController(rootView: RequestFormView(coordinate: newmachine.coordinate) ) present(swiftUIViewController, animated: true, completion: removeNewMachinePin) diff --git a/backend/app.py b/backend/app.py index 248297f0..088cda6b 100644 --- a/backend/app.py +++ b/backend/app.py @@ -2,13 +2,18 @@ import os import time from datetime import datetime +from typing import Any, Dict + from flask import Flask, jsonify, request +from googlemaps import Client as GoogleMaps from PIL import Image, ImageOps -from typing import Dict, Any +from haversine import haversine +from pennyme.locations import COUNTRIES +from pennyme.github_update import push_to_github from slack import WebClient from slack.errors import SlackApiError -from pull_request import push_to_github_and_open_pr +from thefuzz import process as fuzzysearch app = Flask(__name__) @@ -18,8 +23,10 @@ PATH_SERVER_LOCATION = os.path.join("..", "..", "images", "server_locations.json") SLACK_TOKEN = os.environ.get("SLACK_TOKEN") IMG_PORT = "http://37.120.179.15:8000/" +GM_API_KEY = open("../../gpc_api_key.keypair", "r").read() client = WebClient(token=os.environ["SLACK_TOKEN"]) +gm_client = GoogleMaps(GM_API_KEY) with open("blocked_ips.json", "r") as infile: # NOTE: blocking an IP requires restart of app.py via waitress @@ -28,22 +35,27 @@ with open(PATH_MACHINES, "r", encoding="latin-1") as infile: d = json.load(infile) MACHINE_NAMES = { - elem["properties"]["id"]: - f"{elem['properties']['name']} ({elem['properties']['area']})" + elem["properties"][ + "id" + ]: f"{elem['properties']['name']} ({elem['properties']['area']})" for elem in d["features"] } -with open('ip_comment_dict.json', 'r') as f: +with open("ip_comment_dict.json", "r") as f: IP_COMMENT_DICT = json.load(f) + def reload_server_data(): # add server location IDs with open(PATH_SERVER_LOCATION, "r", encoding="latin-1") as infile: d = json.load(infile) for elem in d["features"]: - MACHINE_NAMES[elem["properties"]["id"]] = f"{elem['properties']['name']} ({elem['properties']['area']})" + MACHINE_NAMES[ + elem["properties"]["id"] + ] = f"{elem['properties']['name']} ({elem['properties']['area']})" return MACHINE_NAMES + @app.route("/add_comment", methods=["GET"]) def add_comment(): """ @@ -55,7 +67,7 @@ def add_comment(): ip_address = request.remote_addr if ip_address in blocked_ips: - return jsonify("Blocked IP address") + return jsonify({"error": "User IP address is blocked"}), 403 path_machine_comments = os.path.join(PATH_COMMENTS, f"{machine_id}.json") if os.path.exists(path_machine_comments): @@ -75,7 +87,7 @@ def add_comment(): save_comment(comment, ip_address, machine_id) - return jsonify({"response": 200}) + return jsonify({"message": "Success!"}), 200 def process_uploaded_image(image, img_path): @@ -93,15 +105,16 @@ def process_uploaded_image(image, img_path): img = img.resize((basewidth, hsize), Image.Resampling.LANCZOS) img.save(img_path, quality=95) + @app.route("/upload_image", methods=["POST"]) def upload_image(): machine_id = str(request.args.get("id")) ip_address = request.remote_addr if ip_address in blocked_ips: - return jsonify("Blocked IP address") + return jsonify({"error": "User IP address is blocked"}), 403 if "image" not in request.files: - return "No image file", 400 + return jsonify({"error": "No image file found"}), 400 image = request.files["image"] img_path = os.path.join(PATH_IMAGES, f"{machine_id}.jpg") @@ -109,16 +122,16 @@ def upload_image(): # send message to slack image_slack(machine_id, ip=ip_address) - + return "Image uploaded successfully" -def image_slack( - machine_id: int, - ip: str, - m_name: str = None, - img_slack_text: str = "Image uploaded for machine" - ): +def image_slack( + machine_id: int, + ip: str, + m_name: str = None, + img_slack_text: str = "Image uploaded for machine", +): if m_name is None: MACHINE_NAMES = reload_server_data() m_name = MACHINE_NAMES[int(machine_id)] @@ -137,12 +150,12 @@ def image_slack( "title": { "type": "plain_text", "text": "NEW Image!", - "emoji": True + "emoji": True, }, "image_url": f"{IMG_PORT}{machine_id}.jpg", - "alt_text": text + "alt_text": text, } - ] + ], ) except SlackApiError as e: print("Error sending message: ", e) @@ -151,11 +164,12 @@ def image_slack( raise e - def message_slack(machine_id, comment_text, ip: str): MACHINE_NAMES = reload_server_data() m_name = MACHINE_NAMES[int(machine_id)] - text = f"New comment for machine {machine_id} - {m_name}: {comment_text} (from {ip})" + text = ( + f"New comment for machine {machine_id} - {m_name}: {comment_text} (from {ip})" + ) try: response = client.chat_postMessage( channel="#pennyme_uploads", text=text, username="PennyMe" @@ -165,14 +179,14 @@ def message_slack(machine_id, comment_text, ip: str): assert e.response["error"] raise e + def save_comment(comment: str, ip: str, machine_id: int): - # Create dict hierarchy if needed if ip not in IP_COMMENT_DICT.keys(): IP_COMMENT_DICT[ip] = {} if machine_id not in IP_COMMENT_DICT[ip].keys(): IP_COMMENT_DICT[ip][machine_id] = {} - + # Add comment IP_COMMENT_DICT[ip][machine_id][str(datetime.now())] = comment @@ -180,41 +194,96 @@ def save_comment(comment: str, ip: str, machine_id: int): with open("ip_comment_dict.json", "w") as f: json.dump(IP_COMMENT_DICT, f, indent=4) + @app.route("/create_machine", methods=["POST"]) def create_machine(): """ Receives a comment and adds it to the json file """ - machine_title = str(request.args.get("title")) - address = str(request.args.get("address")) - area = str(request.args.get("area")) - location = (float(request.args.get("lon_coord")), float(request.args.get("lat_coord"))) + title = str(request.args.get("title")).strip() + address = str(request.args.get("address")).strip() + area = str(request.args.get("area")).strip() + + # Identify area + area, score = fuzzysearch.extract(area, COUNTRIES, limit=1)[0] + if score < 90: + return ( + jsonify( + { + "error": "Could not match country. Provide country or US state name in English" + } + ), + 400, + ) + + location = ( + float(request.args.get("lon_coord")), + float(request.args.get("lat_coord")), + ) + # Verify that address matches coordinates + queries = [address, address + area, address + title] + for query in queries: + coordinates = gm_client.geocode(query) + try: + lat = coordinates[0]["geometry"]["location"]["lat"] + lng = coordinates[0]["geometry"]["location"]["lng"] + break + except IndexError: + continue + try: + lat, lng + except NameError: + return jsonify({"error": "Google Maps does not know this address"}), 400 + + dist = haversine((lat, lng), (location[1], location[0])) + if dist > 1: # km + return ( + jsonify( + { + "error": f"Address {query} seems >1km away from coordinates ({lat}, {lng})" + } + ), + 400, + ) + + out = gm_client.reverse_geocode( + [location[1], location[0]], result_type="street_address" + ) + + b = True + if out != []: + ad = out[0]["formatted_address"] + _, score = fuzzysearch.extract(ad, [address], limit=1)[0] + if score > 85: + # Prefer Google Maps address over user address + address = ad + b = False + elif b: + out = gm_client.reverse_geocode( + (location[1], location[0]), result_type="point_of_interest" + ) + if out != []: + address = out[0]["formatted_address"] + else: + out = gm_client.reverse_geocode( + (location[1], location[0]), result_type="postal_code" + ) + if out != []: + postal_code = out[0]["formatted_address"].split(" ")[0] + if postal_code not in address: + address += out[0]["formatted_address"] + try: multimachine = int(request.args.get("multimachine")) except ValueError: # just put the multimachine as a string, we need to correct it then multimachine = str(request.args.get("multimachine")) - - paywall = True if request.args.get("paywall") == "true" else False - # set unique branch name - branch_name = f'new_machine_{round(time.time())}' - - # load the current server locations file - with open("../data/server_locations.json", "r") as infile: - server_locations = json.load(infile) - # make new machine ID: one more than any machine in images / server_loc - existing_machines = [ - item["properties"]["id"] for item in server_locations["features"] - ] - potential_new_machines = [ - int(im.split(".")[0]) for im in os.listdir(PATH_IMAGES) if "jpg" in im - ] - new_machine_id = max(existing_machines + potential_new_machines) + 1 + paywall = True if request.args.get("paywall") == "true" else False # put properties into dictionary properties_dict = { - "name": machine_title, + "name": title, "active": True, "area": area, "address": address, @@ -223,7 +292,7 @@ def create_machine(): "internal_url": "null", "latitude": location[1], "longitude": location[0], - "id": new_machine_id, + "id": -1, # to be updated later "last_updated": str(datetime.today()).split(" ")[0], } # add multimachine or paywall only if not defaults @@ -232,20 +301,14 @@ def create_machine(): if paywall: properties_dict["paywall"] = paywall # add new item to json - server_locations["features"].append( - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": location - }, - "properties": - properties_dict - } - ) - - commit_message = f'add new machine {new_machine_id} named {machine_title}' - push_to_github_and_open_pr(server_locations, branch_name, commit_message) + new_machine_entry = { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": location}, + "properties": properties_dict, + } + # If pushing to new branch: set unique branch name + # branch_name = f"new_machine_{round(time.time())}" + new_machine_id = push_to_github(new_machine_entry) # Upload the image if "image" not in request.files: @@ -260,11 +323,12 @@ def create_machine(): image_slack( new_machine_id, ip=ip_address, - m_name=machine_title, - img_slack_text="New machine proposed:" + m_name=title, + img_slack_text="New machine proposed:", ) - - return jsonify({"response": 200}) + + return jsonify({"message": "Success!"}), 200 + def create_app(): return app diff --git a/backend/pennyme/github_update.py b/backend/pennyme/github_update.py new file mode 100644 index 00000000..623f2676 --- /dev/null +++ b/backend/pennyme/github_update.py @@ -0,0 +1,201 @@ +import requests +import base64 +import json +from datetime import datetime +import time +from pennyme.utils import get_next_free_machine_id + +with open("github_token.json", "r") as infile: + github_infos = json.load(infile) +# Define GitHub repository information +GITHUB_TOKEN = github_infos["token"] +REPO_OWNER = github_infos["owner"] +REPO_NAME = github_infos["repo"] +BASE_BRANCH = "main" # Replace with the appropriate base branch name +DATA_BRANCH = "machine_updates" +HEADERS = { + "Authorization": f"token {GITHUB_TOKEN}", + "accept": "application/vnd.github+json", +} +FILE_PATH = "/data/server_locations.json" + + +def check_branch_exists(branch_name): + # Check if the desired branch exists + branch_check_url = ( + f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/branches/{branch_name}" + ) + branch_check_response = requests.get(branch_check_url, headers=HEADERS) + return branch_check_response.status_code == 200 + + +def get_latest_branch_url(): + """ + Check whether the latest change is on the main or on the data branch + Returns the respective URL as a string + Note: We would need to change this function if we want to search for the branch with + the latest commit. + """ + # check if the branch already exists: + branch_exists = check_branch_exists(DATA_BRANCH) + # if data branch exists, the url points to the branch + repo_url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}" + if branch_exists: + file_url = f"{repo_url}/contents/{FILE_PATH}?ref={DATA_BRANCH}" + else: + file_url = f"{repo_url}/contents/{FILE_PATH}" + return file_url + + +def create_new_branch(branch_name): + # create a new branch if it does not exist yet + if not check_branch_exists(branch_name): + payload = { + "ref": f"refs/heads/{branch_name}", + "sha": get_latest_commit_sha(REPO_OWNER, REPO_NAME, BASE_BRANCH), + } + response = requests.post( + f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/git/refs", + headers=HEADERS, + json=payload, + ) + if response.status_code != 201: + print("Failed to create a new branch.") + return False + return True + return False + + +def push_to_github(machine_update_entry, branch_name=DATA_BRANCH): + """ + Push the modified file to the github branch + machine_update_entry: Dict, only the new machine entry that should + be added to the server_locations json + """ + + # Load latest version of the server_locations + file_url = get_latest_branch_url() + response = requests.get(file_url, headers=HEADERS) + data = response.json() + current_content = data["content"] + current_content_decoded = base64.b64decode(current_content).decode("utf-8") + server_locations = json.loads(current_content_decoded) + + # the sha of the last commit is needed later for pushing + latest_commit_sha = data["sha"] + + machine_id = get_next_free_machine_id("../data/all_locations.json", server_locations['features']) + machine_update_entry["properties"]["id"] = machine_id + + # Update the server_locations + server_locations["features"].append(machine_update_entry) + + # create a new branch if necessary + did_create_new_branch = create_new_branch(branch_name) + + # Update the file on the newly created branch + file_content_encoded = base64.b64encode( + json.dumps(server_locations, indent=4, ensure_ascii=False).encode("utf-8") + ).decode("utf-8") + # make commit message + commit_message = f"add new machine {machine_id} named {machine_update_entry['properties']['name']}" + + payload = { + "message": commit_message, + "content": file_content_encoded, + "branch": branch_name, + "sha": latest_commit_sha, + } + response = requests.put( + f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/contents{FILE_PATH}", + headers=HEADERS, + json=payload, + ) + + if response.status_code != 200: + print("Failed to update the file.") + print(response) + return + + # open a new pull request if the branch did not exist + if did_create_new_branch: + open_pull_request(commit_message, branch_name) + + # tell the app.py what was the final machine ID + return machine_id + +def add_pr_label(pr_id, labels): + url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/issues/{pr_id}/labels" + response = requests.post(url, headers=HEADERS, json={"labels": labels}) + if response.status_code == 200: + print("Labels added successfully.") + else: + print("Failed to add labels.") + + +def open_pull_request(commit_message, branch_name): + # Open a pull request + payload = { + "title": commit_message, + "body": "New machine submitted for review", + "head": branch_name, + "base": BASE_BRANCH + } + response = requests.post( + f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/pulls", + headers=HEADERS, + json=payload, + ) + # Add label to PR if it was created successfully + if response.status_code == 201: + pr_id = response.json()["number"] + add_pr_label(pr_id, ['data', 'bot']) + return True + return False + + + +def get_latest_commit_sha(REPO_OWNER, REPO_NAME, branch): + response = requests.get( + f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/git/refs/heads/{branch}" + ) + return response.json()["object"]["sha"] + + +# Example usage +if __name__ == "__main__": + # branch_name = f'new_machine_{round(time.time())}' + + machine_title, address, area, location = ( + "last test", + "1 Mars street, Mars", + "Marsstate", + [1000, 1000], + ) + multimachine = 2 + paywall = True + + # retrieve new ID + new_machine_id = 1 + # design new item to be added to the machine + new_machine_entry = { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": location}, + "properties": { + "name": machine_title, + "active": True, + "area": area, + "address": address, + "status": "unvisited", + "external_url": "null", + "internal_url": "null", + "latitude": location[1], + "longitude": location[0], + "id": new_machine_id, + "last_updated": str(datetime.today()).split(" ")[0], + "multimachine": multimachine, + "paywall": paywall, + }, + } + + push_to_github(new_machine_entry) diff --git a/backend/pennyme/locations.py b/backend/pennyme/locations.py index ec801633..5167d883 100644 --- a/backend/pennyme/locations.py +++ b/backend/pennyme/locations.py @@ -135,3 +135,314 @@ def remove_html_and(x): "Sweeded": 149, "_Collector Books_": 132, } + +COUNTRIES = [ + "Afghanistan", + "Alabama", + "Alaska", + "Albania", + "Algeria", + "American Samoa", + "Andorra", + "Angola", + "Anguilla", + "Antarctica", + "Antigua and Barbuda", + "Argentina", + "Arizona", + "Arkansas", + "Armenia", + "Aruba", + "Australia", + "Austria", + "Azerbaijan", + "Bahamas", + "Bahrain", + "Bangladesh", + "Barbados", + "Belarus", + "Belgium", + "Belize", + "Benin", + "Bermuda", + "Bhutan", + "Bolivia", + "Bonaire", + "Bosnia and Herzegovina", + "Botswana", + "Bouvet Island", + "Brazil", + "Brunei Darussalam", + "Bulgaria", + "Burkina Faso", + "Burundi", + "Cabo Verde", + "California", + "Cambodia", + "Cameroon", + "Canada", + "Cayman Islands", + "Central African Republic", + "Chad", + "Chile", + "China", + "Christmas Island", + "Cocos (Keeling) Islands", + "Colombia", + "Colorado", + "Comoros", + "Congo", + "Congo, The Democratic Republic of the", + "Connecticut", + "Cook Islands", + "Costa Rica", + "Croatia", + "Cuba", + "Curaçao", + "Cyprus", + "Czech", + "Czech Republic", + "Czechia", + "Côte d'Ivoire", + "Delaware", + "Denmark", + "Djibouti", + "Dominica", + "Dominican Republic", + "Ecuador", + "Egypt", + "El Salvador", + "England", + "Equatorial Guinea", + "Eritrea", + "Estonia", + "Eswatini", + "Ethiopia", + "Falkland Islands", + "Faroe Islands", + "Fiji", + "Finland", + "Florida", + "France", + "French Guiana", + "French Polynesia", + "French Southern Territories", + "Gabon", + "Gambia", + "Georgia", + "Germany", + "Ghana", + "Gibraltar", + "Greece", + "Greenland", + "Grenada", + "Guadeloupe", + "Guam", + "Guatemala", + "Guernsey", + "Guinea", + "Guinea-Bissau", + "Guyana", + "Haiti", + "Hawaii", + "Heard Island and McDonald Islands", + "Vatican City State", + "Honduras", + "Hong Kong", + "Hongkong", + "Hungary", + "Iceland", + "Idaho", + "Illinois", + "India", + "Indiana", + "Indonesia", + "Iowa", + "Iran, Islamic Republic of", + "Iraq", + "Ireland", + "Isle of Man", + "Israel", + "Italy", + "Jamaica", + "Japan", + "Jersey", + "Jordan", + "Kansas", + "Kazakhstan", + "Kentucky", + "Kenya", + "Kiribati", + "North Korea", + "Kuwait", + "Kyrgyzstan", + "Laos", + "Latvia", + "Lebanon", + "Lesotho", + "Liberia", + "Libya", + "Liechtenstein", + "Lithuania", + "Lithunia", + "Louisiana", + "Luxembourg", + "Macao", + "Madagascar", + "Maine", + "Malawi", + "Malaysia", + "Maldives", + "Mali", + "Malta", + "Marshall Islands", + "Martinique", + "Maryland", + "Massachusetts", + "Mauritania", + "Mauritius", + "Mayotte", + "Mexico", + "Michigan", + "Micronesia", + "Minnesota", + "Mississippi", + "Missouri", + "Moldova", + "Monaco", + "Mongolia", + "Montana", + "Montenegro", + "Montserrat", + "Morocco", + "Mozambique", + "Myanmar", + "Namibia", + "Nauru", + "Nebraska", + "Nepal", + "Netherlands", + "Nevada", + "New Caledonia", + "New Hampshire", + "New Jersey", + "New Mexico", + "New York", + "New Zealand", + "Nicaragua", + "Niger", + "Nigeria", + "Niue", + "Norfolk Island", + "North Carolina", + "North Dakota", + "North Macedonia", + "Northern Ireland", + "Northern Mariana Islands", + "Norway", + "Ohio", + "Oklahoma", + "Oman", + "Oregon", + "Pakistan", + "Palau", + "Palestine", + "Panama", + "Papua New Guinea", + "Paraguay", + "Pennsylvania", + "Peru", + "Philippines", + "Pitcairn", + "Poland", + "Portugal", + "Private Rollers", + "Puerto Rico", + "Qatar", + "Rhode Island", + "Romania", + "Russia", + "Russian Federation", + "Rwanda", + "Réunion", + "Saint Barthélemy", + "Saint Helena", + "Saint Kitts and Nevis", + "Saint Lucia", + "Saint Martin", + "Saint Pierre and Miquelon", + "Saint Vincent and the Grenadines", + "Samoa", + "San Marino", + "Sao Tome and Principe", + "Saudi Arabia", + "Scotland", + "Senegal", + "Serbia", + "Seychelles", + "Sierra Leone", + "Singapore", + "Sint Maarten", + "Slovakia", + "Slovenia", + "Solomon Islands", + "Somalia", + "South Africa", + "South Carolina", + "South Dakota", + "South Georgia", + "South Korea", + "South Sudan", + "Spain", + "Sri Lanka", + "St Lucia", + "Sudan", + "Suriname", + "Svalbard", + "Sweden", + "Switzerland", + "Syrian Arab Republic", + "Taiwan", + "Tajikistan", + "Tanzaniaf", + "Tennessee", + "Texas", + "Thailand", + "Timor-Leste", + "Togo", + "Tokelau", + "Tonga", + "Trinidad and Tobago", + "Tunisia", + "Turkey", + "Turkmenistan", + "Turks and Caicos Islands", + "Tuvalu", + "UAE", + "Uganda", + "Ukraine", + "United Arab Emirates", + "United Kingdom", + "United States", + "Uruguay", + "Utah", + "Uzbekistan", + "Vanuatu", + "Venezuela", + "Vermont", + "Viet Nam", + "Virgin Islands", + "Virginia", + "Wales", + "Wallis and Futuna", + "Washington", + "Washington DC", + "West Virginia", + "Western Sahara", + "Wisconsin", + "Wyoming", + "Yemen", + "Zambia", + "Zimbabwe", + "Åland Islands", +] diff --git a/backend/pennyme/utils.py b/backend/pennyme/utils.py new file mode 100644 index 00000000..9d3aacda --- /dev/null +++ b/backend/pennyme/utils.py @@ -0,0 +1,31 @@ +import json +from typing import List, Dict +import os + +PATH_IMAGES = os.path.join("..", "..", "images") + + +def get_next_free_machine_id( + all_locations_path: str, server_locations: List[Dict] +) -> int: + """ + Returns the next available machine ID based on all_locations and server_locations + + Args: + all_locations_path (str): Path to all_locations.json + server_locations (List[Dict]): List of read-in server_locations.json content + + Returns: + int: Next ID + """ + with open(all_locations_path, "r") as infile: + all_locations = json.load(infile) + + # Identify IDs in existing data + all_ids = max([i["properties"]["id"] for i in all_locations["features"]]) + server_ids = max([i["properties"]["id"] for i in server_locations]) + + # identify picture IDs + pic_ids = max([int(im.split(".")[0]) for im in os.listdir(PATH_IMAGES) if "jpg" in im]) + + return max(all_ids, server_ids, pic_ids) + 1 diff --git a/backend/pull_request.py b/backend/pull_request.py deleted file mode 100644 index 425df996..00000000 --- a/backend/pull_request.py +++ /dev/null @@ -1,128 +0,0 @@ -import requests -import base64 -import json -from datetime import datetime -import time - - -def push_to_github_and_open_pr(file_content, branch_name, commit_message): - with open("github_token.json", "r") as infile: - github_infos = json.load(infile) - # Define GitHub repository information - github_token = github_infos["token"] - repo_owner = github_infos["owner"] - repo_name = github_infos["repo"] - base_branch = 'main' # Replace with the appropriate base branch name - - # Create a new branch for the changes - headers = { - 'Authorization': f'token {github_token}', - "accept": 'application/vnd.github+json' - } - payload = { - 'ref': f'refs/heads/{branch_name}', - 'sha': get_latest_commit_sha(repo_owner, repo_name, base_branch), - } - response = requests.post( - f'https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs', - headers=headers, - json=payload - ) - - if response.status_code != 201: - print('Failed to create a new branch.') - return - - # Update the file on the newly created branch - file_path = '/data/server_locations.json' - - file_content_encoded = base64.b64encode( - json.dumps(file_content, indent=4, ensure_ascii=False).encode('utf-8') - ).decode('utf-8') - - url = 'https://api.github.com/repos/jannisborn/PennyMe/contents/data/server_locations.json' - response_sha = requests.get(url).json()["sha"] - - payload = { - 'message': commit_message, - 'content': file_content_encoded, - 'branch': branch_name, - 'sha': response_sha - } - response = requests.put( - f'https://api.github.com/repos/{repo_owner}/{repo_name}/contents{file_path}', - headers=headers, - json=payload - ) - - if response.status_code != 200: - print('Failed to update the file.') - return - - # Open a pull request - payload = { - 'title': commit_message, - 'body': 'New machine submitted for review', - 'head': branch_name, - 'base': base_branch, - } - response = requests.post( - f'https://api.github.com/repos/{repo_owner}/{repo_name}/pulls', - headers=headers, - json=payload - ) - - -def get_latest_commit_sha(repo_owner, repo_name, branch): - response = requests.get( - f'https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}' - ) - return response.json()['object']['sha'] - - -# Example usage -if __name__ == '__main__': - branch_name = f'new_machine_{round(time.time())}' - - machine_title, address, area, location = ( - "Machine on Mars", "1 Mars street, Mars", "Marsstate", [1000, 1000] - ) - multimachine = 2 - paywall = True - - # load the current server locations file - with open("../data/server_locations.json", "r") as infile: - server_locations = json.load(infile) - # retrieve new ID - new_machine_id = max( - [item["properties"]["id"] for item in server_locations["features"]] - ) + 1 - # add new item to json - server_locations["features"].append( - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": location - }, - "properties": - { - "name": machine_title, - "active": True, - "area": area, - "address": address, - "status": "unvisited", - "external_url": "null", - "internal_url": "null", - "latitude": location[1], - "longitude": location[0], - "id": new_machine_id, - "last_updated": str(datetime.today()).split(" ")[0], - "multimachine": multimachine, - "paywall": paywall - } - } - ) - - commit_message = f'add new machine {new_machine_id} named {machine_title}' - push_to_github_and_open_pr(server_locations, branch_name, commit_message) diff --git a/backend/requirements.txt b/backend/requirements.txt index ec6e7699..fe80030d 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -4,4 +4,6 @@ googlemaps tqdm flask slack -Pillow \ No newline at end of file +Pillow +thefuzz +haversine \ No newline at end of file