Skip to content

Commit

Permalink
Location differ - Supporting duplicate links in all locations (#358)
Browse files Browse the repository at this point in the history 
* feat: Allow duplicate URLs in all locations

* chore: ruffing

* ci: remove ruffing

* chore: make logging slightly less verbose

* wip
  • Loading branch information
@jannisborn
jannisborn authored Oct 22, 2024
1 parent 829c31a commit be6ff76
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 74 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/formatting.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ jobs:
- name: Check code formatting with black
run: black --check .

- name: Check code formatting with ruff
run: ruff format --check .

- name: Check sanity of json files
run: python .hooks/check_json.py

148 changes: 77 additions & 71 deletions backend/scripts/location_differ.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,7 @@ def location_differ(
elif url not in device_dict.keys():
device_dict[url] = [geojson]
else:
raise ValueError(
f"Duplicate entry in all_locations must not occur (problem={url}) - {geojson}"
)
device_dict[url].append(geojson)

server_dict = {}
for i, geojson in enumerate(server_data["features"]):
Expand Down Expand Up @@ -266,18 +264,25 @@ def location_differ(
break

if cur_state == "available" and this_state in UNAVAILABLE_STATES:
logger.info(f"{this_title} is currently unavailable")
logger.info(
f"{this_title} (in {area}) is currently unavailable"
)
# Machine is currently unavailable, update this in server dict
if name == "Device":
# Easy case, we just add this machine to server_dict
assert len(device_dict[this_link]) == 1
if len(device_dict[this_link]) > 1:
logger.warning(
f"A url linking to multiple machines retired, maybe check manually: {device_dict[this_link]}"
)
# Retire machine
entry = device_dict[this_link][0]
entry["properties"]["machine_status"] = UNAVAILABLE_MAPPER[
this_state
]
entry["properties"]["last_updated"] = today
server_data["features"].append(entry)
for entry in device_dict[this_link]:
entry["properties"]["machine_status"] = (
UNAVAILABLE_MAPPER[this_state]
)
entry["properties"]["last_updated"] = today
server_data["features"].append(entry)
changes += 1
depr += 1
elif name == "Server":
if len(server_dict[this_link]) > 1:
logger.warning(
Expand All @@ -298,8 +303,8 @@ def location_differ(
server_data["features"][idx]["properties"][
"last_updated"
] = today
changes += 1 # track that we changed this machine
depr += 1
changes += 1 # track that we changed this machine
depr += 1
match = True # machine was found in existing dict
break # to not change a machine found in both dicts twice

Expand All @@ -311,25 +316,26 @@ def location_differ(
# A machine documented as retired is available again
if name == "Device":
# Easy case, we just add this machine to server_dict
assert len(device_dict[this_link]) == 1
entry = device_dict[this_link][0]
entry["properties"]["machine_status"] = "available"
entry["properties"]["last_updated"] = today
entry["properties"]["name"] = entry["properties"][
"name"
].strip()
entry["properties"]["address"] = entry["properties"][
"address"
].strip()
server_data["features"].append(entry)
if len(device_dict[this_link]) > 1:
logger.warning(
f"A url linking to multiple machines got available again, maybe check manually: {device_dict[this_link]}"
)

for entry in device_dict[this_link]:
entry["properties"]["machine_status"] = "available"
entry["properties"]["last_updated"] = today
entry["properties"]["name"] = entry["properties"][
"name"
].strip()
entry["properties"]["address"] = entry["properties"][
"address"
].strip()
server_data["features"].append(entry)
changes += 1
new += 1
elif name == "Server":
# Machine is already documented in server_dict
entry = server_dict[this_link]
if len(entry) > 1:
logger.warning(
"A url linking to multiple machines retired, "
f"maybe check manually: {entry}"
)

# Extract all machines of that URL (usually 1)
idxs = [
Expand All @@ -339,7 +345,7 @@ def location_differ(
]
if len(idxs) > 1:
logger.warning(
f"For {this_link} found {len(idxs)} machines: {idxs}"
f"For {this_link} found {len(idxs)} machines: {idxs} which got available again"
)
# Re-activate all machines of that URL
for idx in idxs:
Expand All @@ -350,8 +356,8 @@ def location_differ(
"last_updated"
] = today

changes += 1 # track that we changed this machine
new += 1
changes += 1
new += 1
match = True # machine was found in existing dict
break # to not change a machine found in both dicts twice

Expand All @@ -363,26 +369,28 @@ def location_differ(
logger.info(f"{this_title} is only temporarily unavailable")
if name == "Device":
# Easy case, we just add this machine to server_dict
assert len(device_dict[this_link]) == 1
entry = device_dict[this_link][0]
entry["properties"]["machine_status"] = "out-of-order"
entry["properties"]["last_updated"] = today
entry["properties"]["name"] = entry["properties"][
"name"
].strip()
entry["properties"]["address"] = entry["properties"][
"address"
].strip()
server_data["features"].append(entry)
if len(device_dict[this_link]) > 1:
logger.warning(
f"A url linking to multiple machines is temporarily unavailable not retired, maybe check manually: {device_dict[this_link]}"
)
for entry in device_dict[this_link]:
entry["properties"]["machine_status"] = "out-of-order"
entry["properties"]["last_updated"] = today
entry["properties"]["name"] = entry["properties"][
"name"
].strip()
entry["properties"]["address"] = entry["properties"][
"address"
].strip()
server_data["features"].append(entry)
changes += 1
elif name == "Server":
# Machine is already documented in server_dict
entry = server_dict[this_link]
if len(entry) > 1:
logger.warning(
"A url linking to multiple machines retired, "
f"maybe check manually: {entry}"
f"A url linking to multiple machines is temporarily unavailable not retired, maybe check manually: {entry}"
)

# Extract all machines of that URL (usually 1)
idxs = [
i
Expand All @@ -401,8 +409,8 @@ def location_differ(
server_data["features"][idx]["properties"][
"last_updated"
] = today
changes += 1

changes += 1 # track that we changed this machine
match = True # machine was found in existing dict
break # to not change a machine found in both dicts twice

Expand All @@ -411,26 +419,30 @@ def location_differ(
logger.info(f"{this_title} got permanently removed")
if name == "Device":
# Easy case, we just add this machine to server_dict
assert len(device_dict[this_link]) == 1
entry = device_dict[this_link][0]
entry["properties"]["machine_status"] = UNAVAILABLE_MAPPER[
this_state
]
entry["properties"]["last_updated"] = today
entry["properties"]["name"] = entry["properties"][
"name"
].strip()
entry["properties"]["address"] = entry["properties"][
"address"
].strip()
server_data["features"].append(entry)
if len(device_dict[this_link]) > 1:
logger.warning(
f"A url linking to multiple machines got removed, maybe check manually: {device_dict[this_link]}"
)

for entry in device_dict[this_link]:
entry["properties"]["machine_status"] = (
UNAVAILABLE_MAPPER[this_state]
)
entry["properties"]["last_updated"] = today
entry["properties"]["name"] = entry["properties"][
"name"
].strip()
entry["properties"]["address"] = entry["properties"][
"address"
].strip()
server_data["features"].append(entry)
changes += 1
elif name == "Server":
# Machine is already documented in server_dict
entry = server_dict[this_link]
if len(entry) > 1:
logger.warning(
"A url linking to multiple machines retired, "
f"maybe check manually: {entry}"
f"A url linking to multiple machines got removed, maybe check manually: {entry}"
)

# Extract all machines of that URL (usually 1)
Expand All @@ -451,8 +463,7 @@ def location_differ(
server_data["features"][idx]["properties"][
"last_updated"
] = today

changes += 1 # track that we changed this machine
changes += 1
match = True # machine was found in existing dict
break # to not change a machine found in both dicts twice
else:
Expand Down Expand Up @@ -650,18 +661,13 @@ def location_differ(
problem_data["features"].append(prelim_to_problem_json(geojson, msg))
else:
logger.info(
f"{j}/{length}: Found machine to be added: {geojson['properties']['name']}"
f"{j}/{length}: Found machine to be added: {geojson['properties']['name']} in {area}"
)
changes += 1
new += 1
machine_idx += 1
server_data["features"].append(geojson)
if changes > 0:
logger.info(
f"Location {area} ({i}/{len(areas)}): Changes in {changes}/"
f"{len(location_raw_list)} machines found."
)
total_changes += changes
total_changes += changes

logger.info(
f"\n Result: {total_changes} changes, {new} new machines found"
Expand Down

0 comments on commit be6ff76

Please sign in to comment.