Skip to content

Commit

Permalink
fixed issue 266
Browse files Browse the repository at this point in the history
  • Loading branch information
m-kovalsky committed Nov 11, 2024
1 parent 9b4f2f1 commit b459e2e
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 22 deletions.
2 changes: 1 addition & 1 deletion src/sempy_labs/_helper_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -1144,7 +1144,7 @@ def _get_max_run_id(lakehouse: str, table_name: str) -> int:
spark = SparkSession.builder.getOrCreate()
query = f"SELECT MAX(RunId) FROM {lakehouse}.{table_name}"
dfSpark = spark.sql(query)
max_run_id = dfSpark.collect()[0][0]
max_run_id = dfSpark.collect()[0][0] or 0

return max_run_id

Expand Down
44 changes: 23 additions & 21 deletions src/sempy_labs/_model_bpa_bulk.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ def run_model_bpa_bulk(
)
lakeT = get_lakehouse_tables(lakehouse=lakehouse, workspace=lakehouse_workspace)
lakeT_filt = lakeT[lakeT["Table Name"] == output_table]
# query = f"SELECT MAX(RunId) FROM {lakehouse}.{output_table}"
if len(lakeT_filt) == 0:
runId = 1
else:
Expand Down Expand Up @@ -151,28 +150,31 @@ def run_model_bpa_bulk(
)
print(e)

df["Severity"].replace(icons.severity_mapping)
if len(df) == 0:
print(f"{icons.yellow_dot} No BPA results to save for the '{wksp}' workspace.")
else:
df["Severity"].replace(icons.severity_mapping)

# Append save results individually for each workspace (so as not to create a giant dataframe)
print(
f"{icons.in_progress} Saving the Model BPA results of the '{wksp}' workspace to the '{output_table}' within the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace..."
)
# Append save results individually for each workspace (so as not to create a giant dataframe)
print(
f"{icons.in_progress} Saving the Model BPA results of the '{wksp}' workspace to the '{output_table}' within the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace..."
)

schema = {
key.replace(" ", "_"): value
for key, value in icons.bpa_schema.items()
}

save_as_delta_table(
dataframe=df,
delta_table_name=output_table,
write_mode="append",
schema=schema,
merge_schema=True,
)
print(
f"{icons.green_dot} Saved BPA results to the '{output_table}' delta table."
)
schema = {
key.replace(" ", "_"): value
for key, value in icons.bpa_schema.items()
}

save_as_delta_table(
dataframe=df,
delta_table_name=output_table,
write_mode="append",
schema=schema,
merge_schema=True,
)
print(
f"{icons.green_dot} Saved BPA results to the '{output_table}' delta table."
)

print(f"{icons.green_dot} Bulk BPA scan complete.")

Expand Down

0 comments on commit b459e2e

Please sign in to comment.