diff --git a/functions/dataframe_utils.py b/functions/dataframe_utils.py index adfaa68..3cbd801 100644 --- a/functions/dataframe_utils.py +++ b/functions/dataframe_utils.py @@ -11,7 +11,7 @@ async def remove_expired_listings(df: pd.DataFrame, limiter: AsyncLimiter) -> pd.DataFrame: """ Asynchronously checks each listing URL in the DataFrame to determine if it has expired, - and removes rows with expired listings, applying rate limiting. + and removes rows with expired listings, applying rate limiting. Also counts the number of expired listings removed. Parameters: df (pd.DataFrame): The DataFrame containing listing URLs and MLS numbers. @@ -32,11 +32,16 @@ async def check_and_mark_expired(row): # Determine indexes of rows to drop (where listing has expired) indexes_to_drop = [index for index, expired in results if expired] - # Log success messages for dropped listings + # Counter for expired listings + expired_count = len(indexes_to_drop) + + # Log success messages for dropped listings and the count of expired listings for index in indexes_to_drop: mls_number = df.loc[index, 'mls_number'] logger.success(f"Removed {mls_number} (Index: {index}) from the dataframe because the listing has expired.") + logger.info(f"Total expired listings removed: {expired_count}") + # Drop the rows from the DataFrame and return the modified DataFrame df_dropped_expired = df.drop(indexes_to_drop) return df_dropped_expired \ No newline at end of file