Skip to content

Commit

Permalink
Rewrite has_data() to check combined and uncombined
Browse files Browse the repository at this point in the history
  • Loading branch information
yngve-sk committed Oct 23, 2024
1 parent be21e08 commit 55fa828
Showing 1 changed file with 28 additions and 1 deletion.
29 changes: 28 additions & 1 deletion src/ert/storage/local_ensemble.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import logging
import os
from datetime import datetime
from functools import lru_cache
from functools import lru_cache, reduce
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
from uuid import UUID
Expand Down Expand Up @@ -359,6 +359,33 @@ def has_data(self) -> List[int]:
exists : List[int]
Returns the realization numbers with responses
"""

# First check for combined
realizations_with_responses_combined = {
response: set(
polars.read_parquet(self._path / f"{response}.parquet")[
"realization"
].unique()
)
if (self._path / f"{response}.parquet").exists()
else {}
for response in self.experiment.response_configuration
}

reals_with_all_responses_from_combined = reduce(
set.intersection, realizations_with_responses_combined.values()

Check failure on line 376 in src/ert/storage/local_ensemble.py

View workflow job for this annotation

GitHub Actions / type-checking (3.12)

Argument 1 to "reduce" has incompatible type "Callable[[set[_T], VarArg(Iterable[Any])], set[_T]]"; expected "Callable[[Collection[Any], Collection[Any]], Collection[Any]]"
)

realizations_missing_responses = reals_with_all_responses_from_combined - set(

Check failure on line 379 in src/ert/storage/local_ensemble.py

View workflow job for this annotation

GitHub Actions / type-checking (3.12)

Unsupported left operand type for - ("Collection[Any]")
range(self.ensemble_size)
)

if not realizations_missing_responses:
return list(reals_with_all_responses_from_combined)

# If not combined, fallback to checking individual folders
# We assume that there is no state where a combine was invoked before
# all responses were saved
return [
i
for i in range(self.ensemble_size)
Expand Down

0 comments on commit 55fa828

Please sign in to comment.