Skip to content

Commit

Permalink
Core & Internals: fix bulk rse loading
Browse files Browse the repository at this point in the history
- settings_by_id will not have any entry for RSEs which don't have
attributes from RSE_SETTINGS set. This resulted in an attribute error.

- name loading was getting AttributeError when some RSEs already had
their name loaded.

- remove code for non-temp-table workflow
  • Loading branch information
Radu Carpa committed Nov 24, 2023
1 parent 289e964 commit 212f73b
Showing 1 changed file with 5 additions and 6 deletions.
11 changes: 5 additions & 6 deletions lib/rucio/core/rse.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

from rucio.common import exception, utils
from rucio.common.cache import make_region_memcached
from rucio.common.config import get_lfn2pfn_algorithm_default, config_get_bool
from rucio.common.config import get_lfn2pfn_algorithm_default
from rucio.common import types
from rucio.common.utils import CHECKSUM_KEY, GLOBALLY_SUPPORTED_CHECKSUMS, Availability
from rucio.core.rse_counter import add_counter, get_counter
Expand Down Expand Up @@ -159,8 +159,7 @@ def bulk_load(rse_id_to_data: "dict[str, RseData]", load_name=False, load_column
Given a dict of RseData objects indexed by rse_id, ensure that the desired fields are initialised
in all objects from the input.
"""
use_temp_tables = config_get_bool('core', 'use_temp_tables', default=True, session=session)
if not use_temp_tables or len(rse_id_to_data) < 4:
if len(rse_id_to_data) < 4: # 4 was selected without particular reason as "seems good enough"
for rse_data in rse_id_to_data.values():
rse_data.ensure_loaded(
load_name=load_name,
Expand Down Expand Up @@ -224,7 +223,7 @@ def bulk_load(rse_id_to_data: "dict[str, RseData]", load_name=False, load_column
session=session))
for rse_id, db_rse in db_rses_by_id.items():
rse_data = rse_id_to_data[rse_id]
settings = rse_data._attributes if rse_data._attributes is not None else settings_by_id[rse_id]
settings = rse_data._attributes if rse_data._attributes is not None else settings_by_id.get(rse_id, {})
columns = _format_get_rse(db_rse=db_rse, rse_attributes=settings, session=session)
rse_data._columns = columns
rse_data._name = columns['rse']
Expand Down Expand Up @@ -281,8 +280,8 @@ def bulk_load(rse_id_to_data: "dict[str, RseData]", load_name=False, load_column
if load_name:
# The name could have been loaded already (when loading columns or info). Skip loading if it's known.
if not load_columns and not load_info:
for rse_id, rse_data in rse_id_to_data.items():
rse_data._name = db_rses_by_id[rse_id].rse
for rse_id in rse_ids_to_load:
rse_id_to_data[rse_id]._name = db_rses_by_id[rse_id].rse


class RseCollection(Generic[T]):
Expand Down

0 comments on commit 212f73b

Please sign in to comment.