Migration workflow

Migration workflow steps

v0.15.0 --> v0.16.0

DB

create new tables

poetry run invenio db create

Mapping

Elasticsearch

poetry run invenio rero es index update-mapping

create ES mapping

poetry run invenio index create  -b rero_mef/places/gnd/mappings/v7/places_gnd/gnd-place-v0.0.1.json 'places_gnd-gnd-place-v0.0.1-20240808'
poetry run invenio index create  -b rero_mef/concepts/gnd/mappings/v7/concepts_gnd/gnd-concept-v0.0.1.json 'concepts_gnd-gnd-concept-v0.0.1-20240808'

aliases

from invenio_search import current_search_client
current_search_client.indices.put_alias('places_gnd-gnd-place-v0.0.1-20240808', 'places_gnd')
current_search_client.indices.put_alias('places_gnd-gnd-place-v0.0.1-20240808', 'places_gnd-gnd-place-v0.0.1')
current_search_client.indices.put_alias('concepts_gnd-gnd-concept-v0.0.1-20240808', 'concepts_gnd')
current_search_client.indices.put_alias('concepts_gnd-gnd-concept-v0.0.1-20240808', 'concepts_gnd-gnd-concept-v0.0.1')

move indexes

index_name=`poetry run invenio rero es index info -i concepts_idref`
echo $index_name
poetry run invenio rero es index move concepts_idref $index_name concepts_idref-idref_concept-v1.0.0-20240808 -v
poetry run invenio index delete $index_name

index_name=`poetry run invenio rero es index info -i concepts_rero`
echo $index_name
poetry run invenio rero es index move concepts_rero $index_name concepts_rero-rero_concept-v1.0.0-20240808 -v
poetry run invenio index delete $index_name

index_name=`poetry run invenio rero es index info -i concepts_mef`
echo $index_name
poetry run invenio rero es index move concepts_mef $index_name concepts_mef-mef_concept-v1.0.0-20240808 -v
poetry run invenio index delete $index_name

index_name=`poetry run invenio rero es index info -i places_idref`
echo $index_name
poetry run invenio rero es index move places_idref $index_name places_idref-idref_place-v1.0.0-20240808 -v
poetry run invenio index delete $index_name

test mapping

poetry run invenio rero es index update-mapping

Database

poetry run invenio alembic upgrade

OAI Harvesting

poetry run invenio oaiharvester initconfig ./data/oaisources.yml -u
poetry run invenio oaiharvester harvestname -n concepts.idref -o -5 -f 1990-01-01
poetry run invenio oaiharvester harvestname -n concepts.gnd -o -5 -f 1990-01-01
poetry run invenio oaiharvester harvestname -n places.idref -o -5 -f 1990-01-01
poetry run invenio oaiharvester harvestname -n places.gnd -o -5 -f 1990-01-01

poetry run invenio oaiharvester set-last-run -n agents.idref -d 2024-10-01
poetry run invenio oaiharvester set-last-run -n agents.gnd -d 2024-10-01
poetry run invenio oaiharvester set-last-run -n concepts.idref -d 2024-10-01
poetry run invenio oaiharvester set-last-run -n concepts.gnd -d 2024-10-01
poetry run invenio oaiharvester set-last-run -n places.idref -d 2024-10-01
poetry run invenio oaiharvester set-last-run -n places.gnd -d 2024-10-01

v0.14.0 --> v0.15.0

Mapping

Elasticsearch

poetry run invenio rero es index update-mapping

Database

poetry run invenio alembic upgrade

# on `gelatine` (`eponine` test) clean WAL
ls -ltr /database/archive/
pg_archivecleanup /database/archive/`last file from archive`

v0.12.0 --> v0.14.0

create ES mapping

poetry run invenio index create  -b rero_mef/places/mef/mappings/v7/places_mef/mef-place-v0.0.1.json 'places_mef-mef-place-v0.0.1-20230823'
poetry run invenio index create  -b rero_mef/places/idref/mappings/v7/places_idref/idref-place-v0.0.1.json 'places_idref-idref-place-v0.0.1-20230823'

aliases

from invenio_search import current_search_client
current_search_client.indices.put_alias('places_mef-mef-place-v0.0.1-20230823', 'places_mef')
current_search_client.indices.put_alias('places_mef-mef-place-v0.0.1-20230823', 'places_mef-mef-place-v0.0.1')
current_search_client.indices.put_alias('places_idref-idref-place-v0.0.1-20230823', 'places_idref')
current_search_client.indices.put_alias('places_idref-idref-place-v0.0.1-20230823', 'places_idref-idref-place-v0.0.1')

test mapping

poetry run invenio rero es index update-mapping

create DB tables

poetry run invenio db create

OAIHarvesterConfig

poetry run invenio oaiharvester initconfig ./data/oaisources.yml

populate places

poetry run invenio oaiharvester harvestname -n places.idref -o -5 -f 1990-01-01

populte timelaps

poetry run invenio oaiharvester harvestname -n concepts.idref -o -5 -f 1990-01-01

create new kubernets CronJob for places.

v0.11.0 --> v0.12.0

update ES mapping

poetry run invenio rero es index update-mapping

name=concepts_idref
index_name=`poetry run invenio rero es index info -i ${name}`
echo $index_name
poetry run invenio rero es index move ${name} $index_name concepts_idref-idref-concept-v0.0.1-20230725 -v
poetry run invenio index delete $index_name

name=concepts_rero
index_name=`poetry run invenio rero es index info -i ${name}`
echo $index_name
poetry run invenio rero es index move ${name} $index_name concepts_rero-rero-concept-v0.0.1-20230725 -v
poetry run invenio index delete $index_name

name=concepts_mef
index_name=`poetry run invenio rero es index info -i ${name}`
echo $index_name
poetry run invenio rero es index move ${name} $index_name concepts_mef-mef-concept-v0.0.1-20230725 -v
poetry run invenio index delete $index_name

name=viaf
index_name=`poetry run invenio rero es index info -i ${name}`
echo $index_name
poetry run invenio rero es index move ${name} $index_name viaf-viaf-v0.0.1-20230725 -v
poetry run invenio index delete $index_name


poetry run invenio rero es index update-mapping

run alembic migration scripts :

poetry run invenio alembic upgrade