Skip to content

Commit

Permalink
dependencies: fix security issues
Browse files Browse the repository at this point in the history
* Bumps dependencies.
* Better verbose prints.
* Cleans code from warnings.

Co-Authored-by: Peter Weber <[email protected]>
  • Loading branch information
rerowep committed Jan 7, 2024
1 parent 4a75ca7 commit 91aa163
Show file tree
Hide file tree
Showing 14 changed files with 626 additions and 617 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/continuous-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ jobs:
- uses: actions/checkout@v3

- name: Setup node
uses: actions/setup-node@v3
uses: actions/setup-node@v4
with:
node-version: '14'
node-version: '16'

- name: Docker compose up
run: docker-compose up -d
Expand Down
4 changes: 2 additions & 2 deletions classes.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ class ReroMefRecord{
dbcommit(self, reindex=False, forceindex=False)
reindex(self, forceindex=False)
get_record_by_pid(cls, pid, with_deleted=False)
get_pid_by_id(cls, id)
get_persistent_identifier(cls, id)
get_pid_by_id(cls, id_)
get_persistent_identifier(cls, id_)
_get_all(cls, with_deleted=False)
get_all_pids(cls, with_deleted=False, limit=100000)
get_all_ids(cls, with_deleted=False, limit=100000)
Expand Down
1,081 changes: 538 additions & 543 deletions poetry.lock

Large diffs are not rendered by default.

42 changes: 23 additions & 19 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,7 @@ python = ">= 3.9, <3.10"
## foo = ">3.0" # finally will upgrade to 5.0
## when the python code will be executed a conflict exception will be raised.
#------------------------------------------------------------------------------
# Default from Invenio
lxml = ">=4.9.1"
marshmallow = ">=3.0.0,<4.0.0"
uwsgi = ">=2.0"
uwsgi-tools = ">=1.1.1"
uwsgitop = ">=0.11"

## Third party inven## Third party invenio modules used by RERO EBOOKS
invenio-oaiharvester = { git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4" }
invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
Expand All @@ -38,32 +33,41 @@ invenio-cache = ">=1.1.1,<1.2.0"
invenio-celery = ">=1.2.4,<1.3.0"
invenio-config = ">=1.0.3,<1.1.0"
invenio-i18n = ">=2.0.0,<3.0.0"
invenio-db = {version = ">=1.0.14,<1.1.0", extras = ["postgresql"]}
invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
# Invenio base bundle
invenio-admin = ">=1.4.0,<1.5.0"
invenio-assets = ">=2.0.0,<3.0.0"
invenio-formatter = ">=1.2.0,<1.3.0"
invenio-logging = {version = ">=1.3.2,<1.4.0", extras = ["sentry-sdk"]}
invenio-mail = ">=1.0.2,<1.1.0"
invenio-rest = ">=1.2.8,<1.3.0"
invenio-formatter = ">=2.0.0,<3.0.0"
invenio-logging = {version = ">=2.0.0,<3.0.0"}
invenio-mail = ">=2.0.0,<3.0.0"
invenio-rest = ">=1.3.0,<1.4.0"
invenio-theme = ">=2.0.0,<3.0.0"
# Invenio auth bundle
invenio-access = ">=1.4.4,<1.5.0"
invenio-accounts = ">=2.1.0,<2.2.0"
invenio-oauth2server = ">=2.0.0,<2.1.0"
invenio-oauthclient = ">=2.2.0,<3.0.0"
invenio-userprofiles = ">=2.2.0,<2.3.0"
invenio-access = ">=2.0.0,<3.0.0"
invenio-accounts = ">=3.0.0,<4.0.0"
invenio-oauth2server = ">=2.0.0,<3.0.0"
invenio-oauthclient = ">=3.0.0,<4.0.0"
invenio-userprofiles = ">=2.3.0,<3.0.0"
# Invenio metadata bundle
invenio-indexer = ">=2.1.0,<2.2.0"
invenio-indexer = ">=2.2.0,<3.0.0"
invenio-jsonschemas = ">=1.1.4,<1.2.0"
invenio-oaiserver = ">=2.2.0,<2.3.0"
invenio-pidstore = ">=1.3.0,<1.4.0"
invenio-records-rest = ">=2.2.0,<2.3.0"
invenio-records-ui = ">=1.2.0,<1.3.0"
invenio-records = "2.1.0,<2.2.0"
invenio-records = ">=2.1.0,<2.2.0"
#invenio-search-ui = ">=2.4.0,<3.0.0"
# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request

# Default from Invenio
lxml = ">=4.9.1"
marshmallow = ">=3.0.0,<4.0.0"
uwsgi = ">=2.0"
uwsgi-tools = ">=1.1.1"
uwsgitop = ">=0.11"
# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
Flask = ">=2.2.0,<2.3.0"
sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
dojson = ">=1.4.0"
# TODO: dojson problem = AttributeError: 'Group' object has no attribute 'resultcallback'
click = "<8.1.0"
Expand All @@ -80,7 +84,7 @@ pydocstyle = ">=6.1.1"
## RERO ILS specific python modules
poethepoet = "*"
# TODO: solve requires invenio-indexer (<2.0.0)
rero-invenio-base = ">=0.2.1"
rero-invenio-base = ">=0.2.0"
sqlitedict = "^2.1.0"
werkzeug = "<2.3.0"
sqlalchemy-continuum = ">=1.3.12,<1.4.0"
Expand Down
20 changes: 11 additions & 9 deletions rero_mef/agents/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def create_from_viaf(enqueue, online, verbose, online_verbose,
"""Create MEF and agents from viaf."""
def get_pids_from_json(json_file):
"""Get all pids from JSON file."""
for record in read_json_record(viaf_file):
for record in read_json_record(json_file):
yield record['pid']

click.secho(
Expand All @@ -69,26 +69,28 @@ def get_pids_from_json(json_file):

if missing:
missing_pids, non_existing_pids = AgentMefRecord. \
get_all_missing_viaf_pids(
verbose=progress or verbose
)
get_all_missing_viaf_pids(verbose=(progress or verbose))
progress_bar = progressbar(
items=missing_pids,
length=len(missing_pids),
verbose=progress
verbose=progress,
label='VIAF missing'
)
elif viaf_file:
progress_bar = progressbar(
items=get_pids_from_json(viaf_file),
length=number_records_in_file(viaf_file.name, 'json'),
verbose=progress
verbose=progress,
label='VIAF file'
)
else:
progress_bar = progressbar(
items=AgentViafRecord.get_all_pids(),
length=counts['viaf']['old'],
verbose=progress
verbose=progress,
label='VIAF all'
)
click.echo('Create MEF and agents from VIAF')
for pid in progress_bar:
if enqueue:
task = task_create_mef_and_agents_from_viaf.delay(
Expand All @@ -111,7 +113,7 @@ def get_pids_from_json(json_file):
if non_existing_pids:
click.echo(
f'Clean VIAF pids from MEF records: {len(non_existing_pids)}')
for pid, viaf_pid in non_existing_pids.items():
for pid, _ in non_existing_pids.items():
# TODO: clean MEF records with non existing VIAF pids:
pass

Expand Down Expand Up @@ -185,7 +187,7 @@ def create_csv_mef(viaf_metadata_file, output_directory, verbose):
click.secho(
f' VIAF input file: {viaf_metadata_file}',
err=True)
message = f' CSV output files: {pidstore}, {metadata}'
# message = f' CSV output files: {pidstore}, {metadata}'

count = create_mef_files(
viaf_metadata_file_name=viaf_metadata_file,
Expand Down
3 changes: 2 additions & 1 deletion rero_mef/agents/gnd/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def gnd_get_record(id_, debug=False):
if records := parse_xml_to_array(BytesIO(response.content)):
trans_record = Transformation(records[0]).json
pid = trans_record.get('pid')
if id_ != trans_record.get('pid'):
if id_ != pid:
msg = f'{msg} | PID changed: {id_} -> {pid}'
trans_record = None
else:
Expand All @@ -119,6 +119,7 @@ def gnd_get_record(id_, debug=False):
else:
msg = f'{msg} | HTTP Error: {status_code}'
except Exception as err:
trans_record = None
msg = f'{msg} | Error: {err}'
if debug:
raise
Expand Down
6 changes: 4 additions & 2 deletions rero_mef/agents/mef/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ def get_all_missing_viaf_pids(cls, verbose=False):
progress = progressbar(
items=AgentViafRecord.get_all_pids(),
length=AgentViafRecord.count(),
verbose=verbose
verbose=verbose,
label='VIAF all'
)
missing_pids = {pid: 1 for pid in progress}
if verbose:
Expand All @@ -92,7 +93,8 @@ def get_all_missing_viaf_pids(cls, verbose=False):
progress = progressbar(
items=query.source(['pid', 'viaf_pid']).scan(),
length=query.count(),
verbose=True
verbose=verbose,
label='VIAF from MEF'
)
non_existing_pids = {hit.pid: hit.viaf_pid for hit in progress
if not missing_pids.pop(hit.viaf_pid, None)}
Expand Down
17 changes: 7 additions & 10 deletions rero_mef/agents/viaf/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@
from .. import AgentGndRecord, AgentIdrefRecord, AgentMefRecord, \
AgentReroRecord
from ..api import Action, ReroIndexer, ReroMefRecord
from ..mef.api import AgentMefRecord
from ..utils import get_entity_class
from ...filter import exists_filter
from ...utils import add_md5, get_entity_class, progressbar, \
requests_retry_session
Expand Down Expand Up @@ -209,7 +207,7 @@ def update_online(agent_class, pid, online):
if agent_class.provider.pid_type in online:
data, msg = agent_class.get_online_record(id_=pid)
if online_verbose:
click.echo(msg)
click.echo(f'\n{msg}')
if data and not data.get('NO TRANSFORMATION'):
agent_record, action = agent_class.create_or_update(
data=data, dbcommit=dbcommit, reindex=reindex)
Expand Down Expand Up @@ -320,22 +318,22 @@ def reindex(self, forceindex=False):
return result

@classmethod
def get_online_record(cls, viaf_source_code, pid, format=None):
def get_online_record(cls, viaf_source_code, pid, rec_format=None):
"""Get VIAF record.
Get's the VIAF record from:
http://www.viaf.org/viaf/sourceID/{source_code}|{pid}
:param viaf_source_code: agent source code
:param pid: pid for agent source code
:param format: raw = get the not transformed VIAF record
:param rec_format: raw = get the not transformed VIAF record
link = get the VIAF link record
:returns: VIAF record as json
"""
viaf_format = '/viaf.json'
if format == 'link':
if rec_format == 'link':
viaf_format = '/justlinks.json'
format = 'raw'
rec_format = 'raw'
viaf_url = current_app.config.get('RERO_MEF_VIAF_BASE_URL')
url = f'{viaf_url}/viaf'
if viaf_source_code.upper() == 'VIAF':
Expand All @@ -346,7 +344,7 @@ def get_online_record(cls, viaf_source_code, pid, format=None):
result = {}
if response.status_code == requests.codes.ok:
msg = f'VIAF get: {pid:<15} {url} | OK'
if format == 'raw':
if rec_format == 'raw':
return response.json(), msg
data_json = response.json()
result['pid'] = data_json.get('viafID')
Expand Down Expand Up @@ -394,7 +392,6 @@ def update_online(self, dbcommit=False, reindex=False):
:param reindex: Reindex record.
:returns: record and actions message.
"""
from rero_mef.api import Action
online_data, _ = self.get_online_record(
viaf_source_code='VIAF',
pid=self.pid
Expand Down Expand Up @@ -491,7 +488,7 @@ def delete(self, force=True, dbcommit=False, delindex=False):
AgentMefRecord.flush_indexes()
# recreate MEF records for agents
for agent_record in old_agent_records.values():
mef, test = agent_record.create_or_update_mef(
mef, _ = agent_record.create_or_update_mef(
dbcommit=True,
reindex=True
)
Expand Down
15 changes: 7 additions & 8 deletions rero_mef/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,13 +236,12 @@ def replace(self, data, commit=False, dbcommit=False, reindex=False):
if self.get('md5'):
new_data = add_md5(new_data)
self.clear()
self = self.update(
return self.update(
data=new_data,
commit=commit,
dbcommit=dbcommit,
reindex=reindex
)
return self

def dbcommit(self, reindex=False, forceindex=False):
"""Commit changes to db."""
Expand Down Expand Up @@ -285,18 +284,18 @@ def get_record_by_pid(cls, pid, with_deleted=False):
db.session.rollback()

@classmethod
def get_pid_by_id(cls, id):
def get_pid_by_id(cls, id_):
"""Get pid by uuid."""
persistent_identifier = cls.get_persistent_identifier(id)
persistent_identifier = cls.get_persistent_identifier(id_)
return str(persistent_identifier.pid_value)

@classmethod
def get_persistent_identifier(cls, id):
def get_persistent_identifier(cls, id_):
"""Get Persistent Identifier."""
return PersistentIdentifier.get_by_object(
cls.provider.pid_type,
cls.object_type,
id
id_
)

@classmethod
Expand Down Expand Up @@ -374,8 +373,8 @@ def get_all_ids(cls, with_deleted=False, limit=100000, date=None):
@classmethod
def get_all_records(cls, with_deleted=False, limit=100000):
"""Get all records. Return a generator iterator."""
for id in cls.get_all_ids(with_deleted=with_deleted, limit=limit):
yield cls.get_record(id)
for id_ in cls.get_all_ids(with_deleted=with_deleted, limit=limit):
yield cls.get_record(id_)

@classmethod
def count(cls, with_deleted=False):
Expand Down
Loading

0 comments on commit 91aa163

Please sign in to comment.