From 8f8b9d179ccdff75627f23728a258c2e7867f757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Ja=C5=A1ek?= Date: Wed, 13 Sep 2023 08:53:55 +0200 Subject: [PATCH 1/5] fix refs in jimi pictures to be only for published content (#184) SDCP-705 --- server/cp/output/formatter/jimi.py | 8 ++++++-- server/tests/output/formatter/jimi_test.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/cp/output/formatter/jimi.py b/server/cp/output/formatter/jimi.py index 5dd8ba0c..b0ad47fa 100644 --- a/server/cp/output/formatter/jimi.py +++ b/server/cp/output/formatter/jimi.py @@ -559,9 +559,13 @@ def _format_refs(self, content, item): [ slug(self._get_original_item(ref)) for ref in superdesk.get_resource_service("news").get( - req=None, lookup={"refs.guid": item["guid"]} + req=None, + lookup={ + "refs.guid": item["guid"], + "pubstatus": "usable", + "state": "published", + }, ) - if ref.get("pubstatus") == "usable" ] ) diff --git a/server/tests/output/formatter/jimi_test.py b/server/tests/output/formatter/jimi_test.py index 50c95bea..7c717a68 100644 --- a/server/tests/output/formatter/jimi_test.py +++ b/server/tests/output/formatter/jimi_test.py @@ -486,7 +486,6 @@ def test_ap_update_keeps_newscomip(self): def test_picture_container_ids(self): resources["news"].service.get.side_effect = [ [ - {"guid": "canceled", "pubstatus": "canceled", "type": "text"}, {"guid": "usable", "pubstatus": "usable", "type": "text"}, { "guid": "usable2", From dad40a300c2441a702cb9b1d39a365fc783a85d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Ja=C5=A1ek?= Date: Thu, 21 Sep 2023 09:16:12 +0200 Subject: [PATCH 2/5] fix jimi formatter for pictures with multiple caption writers (#185) SDCP-715 --- server/cp/output/formatter/jimi.py | 21 ++++++++++++++------- server/tests/output/formatter/jimi_test.py | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/server/cp/output/formatter/jimi.py b/server/cp/output/formatter/jimi.py index b0ad47fa..52b683a5 100644 --- a/server/cp/output/formatter/jimi.py +++ b/server/cp/output/formatter/jimi.py @@ -7,6 +7,7 @@ import cp.ingest.parser.globenewswire as globenewswire import superdesk.etree as sd_etree +from typing import List, Union from num2words import num2words from collections import OrderedDict from celery.utils.functional import uniq @@ -105,6 +106,12 @@ def is_french(item) -> bool: return "fr" in item.get("language", "en") +def text_value(value: Union[str, List[str]]) -> str: + if not isinstance(value, list): + value = [value] + return ", ".join(value) + + class JimiFormatter(Formatter): ENCODING = "utf-8" @@ -513,12 +520,12 @@ def _format_picture_metadata(self, content, item): content.find("Source").text = item["original_source"] if extra.get(cp.ARCHIVE_SOURCE): - etree.SubElement(content, "ArchiveSources").text = extra[cp.ARCHIVE_SOURCE] + etree.SubElement(content, "ArchiveSources").text = text_value(extra[cp.ARCHIVE_SOURCE]) if extra.get(cp.PHOTOGRAPHER_CODE): - etree.SubElement(content, "BylineTitle").text = extra[ + etree.SubElement(content, "BylineTitle").text = text_value(extra[ cp.PHOTOGRAPHER_CODE - ].upper() + ]).upper() if item.get("copyrightnotice"): etree.SubElement(content, "Copyright").text = item["copyrightnotice"][:50] @@ -529,7 +536,7 @@ def _format_picture_metadata(self, content, item): ].replace(" ", " ") if extra.get(cp.CAPTION_WRITER): - etree.SubElement(content, "CaptionWriter").text = extra[cp.CAPTION_WRITER] + etree.SubElement(content, "CaptionWriter").text = text_value(extra[cp.CAPTION_WRITER]) if item.get("ednote"): etree.SubElement(content, "SpecialInstructions").text = item["ednote"] @@ -543,13 +550,13 @@ def _format_picture_metadata(self, content, item): ).text if extra.get(cp.INFOSOURCE): - etree.SubElement(content, "CustomField6").text = extra[cp.INFOSOURCE] + etree.SubElement(content, "CustomField6").text = text_value(extra[cp.INFOSOURCE]) if extra.get(cp.XMP_KEYWORDS): - etree.SubElement(content, "XmpKeywords").text = extra[cp.XMP_KEYWORDS] + etree.SubElement(content, "XmpKeywords").text = text_value(extra[cp.XMP_KEYWORDS]) if extra.get("container"): - etree.SubElement(content, "ContainerIDs").text = extra["container"] + etree.SubElement(content, "ContainerIDs").text = text_value(extra["container"]) else: self._format_refs(content, item) diff --git a/server/tests/output/formatter/jimi_test.py b/server/tests/output/formatter/jimi_test.py index 7c717a68..ca7ef432 100644 --- a/server/tests/output/formatter/jimi_test.py +++ b/server/tests/output/formatter/jimi_test.py @@ -248,6 +248,7 @@ def test_picture(self): "extra": { cp.FILENAME: "NY538", "photographer_code": "stf", + cp.CAPTION_WRITER: ["foo", "bar"], }, "subject": [ {"name": "Americas", "qcode": "A", "scheme": "photo_categories"}, @@ -287,6 +288,7 @@ def test_picture(self): self.assertEqual(updates["copyrightnotice"][:50], item.find("Copyright").text) self.assertEqual(updates["description_text"], item.find("EnglishCaption").text) self.assertEqual("2020-06-03T17:00:56", item.find("DateTaken").text) + self.assertEqual("foo, bar", item.find("CaptionWriter").text) self.assertEqual("media_id", item.find("FileName").text) self.assertEqual("media_id.jpg", item.find("ViewFile").text) From c9922c088421cf7d751e4cd7532ed4739e1bc80f Mon Sep 17 00:00:00 2001 From: Petr Jasek Date: Tue, 26 Sep 2023 11:49:31 +0200 Subject: [PATCH 3/5] update server core SDCP-717 --- server/Procfile | 6 +-- server/requirements.in | 2 +- server/requirements.txt | 87 +++++++++++++++++++++++++---------------- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/server/Procfile b/server/Procfile index 8969669d..31690df9 100644 --- a/server/Procfile +++ b/server/Procfile @@ -1,7 +1,7 @@ rest: gunicorn -c gunicorn_config.py wsgi wamp: python3 -u ws.py -work: celery -A worker -X "${SUPERDESK_CELERY_PREFIX}expiry" worker -beat: celery -A worker beat --pid= +work: celery -A worker worker -X "${SUPERDESK_CELERY_PREFIX}expiry" +beat: celery -A worker beat capi: gunicorn -c gunicorn_config.py content_api.wsgi papi: gunicorn -c gunicorn_config.py prod_api_app -work_expiry: celery -A worker -Q "${SUPERDESK_CELERY_PREFIX}expiry" worker +work_expiry: celery -A worker worker -Q "${SUPERDESK_CELERY_PREFIX}expiry" diff --git a/server/requirements.in b/server/requirements.in index 30035aa1..b06d288b 100755 --- a/server/requirements.in +++ b/server/requirements.in @@ -4,6 +4,6 @@ python3-saml>=1.9,<1.10 python-xmp-toolkit>=2.0.1,<2.1 num2words==0.5.10 -git+https://github.com/superdesk/superdesk-core.git@v2.6.4#egg=superdesk-core +git+https://github.com/superdesk/superdesk-core.git@hotfix/2.6.5#egg=superdesk-core git+https://github.com/superdesk/superdesk-planning.git@v2.6.2#egg=superdesk-planning git+https://github.com/superdesk/superdesk-analytics.git@v2.6.0-rc1#egg=superdesk-analytics diff --git a/server/requirements.txt b/server/requirements.txt index 59616708..640dcc93 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -2,14 +2,16 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --resolver=backtracking requirements.in +# pip-compile requirements.in # -amqp==2.6.1 +amqp==5.1.1 # via kombu arrow==0.13.0 # via # eve-elastic # superdesk-core +async-timeout==4.0.3 + # via redis authlib==0.14.3 # via superdesk-core babel==2.12.1 @@ -24,21 +26,21 @@ blinker==1.4 # flask-mail # raven # superdesk-core -boto3==1.26.131 +boto3==1.28.54 # via superdesk-core -botocore==1.29.131 +botocore==1.31.54 # via # boto3 # s3transfer -cachetools==5.3.0 +cachetools==5.3.1 # via flask-oidc-ex -celery[redis]==4.4.7 +celery[redis]==5.2.7 # via superdesk-core -cerberus==1.3.4 +cerberus==1.3.5 # via # eve # superdesk-core -certifi==2023.5.7 +certifi==2023.7.22 # via # elastic-apm # elasticsearch @@ -49,41 +51,54 @@ cffi==1.15.1 # cryptography chardet==3.0.4 # via superdesk-core -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests ciso8601==1.0.8 # via eve-elastic -click==7.1.2 - # via flask +click==8.1.7 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # flask +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery croniter==0.3.37 # via superdesk-core -cryptography==40.0.2 +cryptography==41.0.4 # via # authlib # jwcrypto -deepdiff==6.3.0 +deepdiff==6.5.0 # via superdesk-planning defusedxml==0.7.1 # via python3-saml -deprecated==1.2.13 +deprecated==1.2.14 # via jwcrypto docopt==0.6.2 # via num2words draftjs-exporter[lxml]==2.1.7 # via superdesk-core -elastic-apm[flask]==6.15.1 +ecs-logging==2.1.0 + # via elastic-apm +elastic-apm[flask]==6.18.0 # via superdesk-core elasticsearch==7.13.4 # via eve-elastic eve==1.1.2 # via superdesk-core -eve-elastic==7.3.1 +eve-elastic==7.3.2 # via superdesk-core events==0.3 # via eve feedparser==6.0.10 # via superdesk-core -flask==1.1.4 +flask==1.1.2 # via # eve # flask-babel @@ -102,7 +117,7 @@ flask-script==2.0.6 # via superdesk-core future==0.18.3 # via python-twitter -gunicorn==20.1.0 +gunicorn==21.2.0 # via -r requirements.in hachoir==3.0a3 # via superdesk-core @@ -122,19 +137,21 @@ itsdangerous==1.1.0 # via # flask # flask-oidc-ex + # superdesk-core jinja2==2.11.3 # via # flask # flask-babel + # superdesk-core jmespath==1.0.1 # via # boto3 # botocore -jwcrypto==1.4.2 +jwcrypto==1.5.0 # via # flask-oidc-ex # python-jwt -kombu==4.6.11 +kombu==5.2.4 # via # celery # superdesk-core @@ -151,7 +168,7 @@ markupsafe==2.0.1 # superdesk-core mongolock==1.3.4 # via superdesk-core -natsort==8.3.1 +natsort==8.4.0 # via croniter num2words==0.5.10 # via -r requirements.in @@ -161,8 +178,12 @@ oauthlib==3.2.2 # via requests-oauthlib ordered-set==4.1.0 # via deepdiff +packaging==23.1 + # via gunicorn pillow==9.2.0 # via superdesk-core +prompt-toolkit==3.0.39 + # via click-repl pyasn1==0.5.0 # via # ldap3 @@ -178,7 +199,7 @@ pymongo==3.11.4 # eve # mongolock # superdesk-core -pyparsing==3.0.9 +pyparsing==3.1.1 # via httplib2 python-dateutil==2.7.5 # via @@ -197,7 +218,7 @@ python-xmp-toolkit==2.0.1 # via -r requirements.in python3-saml==1.9.0 # via -r requirements.in -pytz==2023.3 +pytz==2023.3.post1 # via # celery # eve-elastic @@ -206,17 +227,17 @@ pytz==2023.3 # python-xmp-toolkit # superdesk-core # tzlocal -pyyaml==5.4.1 +pyyaml==6.0.1 # via superdesk-core raven[flask]==6.10.0 # via superdesk-core -redis==3.2.1 +redis==4.5.5 # via # celery # superdesk-core regex==2020.7.14 # via superdesk-core -requests==2.30.0 +requests==2.31.0 # via # python-twitter # requests-oauthlib @@ -225,7 +246,7 @@ requests-oauthlib==1.3.1 # via python-twitter rsa==4.9 # via oauth2client -s3transfer==0.6.1 +s3transfer==0.6.2 # via boto3 sgmllib3k==1.0.0 # via feedparser @@ -240,11 +261,11 @@ six==1.16.0 # python-dateutil superdesk-analytics @ git+https://github.com/superdesk/superdesk-analytics.git@v2.6.0-rc1 # via -r requirements.in -superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.6.4 +superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@hotfix/2.6.5 # via -r requirements.in superdesk-planning @ git+https://github.com/superdesk/superdesk-planning.git@v2.6.2 # via -r requirements.in -typing-extensions==4.5.0 +typing-extensions==4.8.0 # via superdesk-core tzlocal==2.1 # via superdesk-core @@ -257,10 +278,13 @@ urllib3==1.25.11 # elasticsearch # requests # superdesk-core -vine==1.3.0 +vine==5.0.0 # via # amqp # celery + # kombu +wcwidth==0.2.6 + # via prompt-toolkit websockets==10.3 # via superdesk-core werkzeug==1.0.1 @@ -273,6 +297,3 @@ wrapt==1.15.0 # elastic-apm xmlsec==1.3.13 # via python3-saml - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From c730ce2089310cafbf8c8c485cc9af7e83e49765 Mon Sep 17 00:00:00 2001 From: Petr Jasek Date: Wed, 27 Sep 2023 17:54:09 +0200 Subject: [PATCH 4/5] use server core v2.6.5 --- server/requirements.in | 2 +- server/requirements.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/requirements.in b/server/requirements.in index b06d288b..83bb0add 100755 --- a/server/requirements.in +++ b/server/requirements.in @@ -4,6 +4,6 @@ python3-saml>=1.9,<1.10 python-xmp-toolkit>=2.0.1,<2.1 num2words==0.5.10 -git+https://github.com/superdesk/superdesk-core.git@hotfix/2.6.5#egg=superdesk-core +git+https://github.com/superdesk/superdesk-core.git@v2.6.5#egg=superdesk-core git+https://github.com/superdesk/superdesk-planning.git@v2.6.2#egg=superdesk-planning git+https://github.com/superdesk/superdesk-analytics.git@v2.6.0-rc1#egg=superdesk-analytics diff --git a/server/requirements.txt b/server/requirements.txt index 640dcc93..f87658bf 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -26,9 +26,9 @@ blinker==1.4 # flask-mail # raven # superdesk-core -boto3==1.28.54 +boto3==1.28.55 # via superdesk-core -botocore==1.31.54 +botocore==1.31.55 # via # boto3 # s3transfer @@ -246,7 +246,7 @@ requests-oauthlib==1.3.1 # via python-twitter rsa==4.9 # via oauth2client -s3transfer==0.6.2 +s3transfer==0.7.0 # via boto3 sgmllib3k==1.0.0 # via feedparser @@ -261,7 +261,7 @@ six==1.16.0 # python-dateutil superdesk-analytics @ git+https://github.com/superdesk/superdesk-analytics.git@v2.6.0-rc1 # via -r requirements.in -superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@hotfix/2.6.5 +superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.6.5 # via -r requirements.in superdesk-planning @ git+https://github.com/superdesk/superdesk-planning.git@v2.6.2 # via -r requirements.in From 062919c72c4be48b96022b077a14ea5d8f420762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Ja=C5=A1ek?= Date: Thu, 12 Oct 2023 11:10:07 +0200 Subject: [PATCH 5/5] fix jimi formatter picture refs for scheduled items (#186) SDCP-720 --- server/cp/output/formatter/jimi.py | 17 ++++++++++------- server/tests/output/formatter/jimi_test.py | 4 ++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/server/cp/output/formatter/jimi.py b/server/cp/output/formatter/jimi.py index 52b683a5..6be1b6e5 100644 --- a/server/cp/output/formatter/jimi.py +++ b/server/cp/output/formatter/jimi.py @@ -565,14 +565,17 @@ def _format_refs(self, content, item): refs = set( [ slug(self._get_original_item(ref)) - for ref in superdesk.get_resource_service("news").get( - req=None, - lookup={ - "refs.guid": item["guid"], - "pubstatus": "usable", - "state": "published", + for ref in superdesk.get_resource_service("news").search({ + "query": { + "bool": { + "must": [ + {"term": {"refs.guid": item["guid"]}}, + {"term": {"pubstatus": "usable"}}, + {"terms": {"state": ["published", "scheduled"]}}, + ], + }, }, - ) + }) ] ) diff --git a/server/tests/output/formatter/jimi_test.py b/server/tests/output/formatter/jimi_test.py index ca7ef432..6b6913c1 100644 --- a/server/tests/output/formatter/jimi_test.py +++ b/server/tests/output/formatter/jimi_test.py @@ -486,7 +486,7 @@ def test_ap_update_keeps_newscomip(self): self.assertEqual("00000001", item.find("NewsCompID").text) def test_picture_container_ids(self): - resources["news"].service.get.side_effect = [ + resources["news"].service.search.side_effect = [ [ {"guid": "usable", "pubstatus": "usable", "type": "text"}, { @@ -507,7 +507,7 @@ def test_picture_container_ids(self): } ) - resources["news"].service.get.side_effect = None + resources["news"].service.search.side_effect = None self.assertEqual("{}, usable".format(32 * "a"), item.find("ContainerIDs").text)