arxiv: use arxiv.org urls

*ref: cern-sis/issues-inspire/issues/600
inspirehep · Oct 29, 2024 · 4b9cce7 · 4b9cce7
1 parent ff47b1c
commit 4b9cce7
Show file tree

Hide file tree

Showing 19 changed files with 62 additions and 61 deletions.
diff --git a/docs/e2e_tutorial.rst b/docs/e2e_tutorial.rst
@@ -99,8 +99,8 @@ current test session. Many of them (``test-indexer``, ``test-web-e2e.local``) ar
 not recorded. You might notice a few requests to ArXiv like so:
 
 * ``GET http://export.arxiv.org/oai2?verb=GetRecord&metadataPrefix=arXiv&identifier=oai...``
-* ``GET http://export.arxiv.org/pdf/1806.04664``
-* ``GET http://export.arxiv.org/e-print/1806.04664``
+* ``GET https://arxiv.org/pdf/1806.04664``
+* ``GET https://arxiv.org/e-print/1806.04664``
 
 These are live interactions that are recorded, you can find them in
 ``tests/e2e/scenarios/arxiv_in_hp/ArxivService/``. If you need to re-record an interaction, simply

diff --git a/inspirehep/config.py b/inspirehep/config.py
@@ -1232,9 +1232,9 @@
     "CERN": "CDS Hidden",
     "FERMILAB": "Fermilab"
 }
-ARXIV_PDF_URL = "http://export.arxiv.org/pdf/{arxiv_id}"
-ARXIV_PDF_URL_ALTERNATIVE = "http://arxiv.org/pdf/{arxiv_id}"
-ARXIV_TARBALL_URL = "http://export.arxiv.org/e-print/{arxiv_id}"
+ARXIV_PDF_URL = "https://arxiv.org/pdf/{arxiv_id}"
+ARXIV_PDF_URL_ALTERNATIVE = "https://export.arxiv.org/pdf/{arxiv_id}"
+ARXIV_TARBALL_URL = "https://arxiv.org/e-print/{arxiv_id}"
 
 ARXIV_CATEGORIES = {
     'core': [

diff --git a/services.yml b/services.yml
@@ -24,6 +24,7 @@ version: '2.1'
 
 services:
   base:
+    platform: linux/amd64
     image: inspire-build/inspire-next-base
     tty: true
     build:

diff --git a/tests/e2e/scenarios/harvest_core_article_goes_in/ArxivService/eprint_tarball.yaml b/tests/e2e/scenarios/harvest_core_article_goes_in/ArxivService/eprint_tarball.yaml
@@ -6,7 +6,7 @@ request:
     Connection: [keep-alive]
     User-Agent: [python-requests/2.18.4]
   method: GET
-  url: http://export.arxiv.org/e-print/1404.0579
+  url: https://arxiv.org/e-print/1404.0579
 response:
   body: !!binary |
 

diff --git a/tests/e2e/scenarios/harvest_core_article_goes_in/ArxivService/fulltext_pdf.yaml b/tests/e2e/scenarios/harvest_core_article_goes_in/ArxivService/fulltext_pdf.yaml
@@ -6,7 +6,7 @@ request:
     Connection: [keep-alive]
     User-Agent: [python-requests/2.18.4]
   method: GET
-  url: http://export.arxiv.org/pdf/1412.0200
+  url: https://arxiv.org/pdf/1412.0200
 response:
   body: !!binary |
 

diff --git a/...e2e/scenarios/harvest_core_article_manual_accept_goes_in/ArxivService/eprint_tarball.yaml b/...e2e/scenarios/harvest_core_article_manual_accept_goes_in/ArxivService/eprint_tarball.yaml
diff --git a/...s/e2e/scenarios/harvest_core_article_manual_accept_goes_in/ArxivService/fulltext_pdf.yaml b/...s/e2e/scenarios/harvest_core_article_manual_accept_goes_in/ArxivService/fulltext_pdf.yaml
diff --git a/tests/e2e/scenarios/harvest_non_core_article_goes_in/ArxivService/eprint_tarball.yaml b/tests/e2e/scenarios/harvest_non_core_article_goes_in/ArxivService/eprint_tarball.yaml
@@ -6,7 +6,7 @@ request:
     Connection: [keep-alive]
     User-Agent: [python-requests/2.18.4]
   method: GET
-  url: http://export.arxiv.org/e-print/1404.0579
+  url: https://arxiv.org/e-print/1404.0579
 response:
   body: !!binary |
     H4sICDDSqFoCA3JldmlzZWQxNDAzMTgudGV4AOz9fWPbRpYmjv5fnwLdg0RkTCqS4nR33KP5Xdtp

diff --git a/tests/e2e/scenarios/harvest_non_core_article_goes_in/ArxivService/fulltext_pdf.yaml b/tests/e2e/scenarios/harvest_non_core_article_goes_in/ArxivService/fulltext_pdf.yaml
@@ -6,7 +6,7 @@ request:
     Connection: [keep-alive]
     User-Agent: [python-requests/2.18.4]
   method: GET
-  url: http://export.arxiv.org/pdf/1404.0579
+  url: https://arxiv.org/pdf/1404.0579
 response:
   body: !!binary |
     JVBERi0xLjQKJcfsj6IKNSAwIG9iago8PC9MZW5ndGggNiAwIFIvRmlsdGVyIC9GbGF0ZURlY29k

diff --git a/tests/e2e/scenarios/harvest_nucl_th_and_jlab_curation/ArxivService/eprint_tarball.yaml b/tests/e2e/scenarios/harvest_nucl_th_and_jlab_curation/ArxivService/eprint_tarball.yaml
@@ -6,7 +6,7 @@ request:
     Connection: [keep-alive]
     User-Agent: [python-requests/2.18.4]
   method: GET
-  url: http://export.arxiv.org/e-print/1806.05669
+  url: https://arxiv.org/e-print/1806.05669
 response:
   body: !!binary |
     H4sICDDSqFoCA3JldmlzZWQxNDAzMTgudGV4AOz9fWPbRpYmjv5fnwLdg0RkTCqS4nR33KP5Xdtp

diff --git a/tests/e2e/scenarios/harvest_nucl_th_and_jlab_curation/ArxivService/fulltext_pdf.yaml b/tests/e2e/scenarios/harvest_nucl_th_and_jlab_curation/ArxivService/fulltext_pdf.yaml
@@ -6,7 +6,7 @@ request:
     Connection: [keep-alive]
     User-Agent: [python-requests/2.18.4]
   method: GET
-  url: http://export.arxiv.org/pdf/1806.05669
+  url: https://arxiv.org/pdf/1806.05669
 response:
   body: !!binary |
     JVBERi0xLjQKJcfsj6IKNSAwIG9iago8PC9MZW5ndGggNiAwIFIvRmlsdGVyIC9GbGF0ZURlY29k

diff --git a/tests/integration/workflows/fixtures/non_core_2208_00828.json b/tests/integration/workflows/fixtures/non_core_2208_00828.json
@@ -199,7 +199,7 @@
              "filename":"2208.00828.pdf",
              "fulltext":true,
              "material":"preprint",
-             "original_url":"http://export.arxiv.org/pdf/2208.00828"
+             "original_url":"https://arxiv.org/pdf/2208.00828"
           }
        ],
        "references":[

diff --git a/tests/integration/workflows/helpers/mocks.py b/tests/integration/workflows/helpers/mocks.py
@@ -30,7 +30,7 @@
 
 def fake_download_file(workflow, name, url):
     """Mock download_file_to_workflow func."""
-    if url == 'http://export.arxiv.org/e-print/1407.7587':
+    if url == 'https://arxiv.org/e-print/1407.7587':
         workflow.files[name] = pkg_resources.resource_stream(
             __name__,
             os.path.join(
@@ -39,7 +39,7 @@ def fake_download_file(workflow, name, url):
             )
         )
         return workflow.files[name]
-    elif url == 'http://export.arxiv.org/pdf/1407.7587':
+    elif url == 'https://arxiv.org/pdf/1407.7587':
         workflow.files[name] = pkg_resources.resource_stream(
             __name__,
             os.path.join(

diff --git a/tests/integration/workflows/test_article_workflow.py b/tests/integration/workflows/test_article_workflow.py
@@ -388,7 +388,7 @@ def test_run_next_wf_is_not_starting_core_selection_wfs(
     workflow = build_workflow(record, extra_data={"delay": 10})
     mocked_external_services.register_uri(
         "GET",
-        "http://export.arxiv.org/pdf/1802.08709.pdf",
+        "https://arxiv.org/pdf/1802.08709.pdf",
         content=pkg_resources.resource_string(
             __name__, os.path.join("fixtures", "1802.08709.pdf")
         ),
@@ -398,7 +398,7 @@ def test_run_next_wf_is_not_starting_core_selection_wfs(
     )
     mocked_external_services.register_uri(
         "GET",
-        "http://export.arxiv.org/e-print/1802.08709.pdf",
+        "https://arxiv.org/e-print/1802.08709.pdf",
         content=pkg_resources.resource_string(
             __name__, os.path.join("fixtures", "1802.08709.pdf")
         ),
@@ -436,7 +436,7 @@ def test_run_next_wf_is_not_starting_core_selection_wfs(
 
     mocked_external_services.register_uri(
         "GET",
-        "http://export.arxiv.org/pdf/1802.08709.pdf",
+        "https://arxiv.org/pdf/1802.08709.pdf",
         content=pkg_resources.resource_string(
             __name__, os.path.join("fixtures", "1802.08709.pdf")
         ),
@@ -446,7 +446,7 @@ def test_run_next_wf_is_not_starting_core_selection_wfs(
     )
     mocked_external_services.register_uri(
         "GET",
-        "http://export.arxiv.org/e-print/1802.08709.pdf",
+        "https://arxiv.org/e-print/1802.08709.pdf",
         content=pkg_resources.resource_string(
             __name__, os.path.join("fixtures", "1802.08709.pdf")
         ),

diff --git a/..._async/cassettes/test_wf_rejects_automatically_when_previous_matched_wf_was_rejected.yaml b/..._async/cassettes/test_wf_rejects_automatically_when_previous_matched_wf_was_rejected.yaml
@@ -6,7 +6,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/pdf/1407.7587
+    uri: https://arxiv.org/pdf/1407.7587
   response:
     body:
       string: !!binary |
@@ -5419,7 +5419,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/e-print/1407.7587
+    uri: https://arxiv.org/e-print/1407.7587
   response:
     body:
       string: !!binary |
@@ -8590,7 +8590,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/pdf/1407.7587
+    uri: https://arxiv.org/pdf/1407.7587
   response:
     body:
       string: !!binary |

diff --git a/...s/integration_async/cassettes/test_wf_replaces_old_workflow_which_is_in_halted_state.yaml b/...s/integration_async/cassettes/test_wf_replaces_old_workflow_which_is_in_halted_state.yaml
@@ -6,7 +6,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/pdf/1407.7587
+    uri: https://arxiv.org/pdf/1407.7587
   response:
     body:
       string: !!binary |
@@ -5419,7 +5419,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/e-print/1407.7587
+    uri: https://arxiv.org/e-print/1407.7587
   response:
     body:
       string: !!binary |
@@ -8590,7 +8590,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/pdf/1407.7587
+    uri: https://arxiv.org/pdf/1407.7587
   response:
     body:
       string: !!binary |
@@ -19481,7 +19481,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/pdf/1407.7587
+    uri: https://arxiv.org/pdf/1407.7587
   response:
     body:
       string: !!binary |
@@ -24894,7 +24894,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/e-print/1407.7587
+    uri: https://arxiv.org/e-print/1407.7587
   response:
     body:
       string: !!binary |
@@ -28065,7 +28065,7 @@ interactions:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
     method: GET
-    uri: http://export.arxiv.org/pdf/1407.7587
+    uri: https://arxiv.org/pdf/1407.7587
   response:
     body:
       string: !!binary |

diff --git a/tests/unit/workflows/test_workflows_tasks_actions.py b/tests/unit/workflows/test_workflows_tasks_actions.py
@@ -78,7 +78,7 @@ def _get_auto_reject_obj(decision, has_core_keywords, fulltext_used):
 def test_download_documents():
     with requests_mock.Mocker() as requests_mocker:
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1605.03844',
+            'GET', 'https://arxiv.org/pdf/1605.03844',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1605.03844.pdf')),
         )
@@ -90,7 +90,7 @@ def test_download_documents():
             'documents': [
                 {
                     'key': '1605.03844.pdf',
-                    'url': 'http://export.arxiv.org/pdf/1605.03844'
+                    'url': 'https://arxiv.org/pdf/1605.03844'
                 },
             ],
         }  # literature/1458302
@@ -113,12 +113,12 @@ def test_download_documents():
 def test_download_documents_with_multiple_documents():
     with requests_mock.Mocker() as requests_mocker:
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1605.03844',
+            'GET', 'https://arxiv.org/pdf/1605.03844',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1605.03844.pdf')),
         )
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1605.03845',
+            'GET', 'https://arxiv.org/pdf/1605.03845',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1605.03844.pdf')),
         )
@@ -130,11 +130,11 @@ def test_download_documents_with_multiple_documents():
             'documents': [
                 {
                     'key': '1605.03844.pdf',
-                    'url': 'http://export.arxiv.org/pdf/1605.03844'
+                    'url': 'https://arxiv.org/pdf/1605.03844'
                 },
                 {
                     'key': '1605.03845.pdf',
-                    'url': 'http://export.arxiv.org/pdf/1605.03845'
+                    'url': 'https://arxiv.org/pdf/1605.03845'
                 },
             ],
         }  # literature/1458302
@@ -869,7 +869,7 @@ def test_populate_journal_coverage_does_nothing_if_no_journal_is_found(mock_repl
 def test_populate_submission_document():
     with requests_mock.Mocker() as requests_mocker:
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1605.03844',
+            'GET', 'https://arxiv.org/pdf/1605.03844',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1605.03844.pdf')),
         )
@@ -889,7 +889,7 @@ def test_populate_submission_document():
             },
         }
         extra_data = {
-            'submission_pdf': 'http://export.arxiv.org/pdf/1605.03844',
+            'submission_pdf': 'https://arxiv.org/pdf/1605.03844',
         }
         files = MockFiles({})
         assert validate(data['acquisition_source'], subschema) is None
@@ -903,9 +903,9 @@ def test_populate_submission_document():
             {
                 'fulltext': True,
                 'key': 'fulltext.pdf',
-                'original_url': 'http://export.arxiv.org/pdf/1605.03844',
+                'original_url': 'https://arxiv.org/pdf/1605.03844',
                 'source': 'submitter',
-                'url': 'http://export.arxiv.org/pdf/1605.03844',
+                'url': 'https://arxiv.org/pdf/1605.03844',
             },
         ]
         result = obj.data['documents']
@@ -916,7 +916,7 @@ def test_populate_submission_document():
 def test_populate_submission_document_does_not_duplicate_documents():
     with requests_mock.Mocker() as requests_mocker:
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1605.03844',
+            'GET', 'https://arxiv.org/pdf/1605.03844',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1605.03844.pdf')),
         )
@@ -936,7 +936,7 @@ def test_populate_submission_document_does_not_duplicate_documents():
             },
         }
         extra_data = {
-            'submission_pdf': 'http://export.arxiv.org/pdf/1605.03844',
+            'submission_pdf': 'https://arxiv.org/pdf/1605.03844',
         }
         files = MockFiles({})
         assert validate(data['acquisition_source'], subschema) is None
@@ -951,9 +951,9 @@ def test_populate_submission_document_does_not_duplicate_documents():
             {
                 'fulltext': True,
                 'key': 'fulltext.pdf',
-                'original_url': 'http://export.arxiv.org/pdf/1605.03844',
+                'original_url': 'https://arxiv.org/pdf/1605.03844',
                 'source': 'submitter',
-                'url': 'http://export.arxiv.org/pdf/1605.03844',
+                'url': 'https://arxiv.org/pdf/1605.03844',
             },
         ]
         result = obj.data['documents']
@@ -964,7 +964,7 @@ def test_populate_submission_document_does_not_duplicate_documents():
 def test_populate_submission_document_without_pdf():
     with requests_mock.Mocker() as requests_mocker:
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1707.02785',
+            'GET', 'https://arxiv.org/pdf/1707.02785',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1707.02785.html')),
         )
@@ -985,7 +985,7 @@ def test_populate_submission_document_without_pdf():
         assert validate(data['acquisition_source'], subschema) is None
 
         extra_data = {
-            'submission_pdf': 'http://export.arxiv.org/pdf/1707.02785',
+            'submission_pdf': 'https://arxiv.org/pdf/1707.02785',
         }
         files = MockFiles({})
         obj = MockObj(data, extra_data, files=files)
@@ -1492,7 +1492,7 @@ def test_url_is_correctly_escaped():
 def test_populate_submission_document_without_documents():
     with requests_mock.Mocker() as requests_mocker:
         requests_mocker.register_uri(
-            'GET', 'http://export.arxiv.org/pdf/1605.03844',
+            'GET', 'https://arxiv.org/pdf/1605.03844',
             content=pkg_resources.resource_string(
                 __name__, os.path.join('fixtures', '1605.03844.pdf')),
         )