From 41fba518bcc9e93db947c3ed373323cff1af43d4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 20 Jul 2024 19:45:04 -0400 Subject: [PATCH] test: Start testing [settings] section. Fix InvalidUsernameError initialization fcf464c. --- docs/api.rst | 8 +-- docs/overview.rst | 5 ++ scrapyd/exceptions.py | 2 +- tests/conftest.py | 11 +++- .../localproject/localproject/__init__.py | 0 .../localproject/localproject/settings.py | 7 +++ .../localproject/spiders/__init__.py | 4 ++ .../localproject/spiders/example.py | 8 +++ tests/fixtures/localproject/scrapy.cfg | 11 ++++ tests/test_webservice.py | 52 ++++++++++--------- 10 files changed, 78 insertions(+), 30 deletions(-) create mode 100644 tests/fixtures/localproject/localproject/__init__.py create mode 100644 tests/fixtures/localproject/localproject/settings.py create mode 100644 tests/fixtures/localproject/localproject/spiders/__init__.py create mode 100644 tests/fixtures/localproject/localproject/spiders/example.py create mode 100644 tests/fixtures/localproject/scrapy.cfg diff --git a/docs/api.rst b/docs/api.rst index 5e911fdb..19ad1887 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -31,7 +31,7 @@ Example: addversion.json --------------- -Add a version to a project, creating the project if needed. +Add a version to a project in :ref:`eggstorage`, creating the project if needed. Supported request methods ``POST`` @@ -179,7 +179,7 @@ Example: listversions.json ----------------- -Get the versions of a project, in order, with the latest version last. +Get the versions of a project in :ref:`eggstorage`, in :ref:`order`, with the latest version last. Supported request methods ``GET`` @@ -273,7 +273,7 @@ Example: delversion.json --------------- -Delete a version of a project. If no versions of the project remain, delete the project, too. +Delete a version of a project from :ref:`eggstorage`. If no versions of the project remain, delete the project, too. Supported request methods ``POST`` @@ -295,7 +295,7 @@ Example: delproject.json --------------- -Delete a project and its versions. +Delete a project and its versions from :ref:`eggstorage`. Supported request methods ``POST`` diff --git a/docs/overview.rst b/docs/overview.rst index 7013fa8b..ba05965b 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -7,6 +7,11 @@ Projects and versions Scrapyd can manage multiple Scrapy projects. Each project can have multiple versions. The latest version is used by default for starting spiders. +.. _overview-order: + +Version order +------------- + The latest version is the alphabetically greatest, unless all version names are `version specifiers `__ like ``1.0`` or ``1.0rc1``, in which case they are sorted as such. How Scrapyd works diff --git a/scrapyd/exceptions.py b/scrapyd/exceptions.py index 82b3a097..1cdf5e5e 100644 --- a/scrapyd/exceptions.py +++ b/scrapyd/exceptions.py @@ -9,7 +9,7 @@ class ConfigError(ScrapydError): class InvalidUsernameError(ConfigError): """Raised if the username contains a colon.""" - def __init__(self, message): + def __init__(self): super().__init__( "The `username` option contains illegal character ':'. Check and update the Scrapyd configuration file." ) diff --git a/tests/conftest.py b/tests/conftest.py index c0b21f83..7dd5a3d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,10 +22,19 @@ def chdir(monkeypatch, tmpdir): return monkeypatch.chdir(tmpdir) -@pytest.fixture(params=[None, (Config.SECTION, "items_dir", "items")], ids=["default", "items_dir"]) +@pytest.fixture( + params=[ + None, + (Config.SECTION, "items_dir", "items"), + ("settings", "localproject", "tests.fixtures.localbot.settings"), + ], + ids=["default", "items_dir", "settings"], +) def root(request, chdir): config = Config() if request.param: + if request.param[0] != Config.SECTION: + config.cp.add_section(request.param[0]) config.cp.set(*request.param) return Root(config, application(config)) diff --git a/tests/fixtures/localproject/localproject/__init__.py b/tests/fixtures/localproject/localproject/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/localproject/localproject/settings.py b/tests/fixtures/localproject/localproject/settings.py new file mode 100644 index 00000000..9fc67072 --- /dev/null +++ b/tests/fixtures/localproject/localproject/settings.py @@ -0,0 +1,7 @@ +BOT_NAME = "localproject" +SPIDER_MODULES = ["localproject.spiders"] +NEWSPIDER_MODULE = "localproject.spiders" +ROBOTSTXT_OBEY = True +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" +FEED_EXPORT_ENCODING = "utf-8" diff --git a/tests/fixtures/localproject/localproject/spiders/__init__.py b/tests/fixtures/localproject/localproject/spiders/__init__.py new file mode 100644 index 00000000..ebd689ac --- /dev/null +++ b/tests/fixtures/localproject/localproject/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/tests/fixtures/localproject/localproject/spiders/example.py b/tests/fixtures/localproject/localproject/spiders/example.py new file mode 100644 index 00000000..11fa05cf --- /dev/null +++ b/tests/fixtures/localproject/localproject/spiders/example.py @@ -0,0 +1,8 @@ +import scrapy + + +class ExampleSpider(scrapy.Spider): + name = "example" + + def start_requests(self): + pass diff --git a/tests/fixtures/localproject/scrapy.cfg b/tests/fixtures/localproject/scrapy.cfg new file mode 100644 index 00000000..28c8991f --- /dev/null +++ b/tests/fixtures/localproject/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = localproject.settings + +[deploy] +#url = http://localhost:6800/ +project = localproject diff --git a/tests/test_webservice.py b/tests/test_webservice.py index ab78106c..56fc8ee5 100644 --- a/tests/test_webservice.py +++ b/tests/test_webservice.py @@ -21,6 +21,11 @@ def app(chdir): return application +def has_settings(root): + # The configuration is not guaranteed to be accessible here, but it is for now. + return root.scheduler.config.cp.has_section("settings") + + def add_test_version(app, project, version, basename): app.getComponent(IEggStorage).put(io.BytesIO(get_egg_data(basename)), project, version) @@ -148,43 +153,40 @@ def test_daemonstatus(txrequest, root_with_egg): assert_content(txrequest, root_with_egg, "daemonstatus", {}, expected) -def test_list_spiders(txrequest, root): +@pytest.mark.parametrize( + ("extra_args", "spiders"), + [ + ({}, ["spider1", "spider2", "spider3"]), + ({b"_version": [b"r1"]}, ["spider1", "spider2"]), + ], +) +def test_list_spiders(txrequest, root, extra_args, spiders): UtilsCache.invalid_cache("myproject") # test_get_spider_list fills cache root_add_version(root, "myproject", "r1", "mybot") root_add_version(root, "myproject", "r2", "mybot2") - expected = {"status": "ok", "spiders": ["spider1", "spider2", "spider3"]} - assert_content(txrequest, root, "listspiders", {b"project": [b"myproject"]}, expected) - + expected = {"status": "ok", "spiders": spiders} + assert_content(txrequest, root, "listspiders", {b"project": [b"myproject"], **extra_args}, expected) -def test_list_spiders_nonexistent(txrequest, root): - txrequest.args = {b"project": [b"nonexistent"]} - with pytest.raises(error.Error) as exc: - root.children[b"listspiders.json"].render_GET(txrequest) - - assert exc.value.status == b"200" - assert exc.value.message == b"project 'nonexistent' not found" - -def test_list_spiders_version(txrequest, root): - root_add_version(root, "myproject", "r1", "mybot") - root_add_version(root, "myproject", "r2", "mybot2") - - expected = {"status": "ok", "spiders": ["spider1", "spider2"]} - assert_content(txrequest, root, "listspiders", {b"project": [b"myproject"], b"_version": [b"r1"]}, expected) - - -def test_list_spiders_version_nonexistent(txrequest, root): +@pytest.mark.parametrize( + ("args", "param"), + [ + ({b"project": [b"nonexistent"]}, "project"), + ({b"project": [b"myproject"], b"_version": [b"nonexistent"]}, "version"), + ], +) +def test_list_spiders_nonexistent(txrequest, root, args, param): root_add_version(root, "myproject", "r1", "mybot") root_add_version(root, "myproject", "r2", "mybot2") - txrequest.args = {b"project": [b"myproject"], b"_version": [b"nonexistent"]} + txrequest.args = args.copy() with pytest.raises(error.Error) as exc: root.children[b"listspiders.json"].render_GET(txrequest) assert exc.value.status == b"200" - assert exc.value.message == b"version 'nonexistent' not found" + assert exc.value.message == b"%b 'nonexistent' not found" % param.encode() def test_list_versions(txrequest, root_with_egg): @@ -194,11 +196,13 @@ def test_list_versions(txrequest, root_with_egg): def test_list_versions_nonexistent(txrequest, root): expected = {"status": "ok", "versions": []} - assert_content(txrequest, root, "listversions", {b"project": [b"quotesbot"]}, expected) + assert_content(txrequest, root, "listversions", {b"project": [b"localproject"]}, expected) def test_list_projects(txrequest, root_with_egg): expected = {"status": "ok", "projects": ["quotesbot"]} + if has_settings(root_with_egg): + expected["projects"].append("localproject") assert_content(txrequest, root_with_egg, "listprojects", {}, expected)