test: Start testing [settings] section. Fix InvalidUsernameError init…

…ialization fcf464c.

docs/api.rst

@@ -31,7 +31,7 @@ Example:
 addversion.json
 ---------------
 
-Add a version to a project, creating the project if needed.
+Add a version to a project in :ref:`eggstorage`, creating the project if needed.
 
 Supported request methods
   ``POST``
@@ -179,7 +179,7 @@ Example:
 listversions.json
 -----------------
 
-Get the versions of a project, in order, with the latest version last.
+Get the versions of a project in :ref:`eggstorage`, in :ref:`order<overview-order>`, with the latest version last.
 
 Supported request methods
   ``GET``
@@ -273,7 +273,7 @@ Example:
 delversion.json
 ---------------
 
-Delete a version of a project. If no versions of the project remain, delete the project, too.
+Delete a version of a project from :ref:`eggstorage`. If no versions of the project remain, delete the project, too.
 
 Supported request methods
   ``POST``
@@ -295,7 +295,7 @@ Example:
 delproject.json
 ---------------
 
-Delete a project and its versions.
+Delete a project and its versions from :ref:`eggstorage`.
 
 Supported request methods
   ``POST``

docs/overview.rst

@@ -7,6 +7,11 @@ Projects and versions
 
 Scrapyd can manage multiple Scrapy projects. Each project can have multiple versions. The latest version is used by default for starting spiders.
 
+.. _overview-order:
+
+Version order
+-------------
+
 The latest version is the alphabetically greatest, unless all version names are `version specifiers <https://packaging.python.org/en/latest/specifications/version-specifiers/>`__ like ``1.0`` or ``1.0rc1``, in which case they are sorted as such.
 
 How Scrapyd works

scrapyd/exceptions.py

@@ -9,7 +9,7 @@ class ConfigError(ScrapydError):
 class InvalidUsernameError(ConfigError):
     """Raised if the username contains a colon."""
 
-    def __init__(self, message):
+    def __init__(self):
         super().__init__(
             "The `username` option contains illegal character ':'. Check and update the Scrapyd configuration file."
         )

tests/conftest.py

@@ -22,10 +22,19 @@ def chdir(monkeypatch, tmpdir):
     return monkeypatch.chdir(tmpdir)
 
 
-@pytest.fixture(params=[None, (Config.SECTION, "items_dir", "items")], ids=["default", "items_dir"])
+@pytest.fixture(
+    params=[
+        None,
+        (Config.SECTION, "items_dir", "items"),
+        ("settings", "localproject", "tests.fixtures.localbot.settings"),
+    ],
+    ids=["default", "items_dir", "settings"],
+)
 def root(request, chdir):
     config = Config()
     if request.param:
+        if request.param[0] != Config.SECTION:
+            config.cp.add_section(request.param[0])
         config.cp.set(*request.param)
 
     return Root(config, application(config))

tests/fixtures/localproject/localproject/settings.py

@@ -0,0 +1,7 @@
+BOT_NAME = "localproject"
+SPIDER_MODULES = ["localproject.spiders"]
+NEWSPIDER_MODULE = "localproject.spiders"
+ROBOTSTXT_OBEY = True
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEED_EXPORT_ENCODING = "utf-8"

tests/fixtures/localproject/localproject/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

tests/fixtures/localproject/localproject/spiders/example.py

@@ -0,0 +1,8 @@
+import scrapy
+
+
+class ExampleSpider(scrapy.Spider):
+    name = "example"
+
+    def start_requests(self):
+        pass

tests/fixtures/localproject/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = localproject.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = localproject

tests/test_webservice.py

@@ -21,6 +21,11 @@ def app(chdir):
     return application
 
 
+def has_settings(root):
+    # The configuration is not guaranteed to be accessible here, but it is for now.
+    return root.scheduler.config.cp.has_section("settings")
+
+
 def add_test_version(app, project, version, basename):
     app.getComponent(IEggStorage).put(io.BytesIO(get_egg_data(basename)), project, version)
 
@@ -148,43 +153,40 @@ def test_daemonstatus(txrequest, root_with_egg):
     assert_content(txrequest, root_with_egg, "daemonstatus", {}, expected)
 
 
-def test_list_spiders(txrequest, root):
+@pytest.mark.parametrize(
+    ("extra_args", "spiders"),
+    [
+        ({}, ["spider1", "spider2", "spider3"]),
+        ({b"_version": [b"r1"]}, ["spider1", "spider2"]),
+    ],
+)
+def test_list_spiders(txrequest, root, extra_args, spiders):
     UtilsCache.invalid_cache("myproject")  # test_get_spider_list fills cache
 
     root_add_version(root, "myproject", "r1", "mybot")
     root_add_version(root, "myproject", "r2", "mybot2")
 
-    expected = {"status": "ok", "spiders": ["spider1", "spider2", "spider3"]}
-    assert_content(txrequest, root, "listspiders", {b"project": [b"myproject"]}, expected)
-
+    expected = {"status": "ok", "spiders": spiders}
+    assert_content(txrequest, root, "listspiders", {b"project": [b"myproject"], **extra_args}, expected)
 
-def test_list_spiders_nonexistent(txrequest, root):
-    txrequest.args = {b"project": [b"nonexistent"]}
-    with pytest.raises(error.Error) as exc:
-        root.children[b"listspiders.json"].render_GET(txrequest)
-
-    assert exc.value.status == b"200"
-    assert exc.value.message == b"project 'nonexistent' not found"
 
-
-def test_list_spiders_version(txrequest, root):
-    root_add_version(root, "myproject", "r1", "mybot")
-    root_add_version(root, "myproject", "r2", "mybot2")
-
-    expected = {"status": "ok", "spiders": ["spider1", "spider2"]}
-    assert_content(txrequest, root, "listspiders", {b"project": [b"myproject"], b"_version": [b"r1"]}, expected)
-
-
-def test_list_spiders_version_nonexistent(txrequest, root):
+@pytest.mark.parametrize(
+    ("args", "param"),
+    [
+        ({b"project": [b"nonexistent"]}, "project"),
+        ({b"project": [b"myproject"], b"_version": [b"nonexistent"]}, "version"),
+    ],
+)
+def test_list_spiders_nonexistent(txrequest, root, args, param):
     root_add_version(root, "myproject", "r1", "mybot")
     root_add_version(root, "myproject", "r2", "mybot2")
 
-    txrequest.args = {b"project": [b"myproject"], b"_version": [b"nonexistent"]}
+    txrequest.args = args.copy()
     with pytest.raises(error.Error) as exc:
         root.children[b"listspiders.json"].render_GET(txrequest)
 
     assert exc.value.status == b"200"
-    assert exc.value.message == b"version 'nonexistent' not found"
+    assert exc.value.message == b"%b 'nonexistent' not found" % param.encode()
 
 
 def test_list_versions(txrequest, root_with_egg):
@@ -194,11 +196,13 @@ def test_list_versions(txrequest, root_with_egg):
 
 def test_list_versions_nonexistent(txrequest, root):
     expected = {"status": "ok", "versions": []}
-    assert_content(txrequest, root, "listversions", {b"project": [b"quotesbot"]}, expected)
+    assert_content(txrequest, root, "listversions", {b"project": [b"localproject"]}, expected)
 
 
 def test_list_projects(txrequest, root_with_egg):
     expected = {"status": "ok", "projects": ["quotesbot"]}
+    if has_settings(root_with_egg):
+        expected["projects"].append("localproject")
     assert_content(txrequest, root_with_egg, "listprojects", {}, expected)