diff --git a/.travis.yml b/.travis.yml index cd2e8b47..ab198f78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ matrix: env: TOXENV=py27 - python: 2.7 env: TOXENV=pypy - - python: 3.3 - env: TOXENV=py33 - python: 3.4 env: TOXENV=py34 - python: 3.5 diff --git a/docs/news.rst b/docs/news.rst index db805ded..cffd2510 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,17 @@ Release notes ============= +1.2.1 +----- +*Release date: 2019-06-17* + +fixed +~~~~~ +- http header types were breaking newer twisted versions +- DeferredQueue was hiding a pending job when reaching max_proc +- AddVersion's arguments' string types were breaking the environment in windows +- Tests: Updated binary eggs to be scrapy-1.x compatible + 1.2.0 ----- *Release date: 2017-04-12* diff --git a/scrapyd/VERSION b/scrapyd/VERSION index 26aaba0e..6085e946 100644 --- a/scrapyd/VERSION +++ b/scrapyd/VERSION @@ -1 +1 @@ -1.2.0 +1.2.1 diff --git a/scrapyd/config.py b/scrapyd/config.py index 829fa7a7..a0e509ad 100644 --- a/scrapyd/config.py +++ b/scrapyd/config.py @@ -1,5 +1,5 @@ import glob -from io import StringIO +import io from pkgutil import get_data from six.moves.configparser import SafeConfigParser, NoSectionError, NoOptionError from os.path import expanduser @@ -17,10 +17,14 @@ def __init__(self, values=None, extra_sources=()): sources = self._getsources() default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8') self.cp = SafeConfigParser() - self.cp.readfp(StringIO(default_config)) - self.cp.read(sources) - for fp in extra_sources: - self.cp.readfp(fp) + self.cp.readfp(io.StringIO(default_config)) + sources.extend(extra_sources) + for fname in sources: + try: + with io.open(fname) as fp: + self.cp.readfp(fp) + except (IOError, OSError): + pass else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) diff --git a/scrapyd/poller.py b/scrapyd/poller.py index ee4e30cd..aa4097f7 100644 --- a/scrapyd/poller.py +++ b/scrapyd/poller.py @@ -11,11 +11,11 @@ class QueuePoller(object): def __init__(self, config): self.config = config self.update_projects() - self.dq = DeferredQueue(size=1) + self.dq = DeferredQueue() @inlineCallbacks def poll(self): - if self.dq.pending: + if not self.dq.waiting: return for p, q in iteritems(self.queues): c = yield maybeDeferred(q.count) diff --git a/scrapyd/tests/mybot.egg b/scrapyd/tests/mybot.egg index 8525589b..6acb11b0 100644 Binary files a/scrapyd/tests/mybot.egg and b/scrapyd/tests/mybot.egg differ diff --git a/scrapyd/tests/mybot2.egg b/scrapyd/tests/mybot2.egg index c0e615e6..78ef8e72 100644 Binary files a/scrapyd/tests/mybot2.egg and b/scrapyd/tests/mybot2.egg differ diff --git a/scrapyd/tests/mybotunicode.egg b/scrapyd/tests/mybotunicode.egg index 60df5000..8891f915 100644 Binary files a/scrapyd/tests/mybotunicode.egg and b/scrapyd/tests/mybotunicode.egg differ diff --git a/scrapyd/utils.py b/scrapyd/utils.py index 2c6293d0..c9377226 100644 --- a/scrapyd/utils.py +++ b/scrapyd/utils.py @@ -26,7 +26,7 @@ def render_object(self, obj, txrequest): txrequest.setHeader('Access-Control-Allow-Origin', '*') txrequest.setHeader('Access-Control-Allow-Methods', 'GET, POST, PATCH, PUT, DELETE') txrequest.setHeader('Access-Control-Allow-Headers',' X-Requested-With') - txrequest.setHeader('Content-Length', len(r)) + txrequest.setHeader('Content-Length', str(len(r))) return r class UtilsCache: diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py index 5cb7562a..15c56b5f 100644 --- a/scrapyd/webservice.py +++ b/scrapyd/webservice.py @@ -79,9 +79,10 @@ def render_POST(self, txrequest): class AddVersion(WsResource): def render_POST(self, txrequest): - project = txrequest.args[b'project'][0].decode('utf-8') - version = txrequest.args[b'version'][0].decode('utf-8') - eggf = BytesIO(txrequest.args[b'egg'][0]) + eggf = BytesIO(txrequest.args.pop(b'egg')[0]) + args = native_stringify_dict(copy(txrequest.args), keys_only=False) + project = args['project'][0] + version = args['version'][0] self.root.eggstorage.put(eggf, project, version) spiders = get_spider_list(project, version=version) self.root.update_projects() diff --git a/scrapyd/website.py b/scrapyd/website.py index 6f5e189c..05abff61 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -154,6 +154,6 @@ def render(self, txrequest): s += "" txrequest.setHeader('Content-Type', 'text/html; charset=utf-8') - txrequest.setHeader('Content-Length', len(s)) + txrequest.setHeader('Content-Length', str(len(s))) return s.encode('utf-8')