From d49f0554159dec27d080d9f5163adc99a906dbf7 Mon Sep 17 00:00:00 2001 From: iceboy Date: Sat, 1 Apr 2017 16:45:46 -0700 Subject: [PATCH 1/8] remove usages of app.loop since it's deprecated in aiohttp 2.0 --- requirements.txt | 2 +- vj4/app.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index e212c048..3615a593 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -aiohttp<2.0.0 # sockjs doesn't support 2.0+ +aiohttp>=2.0.5 jinja2>=2.9.0 sockjs>=0.5.0 motor diff --git a/vj4/app.py b/vj4/app.py index ea1d692d..38cff9c5 100644 --- a/vj4/app.py +++ b/vj4/app.py @@ -1,5 +1,4 @@ import asyncio -import functools import logging from os import path @@ -47,7 +46,8 @@ def __init__(self): # Initialize components. staticmanifest.init(static_path) locale.load_translations(translation_path) - self.loop.run_until_complete(asyncio.gather(tools.ensure_all_indexes(), bus.init())) + asyncio.get_event_loop().run_until_complete( + asyncio.gather(tools.ensure_all_indexes(), bus.init())) smallcache.init() # Load views. @@ -101,12 +101,12 @@ def get(self, id, create=False, request=None): timeout=self.timeout, loop=self.loop, debug=self.debug)) return self[id] + loop = asyncio.get_event_loop() sockjs.add_endpoint(Application(), handler, name=name, prefix=prefix, - manager=Manager(name, Application(), handler, Application().loop)) - sockjs.add_endpoint(Application(), handler, - name=name + '_with_domain_id', prefix='/d/{domain_id}' + prefix, - manager=Manager(name + '_with_domain_id', Application(), handler, - Application().loop)) + manager=Manager(name, Application(), handler, loop)) + sockjs.add_endpoint( + Application(), handler, name=name + '_with_domain_id', prefix='/d/{domain_id}' + prefix, + manager=Manager(name + '_with_domain_id', Application(), handler, loop)) return conn return decorate From 333526b4c05dcb513984b12cb2ffb8dfc52e7844 Mon Sep 17 00:00:00 2001 From: iceboy Date: Sat, 1 Apr 2017 19:40:47 -0700 Subject: [PATCH 2/8] use sockjs at head to support aiohttp 2.0. --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3615a593..52e6ad11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ aiohttp>=2.0.5 jinja2>=2.9.0 -sockjs>=0.5.0 +# sockjs at head supports aiohttp 2.0, but not released yet. +git+https://github.com/aio-libs/sockjs.git motor hoedown accept From 267b29b99f4b195e932f7fdae70334606732979d Mon Sep 17 00:00:00 2001 From: Wende Tan Date: Sun, 2 Apr 2017 13:47:17 +0800 Subject: [PATCH 3/8] fix pretest --- vj4/handler/problem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vj4/handler/problem.py b/vj4/handler/problem.py index 15cb2fd2..d40d52a5 100644 --- a/vj4/handler/problem.py +++ b/vj4/handler/problem.py @@ -233,8 +233,8 @@ async def post(self, *, pid: document.convert_doc_id, lang: str, code: str, pdoc = await problem.get(self.domain_id, pid) # don't need to check hidden status # create zip file, TODO(twd2): check file size - content = list(zip(self.request.POST.getall('data_input'), - self.request.POST.getall('data_output'))) + content = list(zip((await self.request.post()).getall('data_input'), + (await self.request.post()).getall('data_output'))) output_buffer = io.BytesIO() zip_file = zipfile.ZipFile(output_buffer, 'a', zipfile.ZIP_DEFLATED) config_content = str(len(content)) + '\n' From 47b60c5de637a8d75c44d7bcd0cb2755a4a80b55 Mon Sep 17 00:00:00 2001 From: Wende Tan Date: Sun, 2 Apr 2017 13:54:44 +0800 Subject: [PATCH 4/8] fix domain --- vj4/handler/domain.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vj4/handler/domain.py b/vj4/handler/domain.py index d2be3698..2eaf6114 100644 --- a/vj4/handler/domain.py +++ b/vj4/handler/domain.py @@ -119,9 +119,9 @@ async def post_set_user(self, *, uid: int, role: str): @base.require_perm(builtin.PERM_EDIT_PERM) @base.require_csrf_token @base.sanitize - async def post_set_users(self, *, uid: int, role: str): + async def post_set_users(self, *, uid: int, role: str=None): try: - uids = map(int, self.request.POST.getall('uid')) + uids = map(int, (await self.request.post()).getall('uid')) except ValueError: raise error.ValidationError('uid') if role: @@ -148,7 +148,7 @@ async def post(self, **kwargs): new_roles = dict() for role in self.domain['roles']: perms = 0 - for perm in self.request.POST.getall(role, []): + for perm in (await self.request.post()).getall(role, []): perm = int(perm) if perm in builtin.PERMS_BY_KEY: perms |= perm @@ -181,5 +181,5 @@ async def post_set(self, *, role: str, perm: int=builtin.DEFAULT_PERMISSIONS): @base.require_csrf_token @base.sanitize async def post_delete(self, *, role: str): - await domain.delete_roles(self.domain_id, self.request.POST.getall('role')) + await domain.delete_roles(self.domain_id, (await self.request.post()).getall('role')) self.json_or_redirect(self.url) From d8edf783b86fcefe99f2d1def3c5acd76a032be8 Mon Sep 17 00:00:00 2001 From: Wende Tan Date: Sun, 2 Apr 2017 13:58:42 +0800 Subject: [PATCH 5/8] fix GET --- vj4/handler/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vj4/handler/base.py b/vj4/handler/base.py index 728a225d..2971a7f1 100644 --- a/vj4/handler/base.py +++ b/vj4/handler/base.py @@ -413,7 +413,7 @@ def wrapped(self, **kwargs): def get_argument(func): @functools.wraps(func) def wrapped(self, **kwargs): - return func(self, **kwargs, **self.request.GET) + return func(self, **kwargs, **self.request.query) return wrapped From 52383c7ff04722480d4cd0e9a33aa10ce7ed66ca Mon Sep 17 00:00:00 2001 From: iceboy Date: Sat, 1 Apr 2017 22:57:33 -0700 Subject: [PATCH 6/8] fix twd2 --- vj4/handler/fs.py | 3 ++- vj4/handler/problem.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vj4/handler/fs.py b/vj4/handler/fs.py index 6c83e933..5587e153 100644 --- a/vj4/handler/fs.py +++ b/vj4/handler/fs.py @@ -101,7 +101,8 @@ async def stream_data(self, *, secret: str, headers_only: bool=False): grid_out = await fs.get_by_secret(secret) self.response.content_type = grid_out.content_type or 'application/octet-stream' - self.response.content_length = grid_out.length + # FIXME(iceboy): For some reason setting response.content_length doesn't work in aiohttp 2.0.5. + self.response.headers['Content-Length'] = str(grid_out.length) # Cache control. self.response.last_modified = grid_out.upload_date diff --git a/vj4/handler/problem.py b/vj4/handler/problem.py index d40d52a5..b634afc7 100644 --- a/vj4/handler/problem.py +++ b/vj4/handler/problem.py @@ -233,8 +233,8 @@ async def post(self, *, pid: document.convert_doc_id, lang: str, code: str, pdoc = await problem.get(self.domain_id, pid) # don't need to check hidden status # create zip file, TODO(twd2): check file size - content = list(zip((await self.request.post()).getall('data_input'), - (await self.request.post()).getall('data_output'))) + post = await self.request.post() + content = list(zip(post.getall('data_input'), post.getall('data_output'))) output_buffer = io.BytesIO() zip_file = zipfile.ZipFile(output_buffer, 'a', zipfile.ZIP_DEFLATED) config_content = str(len(content)) + '\n' From 74de35e4435fc45817b53c472b27e77b9df5a772 Mon Sep 17 00:00:00 2001 From: iceboy Date: Sat, 1 Apr 2017 23:07:26 -0700 Subject: [PATCH 7/8] fix some errors --- vj4/handler/problem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vj4/handler/problem.py b/vj4/handler/problem.py index b634afc7..ce6e89f4 100644 --- a/vj4/handler/problem.py +++ b/vj4/handler/problem.py @@ -374,7 +374,7 @@ async def post_edit_reply(self, *, pid: document.convert_doc_id, self.check_perm(builtin.PERM_VIEW_PROBLEM_HIDDEN) psdoc, psrdoc = await problem.get_solution_reply(self.domain_id, psid, psrid) if not psdoc or psdoc['parent_doc_id'] != pdoc['doc_id']: - raise error.DocumentNotFoundError(domain_id, document.TYPE_PROBLEM_SOLUTION, psid) + raise error.DocumentNotFoundError(self.domain_id, document.TYPE_PROBLEM_SOLUTION, psid) if not self.own(psrdoc, builtin.PERM_EDIT_PROBLEM_SOLUTION_REPLY_SELF): self.check_perm(builtin.PERM_EDIT_PROBLEM_SOLUTION_REPLY) await problem.edit_solution_reply(self.domain_id, psid, psrid, content) @@ -391,7 +391,7 @@ async def post_delete_reply(self, *, pid: document.convert_doc_id, self.check_perm(builtin.PERM_VIEW_PROBLEM_HIDDEN) psdoc, psrdoc = await problem.get_solution_reply(self.domain_id, psid, psrid) if not psdoc or psdoc['parent_doc_id'] != pdoc['doc_id']: - raise error.DocumentNotFoundError(domain_id, document.TYPE_PROBLEM_SOLUTION, psid) + raise error.DocumentNotFoundError(self.domain_id, document.TYPE_PROBLEM_SOLUTION, psid) if not self.own(psrdoc, builtin.PERM_DELETE_PROBLEM_SOLUTION_REPLY_SELF): self.check_perm(builtin.PERM_DELETE_PROBLEM_SOLUTION_REPLY) await oplog.add(self.user['_id'], oplog.TYPE_DELETE_SUB_DOCUMENT, sub_doc=psrdoc, @@ -462,7 +462,7 @@ async def get(self, *, pid: document.convert_doc_id, psid: document.convert_doc_ self.check_perm(builtin.PERM_VIEW_PROBLEM_HIDDEN) psdoc, psrdoc = await problem.get_solution_reply(self.domain_id, psid, psrid) if not psdoc or psdoc['parent_doc_id'] != pdoc['doc_id']: - raise error.DocumentNotFoundError(domain_id, document.TYPE_PROBLEM_SOLUTION, psid) + raise error.DocumentNotFoundError(self.domain_id, document.TYPE_PROBLEM_SOLUTION, psid) self.response.content_type = 'text/markdown' self.response.text = psrdoc['content'] @@ -570,7 +570,7 @@ async def post(self, *, pid: document.convert_doc_id, hidden: bool=False, category = self.split_tags(category) tag = self.split_tags(tag) for c in category: - if not (c in builtin.PROBLEM_CATEGORIES \ + if not (c in builtin.PROBLEM_CATEGORIES or c in builtin.PROBLEM_SUB_CATEGORIES): raise error.ValidationError('category') if difficulty_setting not in problem.SETTING_DIFFICULTY_RANGE: From 218244cc41aafdde4a9f5b7cdf85da49f9e5d458 Mon Sep 17 00:00:00 2001 From: iceboy Date: Sat, 1 Apr 2017 23:10:00 -0700 Subject: [PATCH 8/8] remove unused imports --- vj4/handler/judge.py | 1 - vj4/handler/problem.py | 1 - vj4/handler/training.py | 1 - vj4/model/adaptor/discussion.py | 1 - vj4/model/adaptor/training.py | 3 --- vj4/model/adaptor/userfile.py | 3 --- vj4/model/builtin.py | 1 - vj4/model/oplog.py | 2 -- vj4/service/staticmanifest.py | 4 +--- vj4/template.py | 1 - vj4/util/validator.py | 1 - 11 files changed, 1 insertion(+), 18 deletions(-) diff --git a/vj4/handler/judge.py b/vj4/handler/judge.py index e7f7b0c5..65755659 100644 --- a/vj4/handler/judge.py +++ b/vj4/handler/judge.py @@ -2,7 +2,6 @@ import calendar import datetime import logging -import time from bson import objectid from vj4 import app diff --git a/vj4/handler/problem.py b/vj4/handler/problem.py index ce6e89f4..efd982a3 100644 --- a/vj4/handler/problem.py +++ b/vj4/handler/problem.py @@ -1,7 +1,6 @@ import asyncio import datetime import functools -import hashlib import io import zipfile from bson import objectid diff --git a/vj4/handler/training.py b/vj4/handler/training.py index 20b3e5c8..fae755ff 100644 --- a/vj4/handler/training.py +++ b/vj4/handler/training.py @@ -1,5 +1,4 @@ import asyncio -import time from json import decoder from bson import objectid diff --git a/vj4/model/adaptor/discussion.py b/vj4/model/adaptor/discussion.py index e94c84ed..fc38aced 100644 --- a/vj4/model/adaptor/discussion.py +++ b/vj4/model/adaptor/discussion.py @@ -6,7 +6,6 @@ from vj4 import error from vj4.model import document -from vj4.model.adaptor import problem from vj4.service import smallcache from vj4.util import argmethod from vj4.util import validator diff --git a/vj4/model/adaptor/training.py b/vj4/model/adaptor/training.py index eb50f616..7ebea8fa 100644 --- a/vj4/model/adaptor/training.py +++ b/vj4/model/adaptor/training.py @@ -1,11 +1,8 @@ -import asyncio from bson import objectid from pymongo import errors -from vj4 import constant from vj4 import error from vj4.model import document -from vj4.model.adaptor import problem from vj4.util import argmethod from vj4.util import validator diff --git a/vj4/model/adaptor/userfile.py b/vj4/model/adaptor/userfile.py index 54861cb1..2eea763d 100644 --- a/vj4/model/adaptor/userfile.py +++ b/vj4/model/adaptor/userfile.py @@ -1,8 +1,5 @@ -import asyncio from bson import objectid -from pymongo import errors -from vj4 import constant from vj4 import error from vj4.model import builtin from vj4.model import document diff --git a/vj4/model/builtin.py b/vj4/model/builtin.py index 754a3836..7412f5ae 100644 --- a/vj4/model/builtin.py +++ b/vj4/model/builtin.py @@ -4,7 +4,6 @@ import itertools from vj4 import constant -from vj4 import error from vj4.util import version # Permissions. diff --git a/vj4/model/oplog.py b/vj4/model/oplog.py index 85d4ca27..0778ccf8 100644 --- a/vj4/model/oplog.py +++ b/vj4/model/oplog.py @@ -1,6 +1,4 @@ -import itertools from bson import objectid -from pymongo import ReturnDocument from vj4 import db from vj4.util import argmethod diff --git a/vj4/service/staticmanifest.py b/vj4/service/staticmanifest.py index 363bf37d..479997b6 100644 --- a/vj4/service/staticmanifest.py +++ b/vj4/service/staticmanifest.py @@ -1,5 +1,3 @@ -import time - from os import path from vj4.util import json @@ -18,7 +16,7 @@ def init(static_dir): with open(_manifest_path, 'r') as manifest_file: data = json.decode(manifest_file.read()) _manifest = data - except Exception as e: + except Exception: pass diff --git a/vj4/template.py b/vj4/template.py index a2b8e16b..601520d5 100644 --- a/vj4/template.py +++ b/vj4/template.py @@ -15,7 +15,6 @@ from vj4.service import staticmanifest from vj4.util import json from vj4.util import options -from vj4.util import version class Undefined(jinja2.runtime.Undefined): diff --git a/vj4/util/validator.py b/vj4/util/validator.py index 593f81cb..32aa2ef8 100644 --- a/vj4/util/validator.py +++ b/vj4/util/validator.py @@ -2,7 +2,6 @@ from vj4 import constant from vj4 import error -from vj4.model import builtin def is_uid(s):