From df97d3cef564550686838fe6fcb9750d36bdc2bf Mon Sep 17 00:00:00 2001 From: tc-imba Date: Mon, 30 Sep 2019 21:42:07 +0800 Subject: [PATCH] feature: add moss test --- requirements.txt | 1 + vj4/app.py | 3 ++ vj4/handler/contest.py | 18 ++++++- vj4/model/adaptor/contest.py | 4 ++ vj4/model/adaptor/moss.py | 46 +++++++++++++----- vj4/ui/constant/language.js | 56 ++++++++++++---------- vj4/ui/pages/contest_system_test.page.js | 18 +++++++ vj4/ui/templates/homework_system_test.html | 10 ++++ 8 files changed, 119 insertions(+), 37 deletions(-) create mode 100644 vj4/ui/pages/contest_system_test.page.js diff --git a/requirements.txt b/requirements.txt index 1b7ef5f0..bbfea17d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ PyYAML git+https://github.com/iceb0y/aiomongo oauthlib mosspy +rarfile diff --git a/vj4/app.py b/vj4/app.py index d5ede1d1..82751b0f 100644 --- a/vj4/app.py +++ b/vj4/app.py @@ -39,6 +39,9 @@ options.define('oauth_client_id', default='', help='OAuth client id') options.define('oauth_client_secret', default='', help='OAuth client secret') +# moss user id +options.define('moss_user_id', default=987654321, help='Stanford Moss User ID') + _logger = logging.getLogger(__name__) diff --git a/vj4/handler/contest.py b/vj4/handler/contest.py index 50a8e99b..9350398c 100644 --- a/vj4/handler/contest.py +++ b/vj4/handler/contest.py @@ -5,6 +5,7 @@ import datetime import functools import io +import logging import pytz import yaml import zipfile @@ -26,6 +27,8 @@ from vj4.util import pagination from vj4.util.misc import filter_language, filter_content_type +_logger = logging.getLogger(__name__) + def _parse_pids(pids_str): pid_list = list(map(document.convert_doc_id, pids_str.split(','))) @@ -931,6 +934,10 @@ async def _post_homework(self, *, ctype: str, tid: objectid.ObjectId): @app.route('/{ctype:contest|homework}/{tid}/moss', 'contest_moss') class ContestMosstHandler(ContestMixin, base.Handler): + def split_tags(self, s): + s = s.replace('，', ',') # Chinese ', ' + return list(filter(lambda _: _ != '', map(lambda _: _.strip(), s.split(',')))) + @base.route_argument async def post(self, *, ctype: str, **kwargs): if ctype == 'homework': @@ -947,12 +954,19 @@ async def post(self, *, ctype: str, **kwargs): @base.post_argument @base.require_csrf_token @base.sanitize - async def _post_homework(self, *, ctype: str, tid: objectid.ObjectId): + async def _post_homework(self, *, ctype: str, tid: objectid.ObjectId, language: str, wildcards: str, + ignore_limit: int = 10): tdoc = await contest.get(self.domain_id, document.TYPE_HOMEWORK, tid) if not self.own(tdoc, builtin.PERM_EDIT_HOMEWORK_SELF): self.check_perm(builtin.PERM_EDIT_HOMEWORK) doc_type = constant.contest.CTYPE_TO_DOCTYPE[ctype] rdocs = await self.get_latest_records(doc_type, tid) - data = await moss.moss_test(rdocs, language='cc') + wildcards = self.split_tags(wildcards) + + _logger.info('Submit Moss for %s', tid) + moss_url = await moss.moss_test(rdocs, language=language, wildcards=wildcards) + if moss_url: + await contest.update_moss_result(self.domain_id, document.TYPE_HOMEWORK, tid, moss_url=moss_url) + self.json_or_redirect(self.reverse_url('contest_system_test', ctype=ctype, tid=tid)) diff --git a/vj4/model/adaptor/contest.py b/vj4/model/adaptor/contest.py index 98dab2a4..7bb981ff 100644 --- a/vj4/model/adaptor/contest.py +++ b/vj4/model/adaptor/contest.py @@ -348,6 +348,10 @@ async def edit(domain_id: str, doc_type: int, tid: objectid.ObjectId, **kwargs): return await document.set(domain_id, doc_type, tid, **kwargs) +async def update_moss_result(domain_id: str, doc_type: int, tid: objectid.ObjectId, moss_url: str): + return await document.set(domain_id, doc_type, tid, moss_url=moss_url) + + def get_multi(domain_id: str, doc_type: int, fields=None, **kwargs): # TODO(twd2): projection. return document.get_multi(domain_id=domain_id, diff --git a/vj4/model/adaptor/moss.py b/vj4/model/adaptor/moss.py index a1d9c863..a7d8ee3d 100644 --- a/vj4/model/adaptor/moss.py +++ b/vj4/model/adaptor/moss.py @@ -1,3 +1,4 @@ +from bson import objectid from os import path, mkdir from shutil import rmtree from tempfile import mkdtemp @@ -10,13 +11,18 @@ from vj4 import db from vj4 import error from vj4.model import fs +from vj4.util import options # @TODO(tc-imba) the extraction is not safe now. def extract_text_file(file_obj, dest_dir, lang): - # @TODO add extension - with open(path.join(dest_dir, 'main' + '.cpp'), mode='wb') as file: + wildcards = constant.language.LANG_MOSS_WILDCARDS.get(lang, []) + if wildcards: + name = wildcards[0].replace('*', 'main') + else: + name = 'main.txt' + with open(path.join(dest_dir, name), mode='wb') as file: file.write(file_obj.read()) @@ -43,22 +49,40 @@ def extract_rar_file(file_obj, dest_dir, lang): } -async def moss_test(rdocs: list, language: str): +async def moss_test(rdocs: list, language: str, ignore_limit: int = 10, wildcards: list = None): # check the language is supported by moss if language not in constant.language.LANG_MOSS: raise error.LanguageNotSupportedError(language) + ignore_limit = int(ignore_limit) + if ignore_limit < 2: + raise error.InvalidArgumentError('ignore_limit') + + if wildcards is None or len(wildcards) == 0: + wildcards = constant.language.LANG_MOSS_WILDCARDS.get(language, []) + moss_dir = mkdtemp(prefix='cb4.moss.') try: + moss = Moss(options.moss_user_id, language) + moss.setDirectoryMode(1) + moss.setIgnoreLimit(ignore_limit) + for rdoc in rdocs: - grid_out = await fs.get(rdoc['code']) - file_obj = BytesIO(await grid_out.read()) - dest_dir = path.join(moss_dir, str(rdoc['uid'])) - mkdir(dest_dir) if rdoc['code_type'] in EXTRACT_OPEN_FUNC: - EXTRACT_OPEN_FUNC[rdoc['code_type']](file_obj, dest_dir, language) - + try: + grid_out = await fs.get(rdoc['code']) + file_obj = BytesIO(await grid_out.read()) + dest_dir = path.join(moss_dir, str(rdoc['uid'])) + mkdir(dest_dir) + EXTRACT_OPEN_FUNC[rdoc['code_type']](file_obj, dest_dir, language) + for wildcard in wildcards: + moss.addFilesByWildcard(path.join(dest_dir, wildcard)) + except Exception as e: + print(e) + + url = moss.send() + return url + finally: - print(moss_dir) + # print(moss_dir) rmtree(moss_dir, ignore_errors=True) - diff --git a/vj4/ui/constant/language.js b/vj4/ui/constant/language.js index 071b3435..0f38e4ac 100644 --- a/vj4/ui/constant/language.js +++ b/vj4/ui/constant/language.js @@ -71,28 +71,36 @@ export const LANG_CODEMIRROR_MODES = { attachObjectMeta(LANG_CODEMIRROR_MODES, 'exportToPython', false); export const LANG_MOSS = { - c: 'c', - cc: 'cc', - java: 'java', - ml: 'ml', - pascal: 'pascal', - ada: 'ada', - lisp: 'lisp', - scheme: 'scheme', - haskell: 'haskell', - fortran: 'fortran', - ascii: 'ascii', - vhdl: 'vhdl', - perl: 'perl', - matlab: 'matlab', - python: 'python', - mips: 'mips', - prolog: 'prolog', - spice: 'spice', - vb: 'vb', - csharp: 'csharp', - modula2: 'modula2', - a8086: 'a8086', - javascript: 'javascript', - plsql: 'plsql', + c: 'C', + cc: 'C++', + java: 'Java', + // ml: 'ml', + // pascal: 'pascal', + // ada: 'ada', + // lisp: 'lisp', + // scheme: 'scheme', + // haskell: 'haskell', + // fortran: 'fortran', + // ascii: 'ascii', + // vhdl: 'vhdl', + // perl: 'perl', + matlab: 'MATLAB / GNU Octave', + python: 'Python', + // mips: 'mips', + // prolog: 'prolog', + // spice: 'spice', + // vb: 'vb', + // csharp: 'csharp', + // modula2: 'modula2', + // a8086: 'a8086', + // javascript: 'javascript', + // plsql: 'plsql', +}; + +export const LANG_MOSS_WILDCARDS = { + c: ['*.c', '*.h'], + cc: ['*.cpp', '*.cc', '*.c', '*.h', '*.hh', '*.hpp'], + java: ['*.java'], + matlab: ['*.m'], + python: ['*.py'], }; diff --git a/vj4/ui/pages/contest_system_test.page.js b/vj4/ui/pages/contest_system_test.page.js new file mode 100644 index 00000000..08b7c9f3 --- /dev/null +++ b/vj4/ui/pages/contest_system_test.page.js @@ -0,0 +1,18 @@ +import { NamedPage } from 'vj/misc/PageLoader'; +import { LANG_MOSS_WILDCARDS } from 'vj/constant/language'; + +const page = new NamedPage('contest_system_test', async () => { + const $language = $('[name="language"]'); + const $wildcards = $('[name="wildcards"]'); + + const changeWildcards = () => { + const lang = $language.val(); + const wildcards = LANG_MOSS_WILDCARDS[lang] || []; + $wildcards.val(wildcards.join(', ')); + }; + + $language.on('change', changeWildcards); + changeWildcards(); +}); + +export default page; diff --git a/vj4/ui/templates/homework_system_test.html b/vj4/ui/templates/homework_system_test.html index 22f9ca86..d68d3ba3 100644 --- a/vj4/ui/templates/homework_system_test.html +++ b/vj4/ui/templates/homework_system_test.html @@ -42,8 +42,18 @@

{{ _('Export') }}

{{ _('Moss') }}

+ {% if 'moss_url' in tdoc %} +
+ Last result: {{ tdoc['moss_url'] }} +
+ {% endif %}
+
+ {{ form.form_select2(columns=3, label='Code language', name='language', options=vj4.constant.language.LANG_MOSS.items(), row=false) }} + {{ form.form_text(columns=6, label='Filter Wildcards', help_text='Splitted by \', \'. Use default settings of the selected language if empty.', name='wildcards', value='', row=false) }} + {{ form.form_text(columns=3, label='Ignore Threshold', help_text='Ignore if a range of code appears in a number of submissions. Minimum is 2. Default is 10.', name='ignore_limit', value=10, row=false) }} +