From 576aaf668d8ddc573cb210616f37d61a3d76d7a3 Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 24 Sep 2016 09:02:25 +0200 Subject: [PATCH 1/3] cache (and compile) parse_template in run_view_in, fix #1474 --- applications/welcome/controllers/appadmin.py | 2 +- gluon/compileapp.py | 68 ++++++-------------- gluon/restricted.py | 8 +-- gluon/template.py | 16 ++--- gluon/tests/test_appadmin.py | 7 ++ 5 files changed, 33 insertions(+), 68 deletions(-) diff --git a/applications/welcome/controllers/appadmin.py b/applications/welcome/controllers/appadmin.py index 0b8b227f9..4d4e965fb 100644 --- a/applications/welcome/controllers/appadmin.py +++ b/applications/welcome/controllers/appadmin.py @@ -409,7 +409,7 @@ def ccache(): import copy import time import math - from gluon import portalocker + from pydal.contrib import portalocker ram = { 'entries': 0, diff --git a/gluon/compileapp.py b/gluon/compileapp.py index 459c2848c..b3d9cf61f 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -430,13 +430,10 @@ def build_environment(request, response, session, store_current=True): current.T = t current.cache = c - global __builtins__ if is_jython: # jython hack + global __builtins__ __builtins__ = mybuiltin() - elif is_pypy: # apply the same hack to pypy too - __builtins__ = mybuiltin() - elif PY2: - __builtins__['__import__'] = builtin.__import__ # WHY? + environment['request'] = request environment['response'] = response environment['session'] = session @@ -487,7 +484,7 @@ def compile_views(folder, skip_failed_views=False): else: raise Exception("%s in %s" % (e, fname)) else: - filename = ('views/%s.py' % fname).replace('/', '_').replace('\\', '_') + filename = 'views.%s.py' % fname.replace(os.path.sep, '.') filename = pjoin(folder, 'compiled', filename) write_file(filename, data) save_pyc(filename) @@ -503,7 +500,7 @@ def compile_models(folder): path = pjoin(folder, 'models') for fname in listdir(path, '.+\.py$'): data = read_file(pjoin(path, fname)) - modelfile = 'models.'+fname.replace(os.path.sep,'.') + modelfile = 'models.'+fname.replace(os.path.sep, '.') filename = pjoin(folder, 'compiled', modelfile) mktree(filename) write_file(filename, data) @@ -582,11 +579,10 @@ def run_models_in(environment): continue elif compiled: code = getcfs(model, model, lambda: read_pyc(model)) - elif is_gae: + else: code = getcfs(model, model, lambda: compile2(read_file(model), model)) - else: - code = getcfs(model, model, None) + restricted(code, environment, layer=model) @@ -610,7 +606,6 @@ def run_controller_in(controller, function, environment): rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) code = getcfs(filename, filename, lambda: read_pyc(filename)) - restricted(code, environment, layer=filename) elif function == '_TEST': # TESTING: adjust the path to include site packages from gluon.settings import global_settings @@ -629,7 +624,6 @@ def run_controller_in(controller, function, environment): environment['__symbols__'] = environment.keys() code = read_file(filename) code += TEST_CODE - restricted(code, environment, layer=filename) else: filename = pjoin(folder, 'controllers/%s.py' % controller) @@ -644,10 +638,10 @@ def run_controller_in(controller, function, environment): rewrite.THREAD_LOCAL.routes.error_message % badf, web2py_error=badf) code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function) - if is_gae: - layer = filename + ':' + function - code = getcfs(layer, filename, lambda: compile2(code, layer)) - restricted(code, environment, filename) + layer = filename + ':' + function + code = getcfs(layer, filename, lambda: compile2(code, layer)) + + restricted(code, environment, layer=filename) response = current.response vars = response._vars if response.postprocessing: @@ -682,7 +676,7 @@ def run_view_in(environment): if not isinstance(view, str): ccode = parse_template(view, pjoin(folder, 'views'), context=environment) - restricted(ccode, environment, 'file stream') + layer = 'file stream' else: filename = pjoin(folder, 'views', view) if os.path.exists(path): # compiled views @@ -713,16 +707,13 @@ def run_view_in(environment): rewrite.THREAD_LOCAL.routes.error_message % badv, web2py_error=badv) layer = filename - if is_gae: - ccode = getcfs(layer, filename, - lambda: compile2(parse_template(view, - pjoin(folder, 'views'), - context=environment), layer)) - else: - ccode = parse_template(view, - pjoin(folder, 'views'), - context=environment) - restricted(ccode, environment, layer) + # Cache the compiled template + ccode = getcfs(layer, filename, + lambda: compile2(parse_template(view, + pjoin(folder, 'views'), + context=environment), + layer)) + restricted(ccode, environment, layer=layer) def remove_compiled_application(folder): @@ -748,26 +739,3 @@ def compile_application(folder, skip_failed_views=False): compile_controllers(folder) failed_views = compile_views(folder, skip_failed_views) return failed_views - - -def test(): - """ - Example:: - - >>> import traceback, types - >>> environment={'x':1} - >>> open('a.py', 'w').write('print 1/x') - >>> save_pyc('a.py') - >>> os.unlink('a.py') - >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code' - code - >>> exec read_pyc('a.pyc') in environment - 1 - """ - - return - - -if __name__ == '__main__': - import doctest - doctest.testmod() diff --git a/gluon/restricted.py b/gluon/restricted.py index c0725a151..84c46b36e 100644 --- a/gluon/restricted.py +++ b/gluon/restricted.py @@ -202,7 +202,7 @@ def compile2(code, layer): return compile(code, layer, 'exec') -def restricted(code, environment=None, layer='Unknown'): +def restricted(ccode, environment=None, layer='Unknown'): """ Runs code in environment and returns the output. If an exception occurs in code it raises a RestrictedError containing the traceback. Layer is @@ -213,10 +213,6 @@ def restricted(code, environment=None, layer='Unknown'): environment['__file__'] = layer environment['__name__'] = '__restricted__' try: - if isinstance(code, types.CodeType): - ccode = code - else: - ccode = compile2(code, layer) exec(ccode, environment) except HTTP: raise @@ -231,7 +227,7 @@ def restricted(code, environment=None, layer='Unknown'): sys.excepthook(etype, evalue, tb) del tb output = "%s %s" % (etype, evalue) - raise RestrictedError(layer, code, output, environment) + raise RestrictedError(layer, ccode, output, environment) def snapshot(info=None, context=5, code=None, environment=None): diff --git a/gluon/template.py b/gluon/template.py index e620df97d..41da7f2d6 100644 --- a/gluon/template.py +++ b/gluon/template.py @@ -24,9 +24,6 @@ # have web2py from gluon.restricted import RestrictedError from gluon.globals import current - from gluon.cfs import getcfs - from gluon.fileutils import read_file - HAS_CFS = True except ImportError: # do not have web2py current = None @@ -783,14 +780,11 @@ def parse_template(filename, # First, if we have a str try to open the file if isinstance(filename, str): fname = os.path.join(path, filename) - if HAS_CFS: - text = getcfs(fname, fname, lambda: read_file(fname)) - else: - try: - with open(fname, 'rb') as fp: - text = fp.read() - except IOError: - raise RestrictedError(filename, '', 'Unable to find the file') + try: + with open(fname, 'rb') as fp: + text = fp.read() + except IOError: + raise RestrictedError(filename, '', 'Unable to find the file') else: text = filename.read() text = to_native(text) diff --git a/gluon/tests/test_appadmin.py b/gluon/tests/test_appadmin.py index 77bb36d62..c5e7402ed 100644 --- a/gluon/tests/test_appadmin.py +++ b/gluon/tests/test_appadmin.py @@ -16,6 +16,7 @@ from gluon import fileutils from gluon.dal import DAL, Field, Table from gluon.http import HTTP +from gluon.fileutils import open_file DEFAULT_URI = os.getenv('DB', 'sqlite:memory') @@ -76,12 +77,18 @@ def run_function(self): def run_view(self): return run_view_in(self.env) + def run_view_file_stream(self): + view_path = os.path.join(self.env['request'].folder, 'views', 'appadmin.html') + self.env['response'].view = open_file(view_path, 'r') + return run_view_in(self.env) + def _test_index(self): result = self.run_function() self.assertTrue('db' in result['databases']) self.env.update(result) try: self.run_view() + self.run_view_file_stream() except Exception as e: print(e.message) self.fail('Could not make the view') From b4acfd0724845633f667a55e593a15576379554d Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sun, 25 Sep 2016 10:11:47 +0200 Subject: [PATCH 2/3] removed unnecessary env clone for _view_environment --- gluon/compileapp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gluon/compileapp.py b/gluon/compileapp.py index b3d9cf61f..23e666280 100644 --- a/gluon/compileapp.py +++ b/gluon/compileapp.py @@ -441,7 +441,6 @@ def build_environment(request, response, session, store_current=True): lambda name, reload=False, app=request.application:\ local_import_aux(name, reload, app) BaseAdapter.set_folder(pjoin(request.folder, 'databases')) - response._view_environment = copy.copy(environment) custom_import_install() return environment From 4468355d013055be44913d1b2da57cadb360fe7f Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sun, 25 Sep 2016 10:43:42 +0200 Subject: [PATCH 3/3] fix py3 xmlrpc imports, close #1473 --- applications/admin/controllers/pythonanywhere.py | 4 +--- gluon/_compat.py | 2 ++ gluon/contrib/simplejsonrpc.py | 11 ++++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/applications/admin/controllers/pythonanywhere.py b/applications/admin/controllers/pythonanywhere.py index 33dfda6fd..e61b06ab1 100644 --- a/applications/admin/controllers/pythonanywhere.py +++ b/applications/admin/controllers/pythonanywhere.py @@ -4,10 +4,8 @@ import re import gzip import tarfile -from gluon._compat import StringIO -from xmlrpclib import ProtocolError from gluon.contrib.simplejsonrpc import ServerProxy - +from gluon._compat import StringIO, ProtocolError def deploy(): response.title = T('Deploy to pythonanywhere') diff --git a/gluon/_compat.py b/gluon/_compat.py index f7b53dde7..a87a5c943 100644 --- a/gluon/_compat.py +++ b/gluon/_compat.py @@ -31,6 +31,7 @@ from types import ClassType import cgi import cookielib + from xmlrpclib import ProtocolError BytesIO = StringIO reduce = reduce hashlib_md5 = hashlib.md5 @@ -94,6 +95,7 @@ def to_native(obj, charset='utf8', errors='strict'): from urllib.request import FancyURLopener, urlopen from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode from http import cookiejar as cookielib + from xmlrpc.client import ProtocolError import html # warning, this is the python3 module and not the web2py html module hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8')) iterkeys = lambda d: iter(d.keys()) diff --git a/gluon/contrib/simplejsonrpc.py b/gluon/contrib/simplejsonrpc.py index 2a4547f7a..f60dc4c75 100644 --- a/gluon/contrib/simplejsonrpc.py +++ b/gluon/contrib/simplejsonrpc.py @@ -17,12 +17,17 @@ __license__ = "LGPL 3.0" __version__ = "0.05" +import sys +PY2 = sys.version_info[0] == 2 import urllib -from xmlrpclib import Transport, SafeTransport -from cStringIO import StringIO +if PY2: + from xmlrpclib import Transport, SafeTransport + from cStringIO import StringIO +else: + from xmlrpc.client import Transport, SafeTransport + from io import StringIO import random -import sys import json