Merge pull request web2py#1470 from ilvalle/compileapp_getcfs

cache (and compile) views, models and controllers

applications/admin/controllers/pythonanywhere.py

@@ -4,10 +4,8 @@
 import re
 import gzip
 import tarfile
-from gluon._compat import StringIO
-from xmlrpclib import ProtocolError
 from gluon.contrib.simplejsonrpc import ServerProxy
-
+from gluon._compat import StringIO, ProtocolError
 
 def deploy():
     response.title = T('Deploy to pythonanywhere')

applications/welcome/controllers/appadmin.py

@@ -409,7 +409,7 @@ def ccache():
     import copy
     import time
     import math
-    from gluon import portalocker
+    from pydal.contrib import portalocker
 
     ram = {
         'entries': 0,

gluon/_compat.py

@@ -31,6 +31,7 @@
     from types import ClassType
     import cgi
     import cookielib
+    from xmlrpclib import ProtocolError
     BytesIO = StringIO
     reduce = reduce
     hashlib_md5 = hashlib.md5
@@ -94,6 +95,7 @@ def to_native(obj, charset='utf8', errors='strict'):
     from urllib.request import FancyURLopener, urlopen
     from urllib.parse import quote as urllib_quote, unquote as urllib_unquote, urlencode
     from http import cookiejar as cookielib
+    from xmlrpc.client import ProtocolError
     import html # warning, this is the python3 module and not the web2py html module
     hashlib_md5 = lambda s: hashlib.md5(bytes(s, 'utf8'))
     iterkeys = lambda d: iter(d.keys())

gluon/compileapp.py

@@ -430,21 +430,17 @@ def build_environment(request, response, session, store_current=True):
         current.T = t
         current.cache = c
 
-    global __builtins__
     if is_jython:  # jython hack
+        global __builtins__
         __builtins__ = mybuiltin()
-    elif is_pypy:  # apply the same hack to pypy too
-        __builtins__ = mybuiltin()
-    elif PY2:
-        __builtins__['__import__'] = builtin.__import__  # WHY?
+
     environment['request'] = request
     environment['response'] = response
     environment['session'] = session
     environment['local_import'] = \
         lambda name, reload=False, app=request.application:\
         local_import_aux(name, reload, app)
     BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
-    response._view_environment = copy.copy(environment)
     custom_import_install()
     return environment
 
@@ -487,7 +483,7 @@ def compile_views(folder, skip_failed_views=False):
             else:
                 raise Exception("%s in %s" % (e, fname))
         else:
-            filename = ('views/%s.py' % fname).replace('/', '_').replace('\\', '_')
+            filename = 'views.%s.py' % fname.replace(os.path.sep, '.')
             filename = pjoin(folder, 'compiled', filename)
             write_file(filename, data)
             save_pyc(filename)
@@ -503,7 +499,7 @@ def compile_models(folder):
     path = pjoin(folder, 'models')
     for fname in listdir(path, '.+\.py$'):
         data = read_file(pjoin(path, fname))
-        modelfile = 'models.'+fname.replace(os.path.sep,'.')
+        modelfile = 'models.'+fname.replace(os.path.sep, '.')
         filename = pjoin(folder, 'compiled', modelfile)
         mktree(filename)
         write_file(filename, data)
@@ -582,11 +578,10 @@ def run_models_in(environment):
                 continue
             elif compiled:
                 code = getcfs(model, model, lambda: read_pyc(model))
-            elif is_gae:
+            else:
                 code = getcfs(model, model,
                               lambda: compile2(read_file(model), model))
-            else:
-                code = getcfs(model, model, None)
+
             restricted(code, environment, layer=model)
 
 
@@ -610,7 +605,6 @@ def run_controller_in(controller, function, environment):
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
         code = getcfs(filename, filename, lambda: read_pyc(filename))
-        restricted(code, environment, layer=filename)
     elif function == '_TEST':
         # TESTING: adjust the path to include site packages
         from gluon.settings import global_settings
@@ -629,7 +623,6 @@ def run_controller_in(controller, function, environment):
         environment['__symbols__'] = environment.keys()
         code = read_file(filename)
         code += TEST_CODE
-        restricted(code, environment, layer=filename)
     else:
         filename = pjoin(folder, 'controllers/%s.py'
                                  % controller)
@@ -644,10 +637,10 @@ def run_controller_in(controller, function, environment):
                        rewrite.THREAD_LOCAL.routes.error_message % badf,
                        web2py_error=badf)
         code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
-        if is_gae:
-            layer = filename + ':' + function
-            code = getcfs(layer, filename, lambda: compile2(code, layer))
-        restricted(code, environment, filename)
+        layer = filename + ':' + function
+        code = getcfs(layer, filename, lambda: compile2(code, layer))
+
+    restricted(code, environment, layer=filename)
     response = current.response
     vars = response._vars
     if response.postprocessing:
@@ -682,7 +675,7 @@ def run_view_in(environment):
     if not isinstance(view, str):
         ccode = parse_template(view, pjoin(folder, 'views'),
                                context=environment)
-        restricted(ccode, environment, 'file stream')
+        layer = 'file stream'
     else:
         filename = pjoin(folder, 'views', view)
         if os.path.exists(path): # compiled views
@@ -713,16 +706,13 @@ def run_view_in(environment):
                        rewrite.THREAD_LOCAL.routes.error_message % badv,
                        web2py_error=badv)
         layer = filename
-        if is_gae:
-            ccode = getcfs(layer, filename,
-                           lambda: compile2(parse_template(view,
-                                            pjoin(folder, 'views'),
-                                            context=environment), layer))
-        else:
-            ccode = parse_template(view,
-                                   pjoin(folder, 'views'),
-                                   context=environment)
-        restricted(ccode, environment, layer)
+        # Cache the compiled template
+        ccode = getcfs(layer, filename,
+                       lambda: compile2(parse_template(view,
+                                                       pjoin(folder, 'views'),
+                                                       context=environment),
+                                        layer))
+    restricted(ccode, environment, layer=layer)
 
 
 def remove_compiled_application(folder):
@@ -748,26 +738,3 @@ def compile_application(folder, skip_failed_views=False):
     compile_controllers(folder)
     failed_views = compile_views(folder, skip_failed_views)
     return failed_views
-
-
-def test():
-    """
-    Example::
-
-        >>> import traceback, types
-        >>> environment={'x':1}
-        >>> open('a.py', 'w').write('print 1/x')
-        >>> save_pyc('a.py')
-        >>> os.unlink('a.py')
-        >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code'
-        code
-        >>> exec read_pyc('a.pyc') in environment
-        1
-    """
-
-    return
-
-
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod()

gluon/contrib/simplejsonrpc.py

@@ -17,12 +17,17 @@
 __license__ = "LGPL 3.0"
 __version__ = "0.05"
 
+import sys
+PY2 = sys.version_info[0] == 2
 
 import urllib
-from xmlrpclib import Transport, SafeTransport
-from cStringIO import StringIO
+if PY2:
+    from xmlrpclib import Transport, SafeTransport
+    from cStringIO import StringIO
+else:
+    from xmlrpc.client import Transport, SafeTransport
+    from io import StringIO
 import random
-import sys
 import json
 
 

gluon/restricted.py

@@ -202,7 +202,7 @@ def compile2(code, layer):
     return compile(code, layer, 'exec')
 
 
-def restricted(code, environment=None, layer='Unknown'):
+def restricted(ccode, environment=None, layer='Unknown'):
     """
     Runs code in environment and returns the output. If an exception occurs
     in code it raises a RestrictedError containing the traceback. Layer is
@@ -213,10 +213,6 @@ def restricted(code, environment=None, layer='Unknown'):
     environment['__file__'] = layer
     environment['__name__'] = '__restricted__'
     try:
-        if isinstance(code, types.CodeType):
-            ccode = code
-        else:
-            ccode = compile2(code, layer)
         exec(ccode, environment)
     except HTTP:
         raise
@@ -231,7 +227,7 @@ def restricted(code, environment=None, layer='Unknown'):
             sys.excepthook(etype, evalue, tb)
         del tb
         output = "%s %s" % (etype, evalue)
-        raise RestrictedError(layer, code, output, environment)
+        raise RestrictedError(layer, ccode, output, environment)
 
 
 def snapshot(info=None, context=5, code=None, environment=None):

gluon/template.py

@@ -24,9 +24,6 @@
     # have web2py
     from gluon.restricted import RestrictedError
     from gluon.globals import current
-    from gluon.cfs import getcfs
-    from gluon.fileutils import read_file
-    HAS_CFS = True
 except ImportError:
     # do not have web2py
     current = None
@@ -783,14 +780,11 @@ def parse_template(filename,
     # First, if we have a str try to open the file
     if isinstance(filename, str):
         fname = os.path.join(path, filename)
-        if HAS_CFS:
-            text = getcfs(fname, fname, lambda: read_file(fname))
-        else:
-            try:
-                with open(fname, 'rb') as fp:
-                    text = fp.read()
-            except IOError:
-                raise RestrictedError(filename, '', 'Unable to find the file')
+        try:
+            with open(fname, 'rb') as fp:
+                text = fp.read()
+        except IOError:
+            raise RestrictedError(filename, '', 'Unable to find the file')
     else:
         text = filename.read()
     text = to_native(text)

gluon/tests/test_appadmin.py

@@ -16,6 +16,7 @@
 from gluon import fileutils
 from gluon.dal import DAL, Field, Table
 from gluon.http import HTTP
+from gluon.fileutils import open_file
 
 DEFAULT_URI = os.getenv('DB', 'sqlite:memory')
 
@@ -76,12 +77,18 @@ def run_function(self):
     def run_view(self):
         return run_view_in(self.env)
 
+    def run_view_file_stream(self):
+        view_path = os.path.join(self.env['request'].folder, 'views', 'appadmin.html')
+        self.env['response'].view = open_file(view_path, 'r')
+        return run_view_in(self.env)
+
     def _test_index(self):
         result = self.run_function()
         self.assertTrue('db' in result['databases'])
         self.env.update(result)
         try:
             self.run_view()
+            self.run_view_file_stream()
         except Exception as e:
             print(e.message)
             self.fail('Could not make the view')