From c1d81a8f601716b20558c7ec83780bbc293abd1a Mon Sep 17 00:00:00 2001 From: ilvalle Date: Sat, 8 Oct 2016 12:22:47 +0200 Subject: [PATCH] fix few app admin py3 issues, updated qdb (close #1471) --- applications/admin/controllers/appadmin.py | 2 +- applications/admin/controllers/debug.py | 18 +- applications/admin/controllers/webservices.py | 36 +- applications/admin/controllers/wizard.py | 22 +- applications/examples/controllers/appadmin.py | 2 +- applications/welcome/controllers/appadmin.py | 2 +- gluon/contrib/{qdb.py => dbg.py} | 512 +++++++++++------- gluon/contrib/populate.py | 11 +- gluon/debug.py | 32 +- 9 files changed, 394 insertions(+), 243 deletions(-) rename gluon/contrib/{qdb.py => dbg.py} (70%) diff --git a/applications/admin/controllers/appadmin.py b/applications/admin/controllers/appadmin.py index 3060945c8..df6c7f94c 100644 --- a/applications/admin/controllers/appadmin.py +++ b/applications/admin/controllers/appadmin.py @@ -482,7 +482,7 @@ def GetInHMS(seconds): disk['oldest'] = value[0] disk['keys'].append((key, GetInHMS(time.time() - value[0]))) - ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses'] + ram_keys = list(ram) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses'] ram_keys.remove('ratio') ram_keys.remove('oldest') for key in ram_keys: diff --git a/applications/admin/controllers/debug.py b/applications/admin/controllers/debug.py index 4f04ab09d..447948d65 100644 --- a/applications/admin/controllers/debug.py +++ b/applications/admin/controllers/debug.py @@ -4,7 +4,7 @@ import gluon.html import gluon.validators import code -from gluon.debug import communicate, web_debugger, qdb_debugger +from gluon.debug import communicate, web_debugger, dbg_debugger from gluon._compat import thread import pydoc @@ -39,7 +39,7 @@ def reset(): return 'done' -# new implementation using qdb +# new implementation using dbg def interact(): app = request.args(0) or 'admin' @@ -149,7 +149,7 @@ def breakpoints(): if form.accepts(request.vars, session): filename = os.path.join(request.env['applications_parent'], 'applications', form.vars.filename) - err = qdb_debugger.do_set_breakpoint(filename, + err = dbg_debugger.do_set_breakpoint(filename, form.vars.lineno, form.vars.temporary, form.vars.condition) @@ -158,13 +158,13 @@ def breakpoints(): for item in request.vars: if item[:7] == 'delete_': - qdb_debugger.do_clear(item[7:]) + dbg_debugger.do_clear(item[7:]) breakpoints = [{'number': bp[0], 'filename': os.path.basename(bp[1]), 'path': bp[1], 'lineno': bp[2], 'temporary': bp[3], 'enabled': bp[4], 'hits': bp[5], 'condition': bp[6]} - for bp in qdb_debugger.do_list_breakpoint()] + for bp in dbg_debugger.do_list_breakpoint()] return dict(breakpoints=breakpoints, form=form) @@ -193,18 +193,18 @@ def toggle_breakpoint(): else: lineno = None if lineno is not None: - for bp in qdb_debugger.do_list_breakpoint(): + for bp in dbg_debugger.do_list_breakpoint(): no, bp_filename, bp_lineno, temporary, enabled, hits, cond = bp # normalize path name: replace slashes, references, etc... bp_filename = os.path.normpath(os.path.normcase(bp_filename)) if filename == bp_filename and lineno == bp_lineno: - err = qdb_debugger.do_clear_breakpoint(filename, lineno) + err = dbg_debugger.do_clear_breakpoint(filename, lineno) response.flash = T("Removed Breakpoint on %s at line %s", ( filename, lineno)) ok = False break else: - err = qdb_debugger.do_set_breakpoint(filename, lineno) + err = dbg_debugger.do_set_breakpoint(filename, lineno) response.flash = T("Set Breakpoint on %s at line %s: %s") % ( filename, lineno, err or T('successful')) ok = True @@ -224,7 +224,7 @@ def list_breakpoints(): 'applications', request.vars.filename) # normalize path name: replace slashes, references, etc... filename = os.path.normpath(os.path.normcase(filename)) - for bp in qdb_debugger.do_list_breakpoint(): + for bp in dbg_debugger.do_list_breakpoint(): no, bp_filename, bp_lineno, temporary, enabled, hits, cond = bp # normalize path name: replace slashes, references, etc... bp_filename = os.path.normpath(os.path.normcase(bp_filename)) diff --git a/applications/admin/controllers/webservices.py b/applications/admin/controllers/webservices.py index 4f5a352c8..9ae704db2 100644 --- a/applications/admin/controllers/webservices.py +++ b/applications/admin/controllers/webservices.py @@ -81,44 +81,44 @@ def install(app_name, filename, data, overwrite=True): @service.jsonrpc def attach_debugger(host='localhost', port=6000, authkey='secret password'): - import gluon.contrib.qdb as qdb + import gluon.contrib.dbg as dbg import gluon.debug from multiprocessing.connection import Listener if isinstance(authkey, unicode): authkey = authkey.encode('utf8') - if not hasattr(gluon.debug, 'qdb_listener'): + if not hasattr(gluon.debug, 'dbg_listener'): # create a remote debugger server and wait for connection address = (host, port) # family is deduced to be 'AF_INET' - gluon.debug.qdb_listener = Listener(address, authkey=authkey) - gluon.debug.qdb_connection = gluon.debug.qdb_listener.accept() + gluon.debug.dbg_listener = Listener(address, authkey=authkey) + gluon.debug.dbg_connection = gluon.debug.dbg_listener.accept() # create the backend - gluon.debug.qdb_debugger = qdb.Qdb(gluon.debug.qdb_connection) - gluon.debug.dbg = gluon.debug.qdb_debugger + gluon.debug.dbg_debugger = dbg.Qdb(gluon.debug.dbg_connection) + gluon.debug.dbg = gluon.debug.dbg_debugger # welcome message (this should be displayed on the frontend) - print('debugger connected to', gluon.debug.qdb_listener.last_accepted) + print('debugger connected to', gluon.debug.dbg_listener.last_accepted) return True # connection successful! @service.jsonrpc def detach_debugger(): - import gluon.contrib.qdb as qdb + import gluon.contrib.dbg as dbg import gluon.debug # stop current debugger - if gluon.debug.qdb_debugger: + if gluon.debug.dbg_debugger: try: - gluon.debug.qdb_debugger.do_quit() + gluon.debug.dbg_debugger.do_quit() except: pass - if hasattr(gluon.debug, 'qdb_listener'): - if gluon.debug.qdb_connection: - gluon.debug.qdb_connection.close() - del gluon.debug.qdb_connection - if gluon.debug.qdb_listener: - gluon.debug.qdb_listener.close() - del gluon.debug.qdb_listener - gluon.debug.qdb_debugger = None + if hasattr(gluon.debug, 'dbg_listener'): + if gluon.debug.dbg_connection: + gluon.debug.dbg_connection.close() + del gluon.debug.dbg_connection + if gluon.debug.dbg_listener: + gluon.debug.dbg_listener.close() + del gluon.debug.dbg_listener + gluon.debug.dbg_debugger = None return True diff --git a/applications/admin/controllers/wizard.py b/applications/admin/controllers/wizard.py index f4511b965..c3d535485 100644 --- a/applications/admin/controllers/wizard.py +++ b/applications/admin/controllers/wizard.py @@ -7,7 +7,7 @@ import urllib import glob from gluon.admin import app_create, plugin_install -from gluon.fileutils import abspath, read_file, write_file +from gluon.fileutils import abspath, read_file, write_file, open_file def reset(session): @@ -67,7 +67,7 @@ def index(): '..', app, 'wizard.metadata')) if os.path.exists(meta): try: - metafile = open(meta, 'rb') + metafile = open_file(meta, 'rb') try: session.app = pickle.load(metafile) finally: @@ -402,7 +402,7 @@ def make_table(table, fields): def fix_db(filename): params = dict(session.app['params']) - content = read_file(filename, 'rb') + content = read_file(filename, 'r') if 'auth_user' in session.app['tables']: auth_user = make_table('auth_user', session.app['table_auth_user']) content = content.replace('sqlite://storage.sqlite', @@ -424,7 +424,7 @@ def fix_db(filename): domain = settings.login_config.split(':')[0], url = "http://%s/%s/default/user/login" % (request.env.http_host,request.application)) """ - write_file(filename, content, 'wb') + write_file(filename, content, 'w') def make_menu(pages): @@ -493,7 +493,7 @@ def create(options): ### save metadata in newapp/wizard.metadata try: meta = os.path.join(request.folder, '..', app, 'wizard.metadata') - file = open(meta, 'wb') + file = open_file(meta, 'wb') pickle.dump(session.app, file) file.close() except IOError: @@ -521,7 +521,7 @@ def create(options): ### write configuration file into newapp/models/0.py model = os.path.join(request.folder, '..', app, 'models', '0.py') - file = open(model, 'wb') + file = open_file(model, 'w') try: file.write("from gluon.storage import Storage\n") file.write("settings = Storage()\n\n") @@ -534,7 +534,7 @@ def create(options): ### write configuration file into newapp/models/menu.py if options.generate_menu: model = os.path.join(request.folder, '..', app, 'models', 'menu.py') - file = open(model, 'wb') + file = open_file(model, 'w') try: file.write(make_menu(session.app['pages'])) finally: @@ -548,7 +548,7 @@ def create(options): if options.generate_model: model = os.path.join( request.folder, '..', app, 'models', 'db_wizard.py') - file = open(model, 'wb') + file = open_file(model, 'w') try: file.write('### we prepend t_ to tablenames and f_ to fieldnames for disambiguity\n\n') tables = sort_tables(session.app['tables']) @@ -564,7 +564,7 @@ def create(options): if os.path.exists(model): os.unlink(model) if options.populate_database and session.app['tables']: - file = open(model, 'wb') + file = open_file(model, 'w') try: file.write(populate(session.app['tables'])) finally: @@ -574,7 +574,7 @@ def create(options): if options.generate_controller: controller = os.path.join( request.folder, '..', app, 'controllers', 'default.py') - file = open(controller, 'wb') + file = open_file(controller, 'w') try: file.write("""# -*- coding: utf-8 -*- ### required - do no delete @@ -594,7 +594,7 @@ def call(): return service() for page in session.app['pages']: view = os.path.join( request.folder, '..', app, 'views', 'default', page + '.html') - file = open(view, 'wb') + file = open_file(view, 'w') try: file.write( make_view(page, session.app.get('page_' + page, ''))) diff --git a/applications/examples/controllers/appadmin.py b/applications/examples/controllers/appadmin.py index 3060945c8..df6c7f94c 100644 --- a/applications/examples/controllers/appadmin.py +++ b/applications/examples/controllers/appadmin.py @@ -482,7 +482,7 @@ def GetInHMS(seconds): disk['oldest'] = value[0] disk['keys'].append((key, GetInHMS(time.time() - value[0]))) - ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses'] + ram_keys = list(ram) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses'] ram_keys.remove('ratio') ram_keys.remove('oldest') for key in ram_keys: diff --git a/applications/welcome/controllers/appadmin.py b/applications/welcome/controllers/appadmin.py index 4d4e965fb..213b86834 100644 --- a/applications/welcome/controllers/appadmin.py +++ b/applications/welcome/controllers/appadmin.py @@ -482,7 +482,7 @@ def GetInHMS(seconds): disk['oldest'] = value[0] disk['keys'].append((key, GetInHMS(time.time() - value[0]))) - ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses'] + ram_keys = list(ram) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses'] ram_keys.remove('ratio') ram_keys.remove('oldest') for key in ram_keys: diff --git a/gluon/contrib/qdb.py b/gluon/contrib/dbg.py similarity index 70% rename from gluon/contrib/qdb.py rename to gluon/contrib/dbg.py index 6e32e6f73..0dcbcf153 100644 --- a/gluon/contrib/qdb.py +++ b/gluon/contrib/dbg.py @@ -1,13 +1,14 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding:utf-8 -"Queues(Pipe)-based independent remote client-server Python Debugger" +"Queues(Pipe)-based independent remote client-server Python Debugger (new-py3)" + from __future__ import print_function __author__ = "Mariano Reingart (reingart@gmail.com)" __copyright__ = "Copyright (C) 2011 Mariano Reingart" __license__ = "LGPL 3.0" -__version__ = "1.01b" +__version__ = "1.5.2" # remote debugger queue-based (jsonrpc-like interface): # - bidirectional communication (request - response calls in both ways) @@ -24,13 +25,19 @@ import cmd import pydoc import threading +import collections + + +# Speed Ups: global variables +breaks = [] class Qdb(bdb.Bdb): "Qdb Debugger Backend" def __init__(self, pipe, redirect_stdio=True, allow_interruptions=False, - skip=[__name__]): + use_speedups=True, skip=[__name__]): + global breaks kwargs = {} if sys.version_info > (2, 7): kwargs['skip'] = skip @@ -38,12 +45,15 @@ def __init__(self, pipe, redirect_stdio=True, allow_interruptions=False, self.frame = None self.i = 1 # sequential RPC call id self.waiting = False - self.pipe = pipe # for communication + self.pipe = pipe # for communication self._wait_for_mainpyfile = False self._wait_for_breakpoint = False self.mainpyfile = "" self._lineno = None # last listed line numbre + # ignore filenames (avoid spurious interaction specially on py2) + self.ignore_files = [self.canonic(f) for f in (__file__, bdb.__file__)] # replace system standard input and output (send them thru the pipe) + self.old_stdio = sys.stdin, sys.stdout, sys.stderr if redirect_stdio: sys.stdin = self sys.stdout = self @@ -54,6 +64,10 @@ def __init__(self, pipe, redirect_stdio=True, allow_interruptions=False, self.allow_interruptions = allow_interruptions self.burst = 0 # do not send notifications ("burst" mode) self.params = {} # optional parameters for interaction + + # flags to reduce overhead (only stop at breakpoint or interrupt) + self.use_speedups = use_speedups + self.fast_continue = False def pull_actions(self): # receive a remote procedure call from the frontend: @@ -62,14 +76,14 @@ def pull_actions(self): request = self.pipe.recv() if request.get("method") == 'run': return None - response = {'version': '1.1', 'id': request.get('id'), - 'result': None, + response = {'version': '1.1', 'id': request.get('id'), + 'result': None, 'error': None} try: # dispatch message (JSON RPC like) method = getattr(self, request['method']) - response['result'] = method.__call__(*request['args'], - **request.get('kwargs', {})) + response['result'] = method.__call__(*request['args'], + **request.get('kwargs', {})) except Exception as e: response['error'] = {'code': 0, 'message': str(e)} # send the result for normal method calls, not for notifications @@ -83,9 +97,17 @@ def trace_dispatch(self, frame, event, arg): # check for non-interaction rpc (set_breakpoint, interrupt) while self.allow_interruptions and self.pipe.poll(): self.pull_actions() + # check for non-interaction rpc (set_breakpoint, interrupt) + while self.pipe.poll(): + self.pull_actions() + if (frame.f_code.co_filename, frame.f_lineno) not in breaks and \ + self.fast_continue: + return self.trace_dispatch # process the frame (see Bdb.trace_dispatch) + ##if self.fast_continue: + ## return self.trace_dispatch if self.quitting: - return # None + return # None if event == 'line': return self.dispatch_line(frame) if event == 'call': @@ -102,13 +124,13 @@ def user_call(self, frame, argument_list): if self._wait_for_mainpyfile or self._wait_for_breakpoint: return if self.stop_here(frame): - self.interaction(frame, None) - + self.interaction(frame) + def user_line(self, frame): """This function is called when we stop or break at this line.""" if self._wait_for_mainpyfile: if (not self.canonic(frame.f_code.co_filename).startswith(self.mainpyfile) - or frame.f_lineno <= 0): + or frame.f_lineno<= 0): return self._wait_for_mainpyfile = 0 if self._wait_for_breakpoint: @@ -125,14 +147,15 @@ def user_exception(self, frame, info): extype, exvalue, trace = info # pre-process stack trace as it isn't pickeable (cannot be sent pure) msg = ''.join(traceback.format_exception(extype, exvalue, trace)) - trace = traceback.extract_tb(trace) + # in python3.5, convert FrameSummary to tuples (py2.7+ compatibility) + tb = [tuple(fs) for fs in traceback.extract_tb(trace)] title = traceback.format_exception_only(extype, exvalue)[0] # send an Exception notification - msg = {'method': 'exception', - 'args': (title, extype.__name__, exvalue, trace, msg), + msg = {'method': 'exception', + 'args': (title, extype.__name__, repr(exvalue), tb, msg), 'id': None} self.pipe.send(msg) - self.interaction(frame, info) + self.interaction(frame) def run(self, code, interp=None, *args, **kwargs): try: @@ -151,39 +174,58 @@ def _runscript(self, filename): # The script has to run in __main__ namespace (clear it) import __main__ import imp + filename = os.path.abspath(filename) __main__.__dict__.clear() - __main__.__dict__.update({"__name__": "__main__", - "__file__": filename, + __main__.__dict__.update({"__name__" : "__main__", + "__file__" : filename, "__builtins__": __builtins__, - "imp": imp, # need for run - }) + "imp" : imp, # need for run + }) - # avoid stopping before we reach the main script + # avoid stopping before we reach the main script self._wait_for_mainpyfile = 1 self.mainpyfile = self.canonic(filename) self._user_requested_quit = 0 - statement = 'imp.load_source("__main__", "%s")' % filename - # notify and wait frontend to set initial params and breakpoints - self.pipe.send({'method': 'startup', 'args': (__version__, )}) + if sys.version_info>(3,0): + statement = 'imp.load_source("__main__", "%s")' % filename + else: + statement = 'execfile(%r)' % filename + self.startup() + self.run(statement) + + def startup(self): + "Notify and wait frontend to set initial params and breakpoints" + # send some useful info to identify session + thread = threading.current_thread() + # get the caller module filename + frame = sys._getframe() + fn = self.canonic(frame.f_code.co_filename) + while frame.f_back and self.canonic(frame.f_code.co_filename) == fn: + frame = frame.f_back + args = [__version__, os.getpid(), thread.name, " ".join(sys.argv), + frame.f_code.co_filename] + self.pipe.send({'method': 'startup', 'args': args}) while self.pull_actions() is not None: pass - self.run(statement) # General interaction function - def interaction(self, frame, info=None): + def interaction(self, frame): # chache frame locals to ensure that modifications are not overwritten self.frame_locals = frame and frame.f_locals or {} # extract current filename and line number code, lineno = frame.f_code, frame.f_lineno - filename = code.co_filename + filename = self.canonic(code.co_filename) basename = os.path.basename(filename) + # check if interaction should be ignored (i.e. debug modules internals) + if filename in self.ignore_files: + return message = "%s:%s" % (basename, lineno) if code.co_name != "?": message = "%s: %s()" % (message, code.co_name) - # wait user events - self.waiting = True + # wait user events + self.waiting = True self.frame = frame try: while self.waiting: @@ -202,11 +244,10 @@ def interaction(self, frame, info=None): if self.params.get('environment'): kwargs['environment'] = self.do_environment() self.pipe.send({'method': 'interaction', 'id': None, - 'args': (filename, self.frame.f_lineno, line), - 'kwargs': kwargs}) + 'args': (filename, self.frame.f_lineno, line), + 'kwargs': kwargs}) self.pull_actions() - finally: self.waiting = False self.frame = None @@ -228,6 +269,8 @@ def set_trace(self, frame=None): frame = sys._getframe().f_back self._wait_for_mainpyfile = frame.f_code.co_filename self._wait_for_breakpoint = 0 + # reinitialize debugger internal settings + self.fast_continue = False bdb.Bdb.set_trace(self, frame) # Command definitions, called by interaction() @@ -235,34 +278,38 @@ def set_trace(self, frame=None): def do_continue(self): self.set_continue() self.waiting = False + self.fast_continue = self.use_speedups def do_step(self): self.set_step() self.waiting = False + self.fast_continue = False def do_return(self): self.set_return(self.frame) self.waiting = False + self.fast_continue = False def do_next(self): self.set_next(self.frame) self.waiting = False + self.fast_continue = False def interrupt(self): - self.set_step() + self.set_trace() + self.fast_continue = False def do_quit(self): self.set_quit() self.waiting = False + self.fast_continue = False def do_jump(self, lineno): arg = int(lineno) try: self.frame.f_lineno = arg - return arg except ValueError as e: - print('*** Jump failed:', e) - return False + return str(e) def do_list(self, arg): last = None @@ -272,7 +319,7 @@ def do_list(self, arg): else: first = arg elif not self._lineno: - first = max(1, self.frame.f_lineno - 5) + first = max(1, self.frame.f_lineno - 5) else: first = self._lineno + 1 if last is None: @@ -280,11 +327,11 @@ def do_list(self, arg): filename = self.frame.f_code.co_filename breaklist = self.get_file_breaks(filename) lines = [] - for lineno in range(first, last + 1): + for lineno in range(first, last+1): line = linecache.getline(filename, lineno, self.frame.f_globals) if not line: - lines.append((filename, lineno, '', current, "\n")) + lines.append((filename, lineno, '', "", "\n")) break else: breakpoint = "B" if lineno in breaklist else "" @@ -297,6 +344,8 @@ def do_read(self, filename): return open(filename, "Ur").read() def do_set_breakpoint(self, filename, lineno, temporary=0, cond=None): + global breaks # list for speedups! + breaks.append((filename.replace("\\", "/"), int(lineno))) return self.set_break(filename, int(lineno), temporary, cond) def do_list_breakpoint(self): @@ -304,8 +353,8 @@ def do_list_breakpoint(self): if self.breaks: # There's at least one for bp in bdb.Breakpoint.bpbynumber: if bp: - breaks.append((bp.number, bp.file, bp.line, - bp.temporary, bp.enabled, bp.hits, bp.cond, )) + breaks.append((bp.number, bp.file, bp.line, + bp.temporary, bp.enabled, bp.hits, bp.cond, )) return breaks def do_clear_breakpoint(self, filename, lineno): @@ -321,24 +370,33 @@ def do_clear(self, arg): print('*** DO_CLEAR failed', err) def do_eval(self, arg, safe=True): - ret = eval(arg, self.frame.f_globals, - self.frame_locals) + if self.frame: + ret = eval(arg, self.frame.f_globals, + self.frame_locals) + else: + ret = RPCError("No current frame available to eval") if safe: ret = pydoc.cram(repr(ret), 255) return ret - def do_exec(self, arg): - locals = self.frame_locals - globals = self.frame.f_globals - code = compile(arg + '\n', '', 'single') - save_displayhook = sys.displayhook - self.displayhook_value = None - try: - sys.displayhook = self.displayhook - exec code in globals, locals - finally: - sys.displayhook = save_displayhook - return self.displayhook_value + def do_exec(self, arg, safe=True): + if not self.frame: + ret = RPCError("No current frame available to exec") + else: + locals = self.frame_locals + globals = self.frame.f_globals + code = compile(arg + '\n', '', 'single') + save_displayhook = sys.displayhook + self.displayhook_value = None + try: + sys.displayhook = self.displayhook + exec(code, globals, locals) + ret = self.displayhook_value + finally: + sys.displayhook = save_displayhook + if safe: + ret = pydoc.cram(repr(ret), 255) + return ret def do_where(self): "print_stack_trace" @@ -355,29 +413,33 @@ def do_environment(self): env = {'locals': {}, 'globals': {}} # converts the frame global and locals to a short text representation: if self.frame: - for name, value in self.frame_locals.items(): - env['locals'][name] = pydoc.cram(repr( - value), 255), repr(type(value)) - for name, value in self.frame.f_globals.items(): - env['globals'][name] = pydoc.cram(repr( - value), 20), repr(type(value)) + for scope, max_length, vars in ( + ("locals", 255, list(self.frame_locals.items())), + ("globals", 20, list(self.frame.f_globals.items())), ): + for (name, value) in vars: + try: + short_repr = pydoc.cram(repr(value), max_length) + except Exception as e: + # some objects cannot be represented... + short_repr = "**exception** %s" % repr(e) + env[scope][name] = (short_repr, repr(type(value))) return env def get_autocomplete_list(self, expression): "Return list of auto-completion options for expression" try: - obj = self.do_eval(expression) + obj = self.do_eval(expression, safe=False) except: return [] else: return dir(obj) - + def get_call_tip(self, expression): "Return list of auto-completion options for expression" try: obj = self.do_eval(expression) except Exception as e: - return ('', '', str(e)) + return ('', '', str(e)) else: name = '' try: @@ -405,7 +467,7 @@ def get_call_tip(self, expression): break if f is not None: drop_self = 1 - elif callable(obj): + elif isinstance(obj, collections.Callable): # use the obj as a function by default f = obj # Get the __call__ method instead. @@ -416,7 +478,7 @@ def get_call_tip(self, expression): if f: argspec = inspect.formatargspec(*inspect.getargspec(f)) doc = '' - if callable(obj): + if isinstance(obj, collections.Callable): try: doc = inspect.getdoc(obj) except: @@ -442,15 +504,21 @@ def reset(self): self.waiting = False self.frame = None - def post_mortem(self, t=None): + def post_mortem(self, info=None): + "Debug an un-handled python exception" + # check if post mortem mode is enabled: + if not self.params.get('postmortem', True): + return # handling the default - if t is None: + if info is None: # sys.exc_info() returns (type, value, traceback) if an exception is # being handled, otherwise it returns None - t = sys.exc_info()[2] - if t is None: - raise ValueError("A valid traceback must be passed if no " - "exception is being handled") + info = sys.exc_info() + # extract the traceback object: + t = info[2] + if t is None: + raise ValueError("A valid traceback must be passed if no " + "exception is being handled") self.reset() # get last frame: while t is not None: @@ -460,9 +528,24 @@ def post_mortem(self, t=None): filename = code.co_filename line = linecache.getline(filename, lineno) #(filename, lineno, "", current, line, )} + # SyntaxError doesn't execute even one line, so avoid mainpyfile check + self._wait_for_mainpyfile = False + # send exception information & request interaction + self.user_exception(frame, info) - self.interaction(frame) - + def ping(self): + "Minimal method to test that the pipe (connection) is alive" + try: + # get some non-trivial data to compare: + args = (id(object()), ) + msg = {'method': 'ping', 'args': args, 'id': None} + self.pipe.send(msg) + msg = self.pipe.recv() + # check if data received is ok (alive and synchonized!) + return msg['result'] == args + except (IOError, EOFError): + return None + # console file-like object emulation def readline(self): "Replacement for stdin.readline()" @@ -483,9 +566,9 @@ def write(self, text): "Replacement for stdout.write()" msg = {'method': 'write', 'args': (text, ), 'id': None} self.pipe.send(msg) - + def writelines(self, l): - map(self.write, l) + list(map(self.write, l)) def flush(self): pass @@ -493,10 +576,25 @@ def flush(self): def isatty(self): return 0 + def encoding(self): + return None # use default, 'utf-8' should be better... + + def close(self): + # revert redirections and close connection + if sys: + sys.stdin, sys.stdout, sys.stderr = self.old_stdio + try: + self.pipe.close() + except: + pass + + def __del__(self): + self.close() + class QueuePipe(object): "Simulated pipe for threads (using two queues)" - + def __init__(self, name, in_queue, out_queue): self.__name = name self.in_queue = in_queue @@ -520,12 +618,13 @@ class RPCError(RuntimeError): "Remote Error (not user exception)" pass - + class Frontend(object): "Qdb generic Frontend interface" - + def __init__(self, pipe): self.i = 1 + self.info = () self.pipe = pipe self.notifies = [] self.read_lock = threading.RLock() @@ -545,12 +644,13 @@ def send(self, data): finally: self.write_lock.release() - def startup(self): + def startup(self, version, pid, thread_name, argv, filename): + self.info = (version, pid, thread_name, argv, filename) self.send({'method': 'run', 'args': (), 'id': None}) def interaction(self, filename, lineno, line, *kwargs): raise NotImplementedError - + def exception(self, title, extype, exvalue, trace, request): "Show a user_exception" raise NotImplementedError @@ -558,7 +658,7 @@ def exception(self, title, extype, exvalue, trace, request): def write(self, text): "Console output (print)" raise NotImplementedError - + def readline(self, text): "Console input/rawinput" raise NotImplementedError @@ -570,10 +670,10 @@ def run(self): # wait for a message... request = self.recv() else: - # process an asyncronus notification received earlier + # process an asyncronus notification received earlier request = self.notifies.pop(0) return self.process_message(request) - + def process_message(self, request): if request: result = None @@ -584,17 +684,19 @@ def process_message(self, request): elif request.get('method') == 'interaction': self.interaction(*request.get("args"), **request.get("kwargs")) elif request.get('method') == 'startup': - self.startup() + self.startup(*request['args']) elif request.get('method') == 'exception': self.exception(*request['args']) elif request.get('method') == 'write': self.write(*request.get("args")) elif request.get('method') == 'readline': result = self.readline() + elif request.get('method') == 'ping': + result = request['args'] if result: - response = {'version': '1.1', 'id': request.get('id'), - 'result': result, - 'error': None} + response = {'version': '1.1', 'id': request.get('id'), + 'result': result, + 'error': None} self.send(response) return True @@ -603,18 +705,17 @@ def call(self, method, *args): req = {'method': method, 'args': args, 'id': self.i} self.send(req) self.i += 1 # increment the id - while True: + while 1: # wait until command acknowledge (response id match the request) res = self.recv() if 'id' not in res or not res['id']: - # nested notification received (i.e. write)! process it! - self.process_message(res) + # nested notification received (i.e. write)! process it later... + self.notifies.append(res) elif 'result' not in res: # nested request received (i.e. readline)! process it! self.process_message(res) - elif long(req['id']) != long(res['id']): - print("DEBUGGER wrong packet received: expecting id", req[ - 'id'], res['id']) + elif int(req['id']) != int(res['id']): + print("DEBUGGER wrong packet received: expecting id", req['id'], res['id']) # protocol state is unknown elif 'error' in res and res['error']: raise RPCError(res['error']['message']) @@ -624,23 +725,23 @@ def call(self, method, *args): def do_step(self, arg=None): "Execute the current line, stop at the first possible occasion" self.call('do_step') - + def do_next(self, arg=None): "Execute the current line, do not stop at function calls" self.call('do_next') - def do_continue(self, arg=None): + def do_continue(self, arg=None): "Continue execution, only stop when a breakpoint is encountered." self.call('do_continue') - - def do_return(self, arg=None): + + def do_return(self, arg=None): "Continue execution until the current function returns" self.call('do_return') - def do_jump(self, arg): - "Set the next line that will be executed." + def do_jump(self, arg): + "Set the next line that will be executed (None if sucess or message)" res = self.call('do_jump', arg) - print(res) + return res def do_where(self, arg=None): "Print a stack trace, with the most recent frame at the bottom." @@ -649,7 +750,7 @@ def do_where(self, arg=None): def do_quit(self, arg=None): "Quit from the debugger. The program being executed is aborted." self.call('do_quit') - + def do_eval(self, expr): "Inspect the value of the expression" return self.call('do_eval', expr) @@ -677,11 +778,11 @@ def do_clear_breakpoint(self, filename, lineno): def do_clear_file_breakpoints(self, filename): "Remove all breakpoints at filename" self.call('do_clear_breakpoints', filename, lineno) - + def do_list_breakpoint(self): "List all breakpoints" return self.call('do_list_breakpoint') - + def do_exec(self, statement): return self.call('do_exec', statement) @@ -690,7 +791,7 @@ def get_autocomplete_list(self, expression): def get_call_tip(self, expression): return self.call('get_call_tip', expression) - + def interrupt(self): "Immediately stop at the first possible occasion (outside interaction)" # this is a notification!, do not expect a response @@ -700,7 +801,7 @@ def interrupt(self): def set_burst(self, value): req = {'method': 'set_burst', 'args': (value, )} self.send(req) - + def set_params(self, params): req = {'method': 'set_params', 'args': (params, )} self.send(req) @@ -708,15 +809,15 @@ def set_params(self, params): class Cli(Frontend, cmd.Cmd): "Qdb Front-end command line interface" - + def __init__(self, pipe, completekey='tab', stdin=None, stdout=None, skip=None): cmd.Cmd.__init__(self, completekey, stdin, stdout) Frontend.__init__(self, pipe) # redefine Frontend methods: - + def run(self): - while True: + while 1: try: Frontend.run(self) except KeyboardInterrupt: @@ -736,31 +837,30 @@ def exception(self, title, extype, exvalue, trace, request): def write(self, text): print(text, end=' ') - + def readline(self): - return raw_input() - + return input() + def postcmd(self, stop, line): - return not line.startswith("h") # stop + return not line.startswith("h") # stop do_h = cmd.Cmd.do_help - + do_s = Frontend.do_step do_n = Frontend.do_next - do_c = Frontend.do_continue + do_c = Frontend.do_continue do_r = Frontend.do_return - do_j = Frontend.do_jump do_q = Frontend.do_quit def do_eval(self, args): "Inspect the value of the expression" print(Frontend.do_eval(self, args)) - + def do_list(self, args): "List source code for the current file" lines = Frontend.do_list(self, eval(args, {}, {}) if args else None) self.print_lines(lines) - + def do_where(self, args): "Print a stack trace, with the most recent frame at the bottom." lines = Frontend.do_where(self) @@ -772,7 +872,7 @@ def do_environment(self, args=None): print("=" * 78) print(key.capitalize()) print("-" * 78) - for name, value in env[key].items(): + for name, value in list(env[key].items()): print("%-12s = %s" % (name, value)) def do_list_breakpoint(self, arg=None): @@ -794,11 +894,18 @@ def do_set_breakpoint(self, arg): else: self.do_list_breakpoint() + def do_jump(self, args): + "Jump to the selected line" + ret = Frontend.do_jump(self, args) + if ret: # show error message if failed + print("cannot jump:", ret) + do_b = do_set_breakpoint do_l = do_list do_p = do_eval do_w = do_where do_e = do_environment + do_j = do_jump def default(self, line): "Default command" @@ -813,61 +920,85 @@ def print_lines(self, lines): print() -def test(): - def f(pipe): - print("creating debugger") - qdb = Qdb(pipe=pipe, redirect_stdio=False) - print("set trace") +# WORKAROUND for python3 server using pickle's HIGHEST_PROTOCOL (now 3) +# but python2 client using pickles's protocol version 2 +if sys.version_info[0] > 2: + + import multiprocessing.reduction # forking in py2 + + class ForkingPickler2(multiprocessing.reduction.ForkingPickler): + def __init__(self, file, protocol=None, fix_imports=True): + # downgrade to protocol ver 2 + protocol = 2 + super().__init__(file, protocol, fix_imports) + + multiprocessing.reduction.ForkingPickler = ForkingPickler2 - my_var = "Mariano!" - qdb.set_trace() - print("hello world!") - print("good by!") - saraza +def f(pipe): + "test function to be debugged" + print("creating debugger") + qdb_test = Qdb(pipe=pipe, redirect_stdio=False, allow_interruptions=True) + print("set trace") + + my_var = "Mariano!" + qdb_test.set_trace() + print("hello world!") + for i in range(100000): + pass + print("good by!") + + +def test(): + "Create a backend/frontend and time it" if '--process' in sys.argv: from multiprocessing import Process, Pipe - pipe, child_conn = Pipe() + front_conn, child_conn = Pipe() p = Process(target=f, args=(child_conn,)) else: from threading import Thread - from Queue import Queue + from queue import Queue parent_queue, child_queue = Queue(), Queue() front_conn = QueuePipe("parent", parent_queue, child_queue) child_conn = QueuePipe("child", child_queue, parent_queue) p = Thread(target=f, args=(child_conn,)) - + p.start() import time class Test(Frontend): def interaction(self, *args): print("interaction!", args) - + ##self.do_next() def exception(self, *args): print("exception", args) - #raise RuntimeError("exception %s" % repr(args)) - - qdb = Test(front_conn) - time.sleep(5) - while True: - print("running...") - Frontend.run(qdb) - time.sleep(1) - print("do_next") - qdb.do_next() + qdb_test = Test(front_conn) + time.sleep(1) + t0 = time.time() + + print("running...") + while front_conn.poll(): + Frontend.run(qdb_test) + qdb_test.do_continue() p.join() + t1 = time.time() + print("took", t1 - t0, "seconds") + sys.exit(0) -def connect(host="localhost", port=6000, authkey='secret password'): - "Connect to a running debugger backend" - +def start(host="localhost", port=6000, authkey='secret password'): + "Start the CLI server and wait connection from a running debugger backend" + address = (host, port) - from multiprocessing.connection import Client - - print("qdb debugger fronted: waiting for connection to", address) - conn = Client(address, authkey=authkey) + from multiprocessing.connection import Listener + address = (host, port) # family is deduced to be 'AF_INET' + if isinstance(authkey, str): + authkey = authkey.encode("utf8") + listener = Listener(address, authkey=authkey) + print("qdb debugger backend: waiting for connection at", address) + conn = listener.accept() + print('qdb debugger backend: connected to', listener.last_accepted) try: Cli(conn).run() except EOFError: @@ -877,13 +1008,13 @@ def connect(host="localhost", port=6000, authkey='secret password'): def main(host='localhost', port=6000, authkey='secret password'): - "Debug a script and accept a remote frontend" - + "Debug a script (running under the backend) and connect to remote frontend" + if not sys.argv[1:] or sys.argv[1] in ("--help", "-h"): print("usage: pdb.py scriptfile [arg] ...") sys.exit(2) - mainpyfile = sys.argv[1] # Get script filename + mainpyfile = sys.argv[1] # Get script filename if not os.path.exists(mainpyfile): print('Error:', mainpyfile, 'does not exist') sys.exit(1) @@ -893,15 +1024,8 @@ def main(host='localhost', port=6000, authkey='secret password'): # Replace pdb's dir with script's dir in front of module search path. sys.path[0] = os.path.dirname(mainpyfile) - from multiprocessing.connection import Listener - address = (host, port) # family is deduced to be 'AF_INET' - listener = Listener(address, authkey=authkey) - print("qdb debugger backend: waiting for connection at", address) - conn = listener.accept() - print('qdb debugger backend: connected to', listener.last_accepted) - # create the backend - qdb = Qdb(conn, redirect_stdio=True, allow_interruptions=True) + init(host, port, authkey) try: print("running", mainpyfile) qdb._runscript(mainpyfile) @@ -911,32 +1035,57 @@ def main(host='localhost', port=6000, authkey='secret password'): print("The program exited via sys.exit(). Exit status: ", end=' ') print(sys.exc_info()[1]) raise - except: - raise - - conn.close() - listener.close() + except Exception: + traceback.print_exc() + print("Uncaught exception. Entering post mortem debugging") + info = sys.exc_info() + qdb.post_mortem(info) + print("Program terminated!") + finally: + conn.close() + print("qdb debbuger backend: connection closed") +# "singleton" to store a unique backend per process qdb = None -def set_trace(host='localhost', port=6000, authkey='secret password'): +def init(host='localhost', port=6000, authkey='secret password', redirect=True): "Simplified interface to debug running programs" global qdb, listener, conn - - from multiprocessing.connection import Listener + + # destroy the debugger if the previous connection is lost (i.e. broken pipe) + if qdb and not qdb.ping(): + qdb.close() + qdb = None + + from multiprocessing.connection import Client # only create it if not currently instantiated if not qdb: address = (host, port) # family is deduced to be 'AF_INET' - listener = Listener(address, authkey=authkey) - conn = listener.accept() - + print("qdb debugger backend: waiting for connection to", address) + if isinstance(authkey, str): + authkey = authkey.encode("utf8") + conn = Client(address, authkey=authkey) + print('qdb debugger backend: connected to', address) # create the backend - qdb = Qdb(conn) + qdb = Qdb(conn, redirect_stdio=redirect, allow_interruptions=True) + # initial hanshake + qdb.startup() + + +def set_trace(host='localhost', port=6000, authkey='secret password'): + "Simplified interface to start debugging immediately" + init(host, port, authkey) # start debugger backend: qdb.set_trace() +def debug(host='localhost', port=6000, authkey='secret password'): + "Simplified interface to start debugging immediately (no stop)" + init(host, port, authkey) + # start debugger backend: + qdb.do_debug() + def quit(): "Remove trace and quit" @@ -948,24 +1097,25 @@ def quit(): conn.close() conn = None if listener: - listener.close() + listener.close() listener = None if __name__ == '__main__': # When invoked as main program: - if '--test' in sys.argv: + if '--test1' in sys.argv: test() # Check environment for configuration parameters: kwargs = {} for param in 'host', 'port', 'authkey': - if 'QDB_%s' % param.upper() in os.environ: - kwargs[param] = os.environ['QDB_%s' % param.upper()] + if 'DBG_%s' % param.upper() in os.environ: + kwargs[param] = os.environ['DBG_%s' % param.upper()] if not sys.argv[1:]: # connect to a remote debbuger - connect(**kwargs) + start(**kwargs) else: # start the debugger on a script # reimport as global __main__ namespace is destroyed - import qdb - qdb.main(**kwargs) + import dbg + dbg.main(**kwargs) + diff --git a/gluon/contrib/populate.py b/gluon/contrib/populate.py index 92c1fec63..6977e8d5f 100644 --- a/gluon/contrib/populate.py +++ b/gluon/contrib/populate.py @@ -1,6 +1,7 @@ from __future__ import print_function +from gluon._compat import pickle, unicodeT +from gluon.fileutils import open_file import re -import cPickle import random import datetime @@ -32,10 +33,10 @@ def learn(self, text): self.db[item][nextitem] += 1 def save(self, filename): - cPickle.dump(self.db, open(filename, 'wb')) + pickle.dump(self.db, open_file(filename, 'wb')) def load(self, filename): - self.loadd(cPickle.load(open(filename, 'rb'))) + self.loadd(pickle.load(open_file(filename, 'rb'))) def loadd(self, db): self.db = db @@ -43,7 +44,7 @@ def loadd(self, db): def generate(self, length=10000, prefix=False): replacements2 = {' ,': ',', ' \.': '.\n', ' :': ':', ' ;': ';', '\n\s+': '\n'} - keys = self.db.keys() + keys = list(self.db.keys()) key = keys[random.randint(0, len(keys) - 1)] words = key words = words.capitalize() @@ -130,7 +131,7 @@ def populate_generator(table, default=True, compute=False, contents={}): continue # if user supplied it, let it be. field = table[fieldname] - if not isinstance(field.type, (str, unicode)): + if not isinstance(field.type, (str, unicodeT)): continue elif field.type == 'id': continue diff --git a/gluon/debug.py b/gluon/debug.py index a4432590f..6887f27f6 100644 --- a/gluon/debug.py +++ b/gluon/debug.py @@ -87,9 +87,9 @@ def communicate(command=None): return ''.join(result) -# New debugger implementation using qdb and a web UI +# New debugger implementation using dbg and a web UI -import gluon.contrib.qdb as qdb +import gluon.contrib.dbg as dbg from threading import RLock interact_lock = RLock() @@ -109,11 +109,11 @@ def check_fn(self, *args, **kwargs): return check_fn -class WebDebugger(qdb.Frontend): +class WebDebugger(dbg.Frontend): """Qdb web2py interface""" def __init__(self, pipe, completekey='tab', stdin=None, stdout=None): - qdb.Frontend.__init__(self, pipe) + dbg.Frontend.__init__(self, pipe) self.clear_interaction() def clear_interaction(self): @@ -128,7 +128,7 @@ def run(self): run_lock.acquire() try: while self.pipe.poll(): - qdb.Frontend.run(self) + dbg.Frontend.run(self) finally: run_lock.release() @@ -149,23 +149,23 @@ def exception(self, title, extype, exvalue, trace, request): @check_interaction def do_continue(self): - qdb.Frontend.do_continue(self) + dbg.Frontend.do_continue(self) @check_interaction def do_step(self): - qdb.Frontend.do_step(self) + dbg.Frontend.do_step(self) @check_interaction def do_return(self): - qdb.Frontend.do_return(self) + dbg.Frontend.do_return(self) @check_interaction def do_next(self): - qdb.Frontend.do_next(self) + dbg.Frontend.do_next(self) @check_interaction def do_quit(self): - qdb.Frontend.do_quit(self) + dbg.Frontend.do_quit(self) def do_exec(self, statement): interact_lock.acquire() @@ -175,22 +175,22 @@ def do_exec(self, statement): # avoid spurious interaction notifications: self.set_burst(2) # execute the statement in the remote debugger: - return qdb.Frontend.do_exec(self, statement) + return dbg.Frontend.do_exec(self, statement) finally: interact_lock.release() # create the connection between threads: parent_queue, child_queue = Queue.Queue(), Queue.Queue() -front_conn = qdb.QueuePipe("parent", parent_queue, child_queue) -child_conn = qdb.QueuePipe("child", child_queue, parent_queue) +front_conn = dbg.QueuePipe("parent", parent_queue, child_queue) +child_conn = dbg.QueuePipe("child", child_queue, parent_queue) web_debugger = WebDebugger(front_conn) # frontend -qdb_debugger = qdb.Qdb(pipe=child_conn, redirect_stdio=False, skip=None) # backend -dbg = qdb_debugger +dbg_debugger = dbg.Qdb(pipe=child_conn, redirect_stdio=False, skip=None) # backend +dbg = dbg_debugger # enable getting context (stack, globals/locals) at interaction -qdb_debugger.set_params(dict(call_stack=True, environment=True)) +dbg_debugger.set_params(dict(call_stack=True, environment=True)) import gluon.main gluon.main.global_settings.debugging = True