Reimplementation of cinje as a galfi DSL.

cinje/block/function.py

@@ -1,153 +1,115 @@
 # encoding: utf-8
 
-import re
+from __future__ import unicode_literals
 
-from ..util import py, pypy, ensure_buffer
+from marrow.dsl.block.function import FunctionTransformer
 from ..inline.flush import flush_template
+from ..util import ensure_buffer
 
 
+log = __import__('logging').getLogger(__name__)
 
-class Function(object):
+
+class CinjeFunctionTransformer(FunctionTransformer):
 	"""Proces function declarations within templates.
 
+	Used to track if the given function is a template function or not, transform the argument list if such optimization
+	is warranted, and to add the requisite template processing glue suffix. Functions increase scope.
+
 	Syntax:
 
-		: def <name> <arguments>
+		: def <name> <arguments>[ -> flag[, ...]]
 		: end
 
-	"""
+	Inherits:
 
-	priority = -50
+	* `name` - the name of the function
+	* `buffer` - the named collection of buffers
 
-	# Patterns to search for bare *, *args, or **kwargs declarations.
-	STARARGS = re.compile(r'(^|,\s*)\*([^*\s,]+|\s*,|$)')
-	STARSTARARGS = re.compile(r'(^|,\s*)\*\*\S+')
+	Tracks:
 
-	# Automatically add these as keyword-only scope assignments.
-	OPTIMIZE = ['_escape', '_bless', '_args']
+	* `helpers` - helpers utilized within this template function
+	* `added` - context tags added via annotation
+	* `removed` - context tags removed via annotation
 
-	def match(self, context, line):
-		"""Match code lines using the "def" keyword."""
-		return line.kind == 'code' and line.partitioned[0] == 'def'
+	As a reminder, functions are divided into:
 
-	def _optimize(self, context, argspec):
-		"""Inject speedup shortcut bindings into the argument specification for a function.
-
-		This assigns these labels to the local scope, avoiding a cascade through to globals(), saving time.
-
-		This also has some unfortunate side-effects for using these sentinels in argument default values!
-		"""
-
-		argspec = argspec.strip()
-		optimization = ", ".join(i + "=" + i for i in self.OPTIMIZE)
-		split = None
-		prefix = ''
-		suffix = ''
-
-		if argspec:
-			matches = list(self.STARARGS.finditer(argspec))
-
-			if matches:
-				split = matches[-1].span()[1]  # Inject after, a la "*args>_<", as we're positional-only arguments.
-				if split != len(argspec):
-					prefix = ', ' if argspec[split] == ',' else ''
-					suffix = '' if argspec[split] == ',' else ', ' 
-
-			else:  # Ok, we can do this a different way…
-				matches = list(self.STARSTARARGS.finditer(argspec))
-				prefix = ', *, '
-				suffix = ', '
-				if matches:
-					split = matches[-1].span()[0]  # Inject before, a la ">_<**kwargs".  We're positional-only arguments.
-					if split == 0:
-						prefix = '*, '
-					else:
-						suffix = ''
-				else:
-					split = len(argspec)
-					suffix = ''
-
-		else:
-			prefix = '*, '
-
-		if split is None:
-			return prefix + optimization + suffix
+	* `decorator` - any leading `@decorator` invocations prior to the declaration
+	* `declaration` - the function declaration itself as transformed by `process_declaration`
+	* `docstring` - the initial documentation string, if present
+	* `prefix` - any
+	* `function`
+	* `suffix`
+	* `trailer`
+	"""
+
+	__slots__ = ('helpers', 'added', 'removed')
+
+	def __init__(self, decoder):
+		super(CinjeFunctionTransformer, self).__init__(decoder)
 
-		return argspec[:split] + prefix + optimization + suffix + argspec[split:]
+		self.helpers = set()  # Specific helpers utilized within the function.
+		self.added = set()  # Flags added through annotation.
+		self.removed = set()  # Flags removed through annotation.
 
-	def __call__(self, context):
-		input = context.input
+	def process_declaration(self, context, declaration):
+		line, = declaration  # Cinje declarations can only be one line... for now.
 
-		declaration = input.next()
-		line = declaration.partitioned[1]  # We don't care about the "def".
-		line, _, annotation = line.rpartition('->')
+		text, _, annotation = line.line.partition(' ')[2].rpartition('->')
 
-		if annotation and not line:  # Swap the values back.
-			line = annotation
+		if annotation and not text:  # Swap the values back.
+			text = annotation
 			annotation = ''
 
-		name, _, line = line.partition(' ')  # Split the function name.
+		name, _, text = text.partition(' ')  # Split the function name out.
 
-		argspec = line.rstrip()
-		name = name.strip()
+		argspec = text.rstrip()
+		name = self.name = name.strip()
 		annotation = annotation.lstrip()
-		added_flags = []
-		removed_flags = []
+		annotation = {'!dirty', '!text', '!using'} | set(i.lower().strip() for i in annotation.split())
+
+		# TODO: Re-introduce positional named local scoping optimization for non-Pypy runtimes.
+		# TODO: Generalize flag processing like this into galfi.
 
-		if annotation:
-			for flag in (i.lower().strip() for i in annotation.split()):
-				if not flag.strip('!'): continue  # Handle standalone exclamation marks.
+		for flag in annotation:
+			if not flag.strip('!'): continue  # Ignore standalone exclamation marks.
+
+			if flag[0] == '!':
+				flag = flag[1:]
 
-				if flag[0] == '!':
-					flag = flag[1:]
-
-					if flag in context.flag:
-						context.flag.remove(flag)
-						removed_flags.append(flag)
-
-					continue
+				if flag in context:  # We do this rather than discard to track.
+					context.remove(flag)
+					self.removed.add(flag)
 
-				if flag not in context.flag:
-					context.flag.add(flag)
-					added_flags.append(flag)
-
-		if py == 3 and not pypy:
-			argspec = self._optimize(context, argspec)
-
-		# Reconstruct the line.
-
-		line = 'def ' + name + '(' + argspec + '):'
-
-		# yield declaration.clone(line='@cinje.Function.prepare')  # This lets us do some work before and after runtime.
-		yield declaration.clone(line=line)
-
-		context.scope += 1
-
-		for i in ensure_buffer(context, False):
-			yield i
+				continue
+
+			if flag not in context:
+				context.add(flag)
+				self.added.add(flag)
 
-		for i in context.stream:
-			yield i
+		line = line.clone(line='def ' + name + '(' + argspec + '):')
 
-		if 'using' in context.flag:  # Clean up that we were using things.
-			context.flag.remove('using')
+		for line in super(CinjeFunctionTransformer, self).process_declaration(context, [line]):
+			yield line
+
+	def egress(self, context):
+		"""Code to be executed when exiting the context of a function.
 
-		if 'text' in context.flag:
-			context.templates.append(name)
+		Always call super() last in any subclasses.
+		"""
 
-		for i in flush_template(context, reconstruct=False):  # Handle the final buffer yield if any content was generated.
-			yield i
+		if 'dirty' in context:
+			self.suffix.append(*flush_template(context, reconstruct=False))
 
-		if 'text' in context.flag:
-			context.flag.remove('text')
+		if 'text' in context:
+			self.prefix.append(*ensure_buffer(context, False))
+			context.module.templates.add(self.name)
 
-		for flag in added_flags:
-			if flag in context.flag:
-				context.flag.remove(flag)
+		if 'using' in context:
+			self.prefix.append('_using_stack = []')
 
-		for flag in removed_flags:
-			if flag not in context.flag:
-				context.flag.add(flag)
+		context.module.helpers.update(self.helpers)
 
-		context.scope -= 1
-
+		# Reset the manipulated flags to their original state.
+		context.flag.discard(self.added)
+		context.flag.update(self.removed)

cinje/block/module.py

@@ -2,85 +2,88 @@
 
 from __future__ import unicode_literals
 
-from zlib import compress
-from base64 import b64encode
-from collections import deque
+from marrow.dsl.block.module import ModuleTransformer
+from marrow.dsl.compat import py2
+from marrow.dsl.core import Line
 
-from ..util import py, Line
 
-
-def red(numbers):
-	"""Encode the deltas to reduce entropy."""
+class CinjeModuleTransformer(ModuleTransformer):
+	"""A cinje module.
 
-	line = 0
-	deltas = []
+	Where the base `ModuleTransformer` class handles line number mapping and `__futures__` imports for Python 2
+	environments, this specialization adds template function name tracking, automatic importing of helpers, and
+	in-development command-line interface `__main__` handler.
 
-	for value in numbers:
-		deltas.append(value - line)
-		line = value
+	Because cinje modules are so similar to standard Python modules, we don't actually have much work to do.
 
-	return b64encode(compress(b''.join(chr(i).encode('latin1') for i in deltas))).decode('latin1')
-
-
-
-class Module(object):
-	"""Module handler.
+	Global processing flags:
+
+	* `free` - If defined the resulting bytecode will have no runtime dependnecy on cinje itself.
+	* `nomap` - Define to disable emission of line number mappings; this can speed up translation and reduce resulting
+		bytecode size at the cost of increased debugging difficulty.
+	* `raw` - Implies `free`; make no effort to sanitize output. This is **insecure**, but blazingly fast -- use with
+		trusted or pre-sanitized input only!
+	* `unbuffered` - utilize unbuffered output; fragments will be yielded as generated, buffer construction prefixes
+		will not be generated
+
+	Inherits:
 
-	This is the initial scope, and the highest priority to ensure its processing of the preamble happens first.
+	* `buffer` - The named collection of buffers.
+
+	Tracks:
+
+	* `templates` - The names of all module scoped template functions, as a set.
+	* `helpers` - A set of declared used helpers, a shortcut for other transformers.
+	* `_imports` - A mapping of packages to the set of objects acquired from within, from parent class.
+
+	For reference, the buffers of a module are divided into:
+
+	* `comment' - Shbang, encoding declaration, any additional leading comments and whitespace.
+	* `docstring` - the docstring of the module, if present.
+	* `imports` - the initial block of imports, including whitespace.
+	* `prefix` - Any code to be inserted between imports and first non-import line.
+	* `module` - The contents of the module proper.
+	* `suffix` - Any code to be appended to the module, prior to the line mapping.
 	"""
 
-	priority = -100
+	__slots__ = ('templates', 'helpers')  # Additional data tracked by our specialization.
 
-	def match(self, context, line):
-		return 'init' not in context.flag
+	# Line templates for easy re-use later.
+	TEMPLATES = Line('__tmpl__ = ["{}"]')  # Used to record template functions at the module scope.
+	MAIN = Line('if __name__ == "__main__":')  # Used with one of the following.
+	SINGLE = Line('_cli({})', scope=1)  # There is only one template, so this is easy mode vs. the next.
+	MULTI = Line('_cli({_tmpl: _tmpl_fn for _tmpl, _tmpl_fn in locals().items() if _tmpl in __tmpl__})', scope=1)
 
-	def __call__(self, context):
-		input = context.input
-
-		context.flag.add('init')
-		context.flag.add('buffer')
+	def __init__(self, decoder):
+		"""Construct a new module scope."""
 
-		imported = False
+		super(CinjeModuleTransformer, self).__init__(decoder)
 
-		for line in input:
-			if not line.stripped or line.stripped[0] == '#':
-				if not line.stripped.startswith('##') and 'coding:' not in line.stripped:
-					yield line
-				continue
-
-			input.push(line)  # We're out of the preamble, so put that line back and stop.
-			break
-
-		# After any existing preamble, but before other imports, we inject our own.
-
-		if py == 2:
-			yield Line(0, 'from __future__ import unicode_literals')
-			yield Line(0, '')
-
-		yield Line(0, 'import cinje')
-		yield Line(0, 'from cinje.helpers import escape as _escape, bless as _bless, iterate, xmlargs as _args, _interrupt, _json')
-		yield Line(0, '')
-		yield Line(0, '')
-		yield Line(0, '__tmpl__ = []  # Exported template functions.')
-		yield Line(0, '')
-
-		for i in context.stream:
-			yield i
-
-		if context.templates:
-			yield Line(0, '')
-			yield Line(0, '__tmpl__.extend(["' + '", "'.join(context.templates) + '"])')
-			context.templates = []
-
-		# Snapshot the line number mapping.
-		mapping = deque(context.mapping)
-		mapping.reverse()
+		self.templates = set()  # The names of all module scoped template functions, as a set.
+		self.helpers = {'str'} if py2 else set()  # Helpers to import 
+
+	def egress(self, context):
+		"""Executed when exiting the buffered module scope, prior to emitting collapsed lines."""
 
-		yield Line(0, '')
+		capable = not context.flag & {'free', 'raw'}  # Able to utilize helpers.
 
-		if __debug__:
-			yield Line(0, '__mapping__ = [' + ','.join(str(i) for i in mapping) + ']')
+		if self.templates:
+			suffix = self.suffix
+
+			if 'nomap' not in context.flag:  # If mappings are enabled.
+				suffix.append('', self.TEMPLATES.format('", "'.join(self.templates)))
+
+			if __debug__ and capable:
+				self.helpers.add('_cli')
+				suffix.append('', self.MAIN)
+
+				if len(self.templates) == 1:  # Fast path for modules containing a single template function.
+					tmpl, = self.templates
+					suffix.append(self.SINGLE.format(tmpl))
+				elif 'nomap' not in context.flag:  # This requires the mapping be present.
+					suffix.append(self.MULTI)
 
-		yield Line(0, '__gzmapping__ = b"' + red(mapping).replace('"', '\"') + '"')
+		if capable:
+			self._imports['cinje.helper'].update(self.helpers)
 
-		context.flag.remove('init')
+		super(CinjeModuleTransformer, self).egress(context)

cinje/block/using.py

@@ -2,6 +2,8 @@
 
 from ..util import Line, ensure_buffer
 
+# TODO: Implement https://github.com/marrow/cinje/issues/20
+
 
 class Using(object):
 	priority = 25