Inject TagIterator into BlockIterator

.changes/unreleased/Under the Hood-20240123-161107.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Inject TagIterator into BlockIterator for greater flexibility.
+time: 2024-01-23T16:11:07.24321-05:00
+custom:
+  Author: peterallenwebb
+  Issue: "38"

dbt_common/clients/_jinja_blocks.py

@@ -98,22 +98,22 @@ def end_pat(self):
 
 
 class TagIterator:
-    def __init__(self, data):
-        self.data = data
+    def __init__(self, text):
+        self.text = text
         self.blocks = []
         self._parenthesis_stack = []
         self.pos = 0
 
     def linepos(self, end=None) -> str:
-        """Given an absolute position in the input data, return a pair of
+        """Given an absolute position in the input text, return a pair of
         line number + relative position to the start of the line.
         """
         end_val: int = self.pos if end is None else end
-        data = self.data[:end_val]
+        text = self.text[:end_val]
         # if not found, rfind returns -1, and -1+1=0, which is perfect!
-        last_line_start = data.rfind("\n") + 1
+        last_line_start = text.rfind("\n") + 1
         # it's easy to forget this, but line numbers are 1-indexed
-        line_number = data.count("\n") + 1
+        line_number = text.count("\n") + 1
         return f"{line_number}:{end_val - last_line_start}"
 
     def advance(self, new_position):
@@ -123,10 +123,10 @@ def rewind(self, amount=1):
         self.pos -= amount
 
     def _search(self, pattern):
-        return pattern.search(self.data, self.pos)
+        return pattern.search(self.text, self.pos)
 
     def _match(self, pattern):
-        return pattern.match(self.data, self.pos)
+        return pattern.match(self.text, self.pos)
 
     def _first_match(self, *patterns, **kwargs):
         matches = []
@@ -147,7 +147,7 @@ def _first_match(self, *patterns, **kwargs):
     def _expect_match(self, expected_name, *patterns, **kwargs):
         match = self._first_match(*patterns, **kwargs)
         if match is None:
-            raise UnexpectedMacroEOFError(expected_name, self.data[self.pos :])
+            raise UnexpectedMacroEOFError(expected_name, self.text[self.pos :])
         return match
 
     def handle_expr(self, match):
@@ -272,8 +272,8 @@ def __iter__(self):
 
 
 class BlockIterator:
-    def __init__(self, data):
-        self.tag_parser = TagIterator(data)
+    def __init__(self, tag_iterator):
+        self.tag_parser = tag_iterator
         self.current = None
         self.stack = []
         self.last_position = 0
@@ -287,7 +287,7 @@ def current_end(self):
 
     @property
     def data(self):
-        return self.tag_parser.data
+        return self.tag_parser.text
 
     def is_current_end(self, tag):
         return (

dbt_common/clients/jinja.py

@@ -22,7 +22,7 @@
     get_materialization_macro_name,
     get_test_macro_name,
 )
-from dbt_common.clients._jinja_blocks import BlockIterator, BlockData, BlockTag
+from dbt_common.clients._jinja_blocks import BlockIterator, BlockData, BlockTag, TagIterator
 
 from dbt_common.exceptions import (
     CompilationError,
@@ -516,7 +516,7 @@ def render_template(template, ctx: Dict[str, Any], node=None) -> str:
 
 
 def extract_toplevel_blocks(
-    data: str,
+    text: str,
     allowed_blocks: Optional[Set[str]] = None,
     collect_raw_data: bool = True,
 ) -> List[Union[BlockData, BlockTag]]:
@@ -534,4 +534,5 @@ def extract_toplevel_blocks(
     :return: A list of `BlockTag`s matching the allowed block types and (if
         `collect_raw_data` is `True`) `BlockData` objects.
     """
-    return BlockIterator(data).lex_for_blocks(allowed_blocks=allowed_blocks, collect_raw_data=collect_raw_data)
+    tag_iterator = TagIterator(text)
+    return BlockIterator(tag_iterator).lex_for_blocks(allowed_blocks=allowed_blocks, collect_raw_data=collect_raw_data)