add regression tests

shublang/shublang.py

@@ -150,6 +150,7 @@ def sanitize(iterable):
     # TODO change name and add other options
 
     iterable = (x.strip() for x in iterable)
+    iterable = (x for x in iterable if x)
     iterable = (re.sub(r'[\n\t\r\s]+', ' ', x) for x in iterable)
     iterable = (x.encode('ascii', errors='ignore').decode('ascii') for x in iterable)
     iterable = (replace_entities(x) for x in iterable)

tests/test_regression.py

@@ -0,0 +1,14 @@
+"""This serves as a test bed for using shublang in complex real-world scenarios.
+"""
+
+from shublang import evaluate
+
+
+def test_get_non_empty_price():
+    data = ['  \r\t   \n   ', '\n', '\n', '  \tprice: $123,823.00 \n', ' ']
+    expression = 'sanitize | sub("[\$,]", "")'
+
+    data = evaluate(expression, data)
+    assert data == ['price: 123823.00']
+
+    assert evaluate('re_search("(\d+\.\d{2})") | first | float | first', data) == 123823.00