tmplogs

tmplogs

helper/convert_to_databricks.py

@@ -36,6 +36,9 @@
 
 def findargs (contentstring, sourcepatterninit):
   # this function takes in content and sourcestring and uses a recursive function to isolate the args associated with each function invocation match
+  if tmplogs == 'true':
+    print(f"content prior to error: {contentstring}")
+    print(f"source patter prior to error: {sourcepatterninit}") 
   content = contentstring
   source_patterninit = sourcepatterninit
   initlistraw = []
@@ -1016,6 +1019,7 @@ def list_of_strings(arg):
     parser.add_argument("--except_list", type=list_of_strings, default = [str], help = "list of files of file_type under dir_path that you want to exclude from parsing")
     parser.add_argument("--customdp", type=str, default = 'false', help = "set this to true to leverage custom date part target pattern logic")
     parser.add_argument("--noisylogs", type=str, default = 'false', help = "set this to true to output additional logs for debugging")
+    parser.add_argument("--tmplogs", type=str, default = 'false', help = "set this to true to output additional tmp logs for debugging")
     parser.add_argument("--dbtmodelroot", type=str, default = 'models', help = "modify this config if dbt model root is not models/ directory")
     parser.add_argument("--onlypublishagg", type=str, default = 'false', help = "modify this config if you want to skip per file print outs and just receive agg summary")
 
@@ -1056,6 +1060,7 @@ def list_of_strings(arg):
     except_list = args.except_list
     customdp = args.customdp
     noisylogs = args.noisylogs
+    tmplogs = args.tmplogs
     dbtmodelroot = args.dbtmodelroot
     onlypublishagg = args.onlypublishagg
 

helper/pierunner.py.py

@@ -30,7 +30,7 @@
 # COMMAND ----------
 
 # MAGIC %sh
-# MAGIC python3 ./convert_to_databricks.py --sourcedb "snowflake" --dir_path "snowflake/" --parse_mode 'discovery' --parse_first 'syntax' --customdp "true"
+# MAGIC python3 ./convert_to_databricks.py --sourcedb "snowflake" --dir_path "snowflake/" --parse_mode 'discovery' --onlypublishagg 'true'
 
 # COMMAND ----------