Merge pull request #83 from dimagi/sk/long-cols

check for max column lengths

commcare_export/cli.py

@@ -136,20 +136,12 @@ def main_with_args(args):
         print(json.dumps(query.to_jvalue(), indent=4))
         exit(0)
 
-    if not args.username:
-        args.username = input('Please provide a username: ')
-
-    if not args.password:
-        # Windows getpass does not accept unicode
-        args.password = getpass.getpass()
-
     # Build an API client using either the URL provided, or the URL for a known alias
     commcarehq_base_url = commcare_hq_aliases.get(args.commcare_hq, args.commcare_hq)
     api_client = CommCareHqClient(url =commcarehq_base_url,
                                   project = args.project,
                                   version = args.api_version)
 
-    api_client = api_client.authenticated(username=args.username, password=args.password, mode=args.auth_mode)
     checkpoint_manager = None
     if args.output_format == 'xlsx':
         writer = writers.Excel2007TableWriter(args.output)
@@ -169,6 +161,12 @@ def main_with_args(args):
         # Output should be a connection URL
         # Writer had bizarre issues so we use a full connection instead of passing in a URL or engine
         writer = writers.SqlTableWriter(args.output, args.strict_types)
+
+        long_fields = _get_long_fields(query, writer.max_column_length)
+        if long_fields:
+            _print_long_field_warning(long_fields, writer.max_column_length)
+            return 1
+
         checkpoint_manager = CheckpointManager(args.output)
         with checkpoint_manager:
             checkpoint_manager.create_checkpoint_table()
@@ -183,6 +181,15 @@ def main_with_args(args):
             else:
                 logger.warn('No successful runs found, and --since not specified: will import ALL data')
 
+    if not args.username:
+        args.username = input('Please provide a username: ')
+
+    if not args.password:
+        # Windows getpass does not accept unicode
+        args.password = getpass.getpass()
+
+    api_client = api_client.authenticated(username=args.username, password=args.password, mode=args.auth_mode)
+
     if args.since:
         logger.debug('Starting from %s', args.since)
     since = dateutil.parser.parse(args.since) if args.since else None
@@ -211,6 +218,32 @@ def main_with_args(args):
     else:
         print(json.dumps(list(results), indent=4, default=RepeatableIterator.to_jvalue))
 
+
+def _get_long_fields(query, max_length):
+    long_fields_by_table = {}
+    j_query = query.to_jvalue()
+    for table_query in j_query['List']:
+        long_fields = [
+            heading['Lit'] for heading in table_query['Emit']['headings']
+            if len(heading['Lit']) > max_length
+        ]
+        if long_fields:
+            long_fields_by_table[table_query['Emit']['table']] = long_fields
+    return long_fields_by_table
+
+
+def _print_long_field_warning(long_fields, max_length):
+    for table, headers in long_fields.items():
+        logger.error(
+            'Table "%s" has field names longer than the maximum allowed for this database (%s):',
+            table, max_length
+        )
+        for header in headers:
+            logger.error('    %s', header)
+
+    print('\nPlease adjust field names to be within the maximum length limit of {}'.format(max_length))
+
+
 def entry_point():
     main(sys.argv[1:])
 

commcare_export/writers.py

@@ -207,6 +207,19 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.connection.close()
 
+    @property
+    def max_column_length(self):
+        if 'postgres' in self.db_url:
+            # https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+            return 63
+        if 'mysql' in self.db_url:
+            # https://dev.mysql.com/doc/refman/8.0/en/identifiers.html
+            return 64
+        if 'mssql' in self.db_url:
+            # https://docs.microsoft.com/en-us/sql/relational-databases/databases/database-identifiers?view=sql-server-2017
+            return 128
+        raise Exception("Unknown database dialect: {}".format(self.db_url))
+
     @property
     def metadata(self):
         if not hasattr(self, '_metadata') or self._metadata.bind.closed or self._metadata.bind.invalidated: