From 9c67b9b94dc763d3a82eda9ae33f64a2c185fbc9 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Wed, 30 May 2018 17:13:55 +0200 Subject: [PATCH 1/2] check fields for max length --- commcare_export/cli.py | 32 ++++++++++++++++++++++++++++++++ commcare_export/writers.py | 13 +++++++++++++ 2 files changed, 45 insertions(+) diff --git a/commcare_export/cli.py b/commcare_export/cli.py index 5304d35d..1f610740 100644 --- a/commcare_export/cli.py +++ b/commcare_export/cli.py @@ -169,6 +169,12 @@ def main_with_args(args): # Output should be a connection URL # Writer had bizarre issues so we use a full connection instead of passing in a URL or engine writer = writers.SqlTableWriter(args.output, args.strict_types) + + long_fields = _get_long_fields(query, writer.max_column_length) + if long_fields: + _print_long_field_warning(long_fields, writer.max_column_length) + return 1 + checkpoint_manager = CheckpointManager(args.output) with checkpoint_manager: checkpoint_manager.create_checkpoint_table() @@ -211,6 +217,32 @@ def main_with_args(args): else: print(json.dumps(list(results), indent=4, default=RepeatableIterator.to_jvalue)) + +def _get_long_fields(query, max_length): + long_fields_by_table = {} + j_query = query.to_jvalue() + for table_query in j_query['List']: + long_fields = [ + heading['Lit'] for heading in table_query['Emit']['headings'] + if len(heading['Lit']) > max_length + ] + if long_fields: + long_fields_by_table[table_query['Emit']['table']] = long_fields + return long_fields_by_table + + +def _print_long_field_warning(long_fields, max_length): + for table, headers in long_fields.items(): + logger.error( + 'Table "%s" has field names longer than the maximum allowed for this database (%s):', + table, max_length + ) + for header in headers: + logger.error(' %s', header) + + print('\nPlease adjust field names to be within the maximum length limit of {}'.format(max_length)) + + def entry_point(): main(sys.argv[1:]) diff --git a/commcare_export/writers.py b/commcare_export/writers.py index 7e52ee02..47bb3f58 100644 --- a/commcare_export/writers.py +++ b/commcare_export/writers.py @@ -207,6 +207,19 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.connection.close() + @property + def max_column_length(self): + if 'postgres' in self.db_url: + # https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS + return 63 + if 'mysql' in self.db_url: + # https://dev.mysql.com/doc/refman/8.0/en/identifiers.html + return 64 + if 'mssql' in self.db_url: + # https://docs.microsoft.com/en-us/sql/relational-databases/databases/database-identifiers?view=sql-server-2017 + return 128 + raise Exception("Unknown database dialect: {}".format(self.db_url)) + @property def metadata(self): if not hasattr(self, '_metadata') or self._metadata.bind.closed or self._metadata.bind.invalidated: From b5cbff42249603327e77417bf0395dab4d464fe3 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Wed, 30 May 2018 17:16:30 +0200 Subject: [PATCH 2/2] get missing auth details later in process --- commcare_export/cli.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/commcare_export/cli.py b/commcare_export/cli.py index 1f610740..a3c73b88 100644 --- a/commcare_export/cli.py +++ b/commcare_export/cli.py @@ -136,20 +136,12 @@ def main_with_args(args): print(json.dumps(query.to_jvalue(), indent=4)) exit(0) - if not args.username: - args.username = input('Please provide a username: ') - - if not args.password: - # Windows getpass does not accept unicode - args.password = getpass.getpass() - # Build an API client using either the URL provided, or the URL for a known alias commcarehq_base_url = commcare_hq_aliases.get(args.commcare_hq, args.commcare_hq) api_client = CommCareHqClient(url =commcarehq_base_url, project = args.project, version = args.api_version) - api_client = api_client.authenticated(username=args.username, password=args.password, mode=args.auth_mode) checkpoint_manager = None if args.output_format == 'xlsx': writer = writers.Excel2007TableWriter(args.output) @@ -189,6 +181,15 @@ def main_with_args(args): else: logger.warn('No successful runs found, and --since not specified: will import ALL data') + if not args.username: + args.username = input('Please provide a username: ') + + if not args.password: + # Windows getpass does not accept unicode + args.password = getpass.getpass() + + api_client = api_client.authenticated(username=args.username, password=args.password, mode=args.auth_mode) + if args.since: logger.debug('Starting from %s', args.since) since = dateutil.parser.parse(args.since) if args.since else None