Merge pull request #74 from dimagi/sk/alembic

switch to using alembic for migrations

.travis.yml

@@ -1,15 +1,31 @@
-dist: precise # https://github.com/travis-ci/travis-ci/issues/8331
 language: python
+sudo: required
 python:
  - "2.7"
  - "3.6"
+env:
+  global:
+    - MSSQL_SA_PASSWORD=Password@123
+before_install:
+ - docker pull microsoft/mssql-server-linux:2017-latest
+ - docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PASSWORD" -p 1433:1433 --name mssql1 -d microsoft/mssql-server-linux:2017-latest
+ - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+ - echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/14.04/prod trusty main" | sudo tee /etc/apt/sources.list.d/mssql-release.list
+ - sudo apt-get update -qq
 install:
  - pip install -e .
- - pip install pymysql psycopg2
+ - pip install pymysql psycopg2 pyodbc
  - pip install coverage coveralls
+ - sudo ACCEPT_EULA=Y apt-get install msodbcsql17
+before_script:
+ - mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'travis'@'%';";
+ - docker ps -a
+ - odbcinst -q -d
+ - .travis/wait.sh
 script: coverage run setup.py test
 after_success:
  - coveralls
 services:
  - postgres
  - mysql
+ - docker

.travis/wait.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo "Waiting MSSQL docker to launch on 1433..."
+
+while ! nc -z localhost 1433; do
+  sleep 0.1
+done
+
+echo "MSSQL launched"

MANIFEST.in

@@ -1 +1 @@
-recursive-include migrations *.py *.cfg
+recursive-include migrations *.py *.ini

README.md

@@ -86,13 +86,13 @@ CommCare Export uses SQLAlachemy's [create_engine](http://docs.sqlalchemy.org/en
 
 ```
 # Postgres
-postgresql://scott:tiger@localhost/mydatabase
+postgresql+psycopg2://scott:tiger@localhost/mydatabase
 
 # MySQL
-mysql://scott:tiger@localhost/foo
+mysql+pymysql://scott:tiger@localhost/mydatabase
 
-# SQLite
-sqlite:///foo.db
+# MSSQL
+mssql+pyodbc://scott:tiger@localhost/mydatabases?driver=ODBC+Driver+17+for+SQL+Server
 ```
 
 
@@ -316,7 +316,7 @@ $ pip install openpyxl
 $ pip install xlwt
 
 # To sync with a SQL database
-$ pip install SQLAlchemy alembic
+$ pip install SQLAlchemy alembic psycopg2 pymysql pyodbc
 ```
 
 Contributing
@@ -396,3 +396,41 @@ https://pypi.python.org/pypi/commcare-export
 5\. Create a release on github
 
 https://github.com/dimagi/commcare-export/releases
+
+Testing databases
+-----------------
+Supported databases are PostgreSQL, MySQL, MSSQL
+
+Postgresql
+==========
+```
+$ docker pull postgres 9.6
+$ docker run --name ccexport-postgres -p 5432:5432 -d postgres:9.6
+```
+
+MySQL
+=====
+```
+$ docker pull mysql
+$ docker run --name ccexport-mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=pw -e MYSQL_USER=travis -e MYSQL_PASSWORD='' -d mysql
+
+# create travis user
+$ docker run -it --link ccexport-mysql:mysql --rm mysql sh -c 'exec mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD"'
+mysql> CREATE USER 'travis'@'%';
+mysql> GRANT ALL PRIVILEGES ON *.* TO 'travis'@'%';
+```
+
+MSSQL
+=====
+```
+$ docker pull microsoft/mssql-server-linux:2017-latest
+$ docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=Password@123" -p 1433:1433 --name mssql1 -d microsoft/mssql-server-linux:2017-latest
+
+# install driver
+$ curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+$ echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/16.04/prod xenial main" | sudo tee /etc/apt/sources.list.d/mssql-release.list
+
+$ sudo apt-get update -qq
+$ sudo ACCEPT_EULA=Y apt-get install msodbcsql17
+$ odbcinst -q -d
+```

commcare_export/checkpoint.py

@@ -29,17 +29,11 @@ def set_checkpoint(self, query, query_md5, checkpoint_time=None, run_complete=Fa
             self._cleanup(query_md5)
 
     def create_checkpoint_table(self):
-        import migrate.versioning.api
-        import migrate.exceptions
-        try:
-            migrate.versioning.api.version_control(self.db_url, self.migrations_repository)
-        except migrate.exceptions.DatabaseAlreadyControlledError:
-            pass
-        db_version = migrate.versioning.api.db_version(self.db_url, self.migrations_repository)
-        repo_version = migrate.versioning.api.version(self.migrations_repository)
-        if repo_version > db_version:
-            migrate.versioning.api.upgrade(self.db_url, self.migrations_repository)
-
+        from alembic import command, config
+        cfg = config.Config(os.path.join(self.migrations_repository, 'alembic.ini'))
+        with self.engine.begin() as connection:
+            cfg.attributes['connection'] = connection
+            command.upgrade(cfg, "head")
 
     def _insert_checkpoint(self, **row):
         table = self.table(self.table_name)

commcare_export/writers.py

@@ -182,8 +182,8 @@ class SqlMixin(object):
     (TODO) with "upsert" based on primary key.
     """
 
-    MIN_VARCHAR_LEN = 32  # Since SQLite does not actually support ALTER COLUMN type, let's maximize the chance that we do not have to write workarounds by starting medium
-    MAX_VARCHAR_LEN = 255  # Arbitrary point at which we switch to TEXT; for postgres VARCHAR == TEXT anyhow and for Sqlite it doesn't matter either
+    MIN_VARCHAR_LEN = 32
+    MAX_VARCHAR_LEN = 255  # Arbitrary point at which we switch to TEXT; for postgres VARCHAR == TEXT anyhow
 
     def __init__(self, db_url, poolclass=None):
         try:
@@ -240,18 +240,13 @@ def __init__(self, db_url, strict_types=False, poolclass=None):
         super(SqlTableWriter, self).__init__(db_url, poolclass=poolclass)
         self.strict_types = strict_types
 
-    @property
-    def is_sqllite(self):
-        return 'sqlite' in self.connection.engine.driver
-
     def best_type_for(self, val):
-        if not self.is_sqllite:
-            if isinstance(val, bool):
-                return self.sqlalchemy.Boolean()
-            elif isinstance(val, datetime.datetime):
-                return self.sqlalchemy.DateTime()
-            elif isinstance(val, datetime.date):
-                return self.sqlalchemy.Date()
+        if isinstance(val, bool):
+            return self.sqlalchemy.Boolean()
+        elif isinstance(val, datetime.datetime):
+            return self.sqlalchemy.DateTime()
+        elif isinstance(val, datetime.date):
+            return self.sqlalchemy.Date()
 
         if isinstance(val, int):
             return self.sqlalchemy.Integer()
@@ -260,8 +255,6 @@ def best_type_for(self, val):
             # 1. PostgreSQL is the best; you can use TEXT everywhere and it works like a charm.
             # 2. MySQL cannot build an index on TEXT due to the lack of a field length, so we
             #    try to use VARCHAR when possible.
-            # 3. But SQLite really barfs on altering columns. Luckily it does actually have real types,
-            #    so we count on other parts of this code to not bother running column alterations
             if len(val) < self.MAX_VARCHAR_LEN: # FIXME: Is 255 an interesting cutoff?
                 return self.sqlalchemy.Unicode( max(len(val), self.MIN_VARCHAR_LEN), collation=self.collation)
             else:
@@ -294,6 +287,13 @@ def compatible(self, source_type, dest_type):
             self.sqlalchemy.DateTime: (self.sqlalchemy.String, self.sqlalchemy.Text, self.sqlalchemy.Date),
             self.sqlalchemy.Date: (self.sqlalchemy.String, self.sqlalchemy.Text),
         }
+
+        # add dialect specific types
+        try:
+            compatibility[self.sqlalchemy.Boolean] += (self.sqlalchemy.dialects.mssql.base.BIT,)
+        except AttributeError:
+            pass
+
         for _type, types in compatibility.items():
             if isinstance(source_type, _type):
                 return isinstance(dest_type, (_type,) + types)
@@ -374,10 +374,6 @@ def get_cols():
                     new_type = self.strict_types_compatibility_check(ty, current_ty)
                 elif not self.compatible(ty, current_ty):
                     new_type = self.least_upper_bound(ty, current_ty)
-                    if self.is_sqllite:
-                        logger.warn('Type mismatch detected for column %s (%s != %s) '
-                                    'but sqlite does not support changing column types', columns[column], current_ty, new_type)
-                        continue
 
                 if new_type:
                     logger.warn('Altering column %s from %s to %s for value: "%s:%s"', columns[column], current_ty, new_type, type(val), val)

migrations/README.md

@@ -0,0 +1,16 @@
+## Migrations
+
+Migrations use [alembic](http://alembic.zzzcomputing.com/en/latest).
+
+**Create new migration**
+
+```
+$ alembic -c migrations/alembic.ini revision -m "description"
+```
+
+
+**Run migrations from command line**
+
+```
+$ alembic -c migrations/alembic.ini -x "url=<db url>" upgrade <version e.g. 'head'>
+```

migrations/alembic.ini

@@ -0,0 +1,37 @@
+[alembic]
+script_location = migrations
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

migrations/env.py

@@ -0,0 +1,32 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+from sqlalchemy import create_engine
+
+config = context.config
+fileConfig(config.config_file_name)
+target_metadata = None
+
+
+def run_migrations_online():
+    connectable = config.attributes.get('connection', None)
+
+    if connectable is None:
+        cmd_line_url = context.get_x_argument(as_dictionary=True).get('url')
+        if cmd_line_url:
+            connectable = create_engine(cmd_line_url)
+        else:
+            raise Exception("No connection URL. Use '-x url=<url>'")
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+run_migrations_online()

migrations/script.py.mako

@@ -0,0 +1,23 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}