From 65ba27fdea3f0134e7fce908c180ca30678f254d Mon Sep 17 00:00:00 2001 From: stepan Date: Tue, 28 Apr 2015 16:34:27 +0300 Subject: [PATCH] rebase to dev, add supporting config, structures as json or yaml --- bin/pgup | 14 ++++++++----- pgup/__init__.py | 2 +- pgup/build_diff.py | 52 +--------------------------------------------- pgup/build_init.py | 23 +++++++++++++------- pgup/config.py | 15 ++++++++----- pgup/diff_maker.py | 3 ++- pgup/table.py | 12 +++++------ 7 files changed, 44 insertions(+), 77 deletions(-) diff --git a/bin/pgup b/bin/pgup index c01637b..7b4c1e1 100755 --- a/bin/pgup +++ b/bin/pgup @@ -5,7 +5,6 @@ import sys import argparse import logging import logging.handlers -import yaml from pake.shell import rm from pgup import Config as PgupConfig from pgup import build_init @@ -37,13 +36,18 @@ for dbname in pgup_config.databases: args = parser.parse_args() argv = vars(args) -logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") -logger = logging.getLogger('pgup') if args.verbose: - logger.setLevel(logging.DEBUG) + loglvl = logging.DEBUG else: - logger.setLevel(logging.INFO) + loglvl = logging.INFO + +logging.basicConfig( + level=loglvl, + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s" +) +logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) +logger = logging.getLogger('pgup') # Checking, that any of structures exists ANY_STRUCTURE_EXISTS = False diff --git a/pgup/__init__.py b/pgup/__init__.py index bdc0ce3..df7dba3 100644 --- a/pgup/__init__.py +++ b/pgup/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -__version__ = "0.2.5" +__version__ = "0.3.0" from config import Config from build_init import build_init from build_diff import build_diff diff --git a/pgup/build_diff.py b/pgup/build_diff.py index 65b7c17..fc9850f 100644 --- a/pgup/build_diff.py +++ b/pgup/build_diff.py @@ -5,6 +5,7 @@ from diff_maker import DiffMaker from counter import Counter + def build_diff(args, argv, structures, pgup_config): logger = logging.getLogger("pgup.build_diff") @@ -41,54 +42,3 @@ def build_diff(args, argv, structures, pgup_config): logger.info("{}: {}".format(dbname, DBDIR)) else: logger.info("{}: Queries not exists".format(dbname)) - - -""" -def diff(commit, config): - pipe = git("diff --name-status {} HEAD".format(commit), pipe=True) - regexp = "\\\\|".join( ["^[ADMR]\\\\s\\\\+{}".format(db) for db in config.databases] ) - diff = [] - res = pipe.grep(regexp).strip() - if res: - diff = res.split("\n") - - structure = {} - patch = {} - for db in config.databases: - structure[db] = Structure(db, config) - patch[db] = Patch(db, config) - - for line in diff: - action, fpath = line.split("\t") - split = fpath.split("/") - db, schema, path = split[0], split[1], split[2] - - structure[db].add_schema(schema) - patch[db].add_file(path, fpath, action) - - # load structure of commit to update - # and make drop statements - dump = git("rev-parse --abbrev-ref HEAD").strip() - if dump == "HEAD": - HEAD = git("rev-parse HEAD").strip() - else: - HEAD = dump - - git("checkout {}".format(commit)) - for db in config.databases: - structure[db].load_files() - patch[db].drop_statements() - git("checkout {}".format(HEAD)) - - queries = {} - names = {} - for db in config.databases: - queries[db], names[db] = patch[db].make(structure[db]) - - response = { - "overview": Patch.overview(), - "queries": queries, - "names": names - } - return response -""" \ No newline at end of file diff --git a/pgup/build_init.py b/pgup/build_init.py index 8b6cbe9..d9fca3d 100644 --- a/pgup/build_init.py +++ b/pgup/build_init.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -import re +import os import io +import json import logging import collections @@ -11,9 +12,16 @@ from counter import Counter -def parse_structure(structure_path): - with open(structure_path) as fstream: - structure = yaml.load(fstream) +def parse_structure(structure_string): + if structure_string.endswith(".yaml"): + fpath = structure_string + if os.path.exists(fpath): + with open(fpath) as fstream: + structure = yaml.load(fstream) + else: + raise Exception("Structure file not exists: {}".format(fpath)) + else: + structure = json.loads(structure_string) created = collections.defaultdict(int) @@ -40,17 +48,18 @@ def parse_structure(structure_path): "queries": queries, "names": names, "overview": u" / ".join( - [u"{} {}".format(name, count) for name, count in created.iteritems()] + [u"{} {}".format(obj_type, count) for obj_type, count in created.iteritems()] ) } + def build_init(args, argv, structures, pgup_config): logger = logging.getLogger("pgup.build_init") data = [] for dbname, param in structures: if argv[param]: - structure_path = argv[param] - data.append( (dbname, parse_structure(structure_path)) ) + structure_string = argv[param] + data.append((dbname, parse_structure(structure_string))) for dbname, dbdata in data: if dbdata["queries"]: diff --git a/pgup/config.py b/pgup/config.py index 305d9b1..35a3bda 100644 --- a/pgup/config.py +++ b/pgup/config.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import json import yaml from errors import ConfigException @@ -7,12 +8,16 @@ class Config(object): _cache = {} - def __init__(self, fpath="/etc/pgup.yaml"): - if os.path.exists(fpath): - with open(fpath) as fstream: - config = yaml.load(fstream) + def __init__(self, config_string="/etc/pgup.yaml"): + if config_string.endswith(".yaml"): + fpath = config_string + if os.path.exists(fpath): + with open(fpath) as fstream: + config = yaml.load(fstream) + else: + raise ConfigException("Config not exists: {}".format(fpath)) else: - raise ConfigException("Config not found: {}".format(fpath)) + config = json.loads(config_string) if type(config) == dict: self.__dict__ = config diff --git a/pgup/diff_maker.py b/pgup/diff_maker.py index d2ae971..c70b1d4 100644 --- a/pgup/diff_maker.py +++ b/pgup/diff_maker.py @@ -4,6 +4,7 @@ from table import Table, Column from procedure import SqlFile, Procedure + class DbChanges(object): def __init__(self, db, config): self._db = db @@ -109,7 +110,7 @@ def _get_diff(self, commit, config): from commit which started with config.databases names """ - regexp = "\\\\|".join( ["^[ADMR]\\\\s\\\\+{}".format(db) for db in config.databases] ) + regexp = "\\\\|".join(["^[ADMR]\\\\s\\\\+{}".format(db) for db in config.databases]) pipe = git("diff --name-status {} HEAD".format(self._commit), pipe=True) res = pipe.grep(regexp).strip() if res: diff --git a/pgup/table.py b/pgup/table.py index 3b69a83..7ef7b09 100644 --- a/pgup/table.py +++ b/pgup/table.py @@ -74,7 +74,7 @@ def create(self): columns = [] comments_on_columns = [] for name, clm in self._columns.iteritems(): - columns.append({"name": name, "definition":clm.definition}) + columns.append({"name": name, "definition": clm.definition}) comments_on_columns.append({"name": name, "comment": clm.description}) Table._create += 1 @@ -186,10 +186,10 @@ def add(self): if self.not_null: actions.append(Column.SET_NOT_NULL.format(name=self.name)) - if self.default != None: + if self.default is None: actions.append.append(Column.SET_DEFAULT.format(name=self.name, default=self.default)) - if self.description == None: + if self.description is None: comment = u"{} IS 'NULL'".format(self.name) else: comment = u"{} IS '{}'".format(self.name, self.description) @@ -213,8 +213,6 @@ def __init__(self, index, params): self.not_null = params["not_null"] if self.not_null: self.definition = u"{} NOT NULL".format(self.definition) - else: - self.not_null = None if "default" in params: self.default = params["default"] self.definition = u"{} DEFAULT {}".format(self.definition, self.default) @@ -244,7 +242,7 @@ def __eq__(self, column): eq = False if self.default != column.default: - if column.default == None: + if column.default is None: Column.actions.append(Column.DROP_DEFAULT.format(name=column.name)) Column._drop_default += 1 else: @@ -253,7 +251,7 @@ def __eq__(self, column): eq = False if self.description != column.description: - if column.description == None: + if column.description is None: Column.comment = u"{} IS 'NULL'".format(column.name) else: Column.comment = u"{} IS '{}'".format(column.name, column.description)