Merge branch 'backup-tool' into develop

.github/requirements.txt

@@ -1,5 +1,6 @@
 PyYAML >=5.4
 distutils-pytest
+lark-parser
 pytest >=3.6.0
 pytest-dependency >=0.2
 python-dateutil

CHANGES.rst

@@ -8,6 +8,8 @@ Changelog
 New features
 ------------
 
++ `#52`_, `#70`_: Add a `backup-tool` script.
+
 + `#54`_: Add command line flags `--directory <dir>` to
   `archive-tool create`.  The script will change into this directory
   prior creating the archive if provided.
@@ -84,6 +86,7 @@ Internal changes
 .. _#48: https://github.com/RKrahl/archive-tools/pull/48
 .. _#50: https://github.com/RKrahl/archive-tools/issues/50
 .. _#51: https://github.com/RKrahl/archive-tools/pull/51
+.. _#52: https://github.com/RKrahl/archive-tools/issues/52
 .. _#53: https://github.com/RKrahl/archive-tools/issues/53
 .. _#54: https://github.com/RKrahl/archive-tools/pull/54
 .. _#55: https://github.com/RKrahl/archive-tools/issues/55
@@ -100,6 +103,7 @@ Internal changes
 .. _#66: https://github.com/RKrahl/archive-tools/pull/66
 .. _#67: https://github.com/RKrahl/archive-tools/pull/67
 .. _#68: https://github.com/RKrahl/archive-tools/pull/68
+.. _#70: https://github.com/RKrahl/archive-tools/pull/70
 
 
 0.5.1 (2020-12-12)

MANIFEST.in

@@ -11,5 +11,6 @@ include tests/data/manifest.yaml
 include tests/data/msg.txt
 include tests/data/rnd.dat
 include tests/data/rnd2.dat
+include tests/data/rnd2bis.dat
 include tests/pytest.ini
 include tests/test_*.py

README.rst

@@ -56,6 +56,10 @@ Required library packages:
 
 + `PyYAML`_
 
++ `lark-parser`_
+
+  Required for the `backup-tool.py` script.
+
 Optional library packages:
 
 + `imapclient`_
@@ -136,6 +140,7 @@ permissions and limitations under the License.
 
 .. _PyPI site: https://pypi.org/project/archive-tools/
 .. _PyYAML: http://pyyaml.org/wiki/PyYAML
+.. _lark-parser: https://github.com/lark-parser/lark
 .. _imapclient: https://github.com/mjs/imapclient/
 .. _python-dateutil: https://dateutil.readthedocs.io/en/stable/
 .. _setuptools_scm: https://github.com/pypa/setuptools_scm/

archive/bt/__init__.py

@@ -0,0 +1,49 @@
+"""Internal modules used by the backup-tool command line tool.
+"""
+
+import argparse
+import importlib
+import logging
+import sys
+from archive.exception import ArchiveError, ConfigError
+from archive.bt.config import Config
+
+log = logging.getLogger(__name__)
+subcmds = ( "create", "index", )
+
+def backup_tool():
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument('-v', '--verbose', action='store_true',
+                           help=("verbose diagnostic output"))
+    subparsers = argparser.add_subparsers(title='subcommands', dest='subcmd')
+    for sc in subcmds:
+        m = importlib.import_module('archive.bt.%s' % sc)
+        m.add_parser(subparsers)
+    args = argparser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+    if not hasattr(args, "func"):
+        argparser.error("subcommand is required")
+
+    try:
+        config = Config(args)
+    except ConfigError as e:
+        print("%s: configuration error: %s" % (argparser.prog, e),
+              file=sys.stderr)
+        sys.exit(2)
+
+    if config.policy:
+        log.info("%s %s: host:%s, policy:%s", argparser.prog, args.subcmd,
+                 config.host, config.policy)
+    else:
+        log.info("%s %s: host:%s", argparser.prog, args.subcmd, config.host)
+
+    try:
+        sys.exit(args.func(args, config))
+    except ArchiveError as e:
+        print("%s: error: %s" % (argparser.prog, e),
+              file=sys.stderr)
+        sys.exit(1)

archive/bt/config.py

@@ -0,0 +1,99 @@
+"""Configuration for the backup-tool command line tool.
+"""
+
+import datetime
+import os
+from pathlib import Path
+import pwd
+import socket
+from archive.archive import DedupMode
+import archive.config
+from archive.exception import ConfigError
+
+
+def get_config_file():
+    try:
+        return os.environ['BACKUP_CFG']
+    except KeyError:
+        return "/etc/backup.cfg"
+
+class Config(archive.config.Config):
+
+    defaults = {
+        'dirs': None,
+        'excludes': "",
+        'backupdir': None,
+        'targetdir': "%(backupdir)s",
+        'name': "%(host)s-%(date)s-%(schedule)s.tar.bz2",
+        'schedules': None,
+        'dedup': 'link',
+    }
+    args_options = ('policy', 'user')
+
+    def __init__(self, args):
+        for o in self.args_options:
+            if not hasattr(args, o):
+                setattr(args, o, None)
+        host = socket.gethostname()
+        config_file = get_config_file()
+        if args.user:
+            args.policy = 'user'
+        if args.policy:
+            sections = ("%s/%s" % (host, args.policy), host, args.policy)
+        else:
+            sections = (host,)
+        self.config_file = config_file
+        super().__init__(args, config_section=sections)
+        if not self.config_file:
+            raise ConfigError("configuration file %s not found" % config_file)
+        self['host'] = host
+        self['date'] = datetime.date.today().strftime("%y%m%d")
+        if args.user:
+            try:
+                self['home'] = pwd.getpwnam(args.user).pw_dir
+            except KeyError:
+                pass
+
+    @property
+    def host(self):
+        return self.get('host')
+
+    @property
+    def policy(self):
+        return self.get('policy')
+
+    @property
+    def user(self):
+        return self.get('user')
+
+    @property
+    def schedules(self):
+        return self.get('schedules', required=True, split='/')
+
+    @property
+    def name(self):
+        return self.get('name', required=True)
+
+    @property
+    def dirs(self):
+        return self.get('dirs', required=True, split=True, type=Path)
+
+    @property
+    def excludes(self):
+        return self.get('excludes', split=True, type=Path)
+
+    @property
+    def backupdir(self):
+        return self.get('backupdir', required=True, type=Path)
+
+    @property
+    def targetdir(self):
+        return self.get('targetdir', required=True, type=Path)
+
+    @property
+    def dedup(self):
+        return self.get('dedup', required=True, type=DedupMode)
+
+    @property
+    def path(self):
+        return self.targetdir / self.name

archive/bt/create.py

@@ -0,0 +1,118 @@
+"""Create a backup.
+"""
+
+from collections.abc import Sequence
+import datetime
+import logging
+import os
+import pwd
+from archive.archive import Archive
+from archive.exception import ArchiveCreateError
+from archive.index import ArchiveIndex
+from archive.manifest import Manifest, DiffStatus, diff_manifest
+from archive.tools import tmp_umask
+from archive.bt.schedule import ScheduleDate, BaseSchedule, NoFullBackupError
+
+
+log = logging.getLogger(__name__)
+
+def get_prev_backups(config):
+    idx_file = config.backupdir / ".index.yaml"
+    if idx_file.is_file():
+        log.debug("reading index file %s", str(idx_file))
+        with idx_file.open("rb") as f:
+            idx = ArchiveIndex(f)
+    else:
+        log.debug("index file not found")
+        idx = ArchiveIndex()
+    idx.sort()
+    f_d = dict(host=config.host, policy=config.policy)
+    if config.policy == 'user':
+        f_d['user'] = config.user
+    return list(filter(lambda i: i >= f_d, idx))
+
+def filter_fileinfos(base, fileinfos):
+    for stat, fi1, fi2 in diff_manifest(base, fileinfos):
+        if stat == DiffStatus.MISSING_B or stat == DiffStatus.MATCH:
+            continue
+        yield fi2
+
+def get_schedule(config):
+    last_schedule = None
+    schedules = []
+    for s in config.schedules:
+        try:
+            n, t = s.split(':')
+        except ValueError:
+            n = t = s
+        cls = BaseSchedule.SubClasses[t]
+        sd_str = config.get('schedule.%s.date' % n, required=True)
+        last_schedule = cls(n, ScheduleDate(sd_str), last_schedule)
+        schedules.append(last_schedule)
+    now = datetime.datetime.now()
+    for s in schedules:
+        if s.match_date(now):
+            return s
+    else:
+        log.debug("no schedule date matches now")
+        return None
+
+def get_fileinfos(config, schedule):
+    fileinfos = Manifest(paths=config.dirs, excludes=config.excludes)
+    try:
+        base_archives = schedule.get_base_archives(get_prev_backups(config))
+    except NoFullBackupError:
+        raise ArchiveCreateError("No previous full backup found, can not "
+                                 "create %s archive" % schedule.name)
+    for p in [i.path for i in base_archives]:
+        log.debug("considering %s to create differential archive", p)
+        with Archive().open(p) as base:
+            fileinfos = filter_fileinfos(base.manifest, fileinfos)
+    return fileinfos
+
+def chown(path, user):
+    try:
+        pw = pwd.getpwnam(user)
+    except KeyError:
+        log.warn("User %s not found in password database", user)
+        return
+    try:
+        os.chown(path, pw.pw_uid, pw.pw_gid)
+    except OSError as e:
+        log.error("chown %s: %s: %s", path, type(e).__name__, e)
+
+def create(args, config):
+    schedule = get_schedule(config)
+    if schedule is None:
+        return 0
+    config['schedule'] = schedule.name
+    fileinfos = get_fileinfos(config, schedule)
+    if not isinstance(fileinfos, Sequence):
+        fileinfos = list(fileinfos)
+    if not fileinfos:
+        log.debug("nothing to archive")
+        return 0
+
+    log.debug("creating archive %s", config.path)
+
+    tags = [
+        "host:%s" % config.host,
+        "policy:%s" % config.policy,
+        "schedule:%s" % schedule.name,
+        "type:%s" % schedule.ClsName,
+    ]
+    if config.user:
+        tags.append("user:%s" % config.user)
+    with tmp_umask(0o277):
+        arch = Archive().create(config.path, fileinfos=fileinfos, tags=tags,
+                                dedup=config.dedup)
+        if config.user:
+            chown(arch.path, config.user)
+    return 0
+
+def add_parser(subparsers):
+    parser = subparsers.add_parser('create', help="create a backup")
+    clsgrp = parser.add_mutually_exclusive_group()
+    clsgrp.add_argument('--policy', default='sys')
+    clsgrp.add_argument('--user')
+    parser.set_defaults(func=create)

archive/bt/index.py

@@ -0,0 +1,29 @@
+"""Update the index of backups.
+"""
+
+import logging
+from archive.index import ArchiveIndex
+
+
+log = logging.getLogger(__name__)
+
+def update_index(args, config):
+    idx_file = config.backupdir / ".index.yaml"
+    if idx_file.is_file():
+        log.debug("reading index file %s", str(idx_file))
+        with idx_file.open("rb") as f:
+            idx = ArchiveIndex(f)
+    else:
+        log.debug("index file not found")
+        idx = ArchiveIndex()
+    idx.add_archives(config.backupdir.glob("*.tar*"), prune=args.prune)
+    idx.sort()
+    with idx_file.open("wb") as f:
+        idx.write(f)
+    return 0
+
+def add_parser(subparsers):
+    parser = subparsers.add_parser('index', help="update backup index")
+    parser.add_argument('--no-prune', action='store_false', dest='prune',
+                        help="do not remove missing backups from the index")
+    parser.set_defaults(func=update_index)