From 031195999a413e6b38c4d66ed22ff0b8af5d0de9 Mon Sep 17 00:00:00 2001 From: Igor Ovsyannikov Date: Sun, 7 Sep 2025 11:52:26 +0300 Subject: [PATCH] feat: initial commit --- .gitignore | 10 ++ .python-version | 1 + README.md | 0 examples/complete.toml | 55 +++++++++ main.py | 6 + main.spec | 38 ++++++ pyproject.toml | 22 ++++ src/lohup/__init__.py | 1 + src/lohup/app.py | 110 +++++++++++++++++ src/lohup/cli.py | 101 +++++++++++++++ src/lohup/config.py | 274 +++++++++++++++++++++++++++++++++++++++++ src/lohup/expander.py | 28 +++++ src/lohup/logger.py | 61 +++++++++ src/lohup/restic.py | 58 +++++++++ src/lohup/util.py | 100 +++++++++++++++ tests/test_validate.py | 58 +++++++++ uv.lock | 201 ++++++++++++++++++++++++++++++ 17 files changed, 1124 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 README.md create mode 100644 examples/complete.toml create mode 100644 main.py create mode 100644 main.spec create mode 100644 pyproject.toml create mode 100644 src/lohup/__init__.py create mode 100644 src/lohup/app.py create mode 100644 src/lohup/cli.py create mode 100644 src/lohup/config.py create mode 100644 src/lohup/expander.py create mode 100644 src/lohup/logger.py create mode 100644 src/lohup/restic.py create mode 100644 src/lohup/util.py create mode 100644 tests/test_validate.py create mode 100644 uv.lock diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..505a3b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# Virtual environments +.venv diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/examples/complete.toml b/examples/complete.toml new file mode 100644 index 0000000..adef167 --- /dev/null +++ b/examples/complete.toml @@ -0,0 +1,55 @@ +[settings] +backup-base-dir = "/mnt/snap1" + +[settings.globalvars] +BTRVOL = "snap1" +CONF_BASE = "/home/osmium/backup" + + +[repo.cloud] +kind = "s3" +# restic by default expects path-style bucket URL +url = "https://s3.some-storage.localdomain//my-bucket/restic-backups/" +region = "us-east-1" +access-key-file = "$CONF_BASE/.s3/access.key" +secret-key-file = "$CONF_BASE/.s3/secret.key" +repo-key-file = "$CONF_BASE/repo-s3.password" + +[repo.localdir] +kind = "local" +path = "$CONF_BASE/local-repo" +restic-key-file = "$CONF_BASE/repo-local.password" +default = true + + +[[hooks.before-all]] +kind = "btrfs" +action = "snapshot" +subvolume = "/home" +snapshot = "$BDIR" + +[[hooks.before-all]] +kind = "command" +command = "du -sh $CONF_BASE/local-repo" + +[[hooks.after-all]] +kind = "btrfs" +action = "delete" +subvolume = "$BDIR" + + +[profiles.documents] +paths = ["$BDIR/Documents"] + +[profiles.code] +paths = ["$BDIR/code"] +exclude-paths = [ + ".cache", + "venv", + ".venv", + ".terragrunt-cache", + "node_modules" +] + +[profiles.flatpak-list] +command = "flatpak list" diff --git a/main.py b/main.py new file mode 100644 index 0000000..fd18e75 --- /dev/null +++ b/main.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +from lohup.cli import cli + +cli() + + diff --git a/main.spec b/main.spec new file mode 100644 index 0000000..2ba8dd9 --- /dev/null +++ b/main.spec @@ -0,0 +1,38 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['main.py'], + pathex=[], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.datas, + [], + name='main', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..211cc0c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "lohup" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.13" +dependencies = [ + "click>=8.2.1", + "humanize>=4.13.0", +] + +[dependency-groups] +dev = [ + "pyinstaller>=6.15.0", + "pytest>=8.4.1", +] + +[project.scripts] +lohup = "lohup.cli:cli" + +[tool.uv] +package = true diff --git a/src/lohup/__init__.py b/src/lohup/__init__.py new file mode 100644 index 0000000..169fba9 --- /dev/null +++ b/src/lohup/__init__.py @@ -0,0 +1 @@ +from lohup.app import Lohup diff --git a/src/lohup/app.py b/src/lohup/app.py new file mode 100644 index 0000000..e1b31b7 --- /dev/null +++ b/src/lohup/app.py @@ -0,0 +1,110 @@ +import subprocess as procs + +from lohup import config +from lohup.logger import BasicLogger, LogLevel +from lohup.restic import Restic + + +class Lohup: + def __init__(self, config_path: str | None, logger=None): + self._config_path = config_path or "lohup.toml" + self.config = None + self.log = logger or BasicLogger(level=LogLevel.INFO) + + def load(self): + self.config = config.TomlConfig.from_file(self._config_path, logger=self.log) + + def invoke_restic(self, repo: str, args: tuple[str, ...]): + spec = self.config.repos.get(repo) + if spec is None: + raise KeyError(f"Unknown repo: {repo}") + restic = Restic(spec, self.log) + try: + restic.run(args) + except procs.CalledProcessError as e: + self.log.error(e) + + def _exechooks(self, hooks: list): + for h in hooks: + match h: + case config.BtrfsHook(): + self._btrfs(h) + case config.CommandHook(): + procs.check_call(h.command.split()) + + @staticmethod + def _btrfs(spec: config.BtrfsHook): + cmd = ["btrfs"] + if spec.action == "snapshot": + cmd += ["subvolume", "snapshot", spec.subvolume, spec.snapshot] + elif spec.action == "delete": + cmd += ["subvolume", "delete", spec.subvolume] + else: + raise ValueError(f"Unknown btrfs action: {spec.action}") + procs.check_call(cmd) + + def _repo_for(self, profile: config.Profile): + default_list = list(filter(lambda x: x.default, self.config.repos.values())) + default = default_list.pop() if default_list else None + if profile.repo is not None: + if repo := self.config.repos.get(profile.repo): + return repo + else: + raise KeyError(f"No repo attached to profile: {profile.name!r}") + if default is None: + raise KeyError(f"No repo attached to profile: {profile.name!r}") + return default + + def backup(self, profile: str): + spec = self._profile_for(profile) + repo = self._repo_for(spec) + restic = Restic(repo, log=self.log) + self._exechooks(self.config.hooks.before_all) + try: + self._invoke_profile(restic, profile=profile) + finally: + self._exechooks(self.config.hooks.after_all) + self.log.info("Finished!") + + def backup_all(self): + restics = {} + for name, spec in self.config.profiles.items(): + restic = Restic(self._repo_for(spec), log=self.log) + restics[name] = restic + + self._exechooks(self.config.hooks.before_all) + try: + for name, spec in self.config.profiles.items(): + restic = restics[name] + self._invoke_profile(restic, profile=spec) + finally: + self._exechooks(self.config.hooks.after_all) + self.log.info("Finished!") + + def snapshots(self, repo: str, is_json=False): + spec = self.config.repos.get(repo) + if spec is None: + raise KeyError(f"Unknown repo: {repo}") + restic = Restic(spec, log=self.log) + format = "json" if is_json else "text" + return restic.snapshots(format=format) + + def _invoke_profile(self, restic: Restic, profile: config.Profile): + args = ["backup", "--tag", profile.name] + match profile: + case config.PathsProfile(): + args.append("--read-concurrency=6") + for pth in profile.exclude_paths: + args.extend(["-e", pth]) + args.extend(profile.paths) + restic.run(args) + case config.CommandProfile(): + args.append("--stdin") + restic.pipe_stdout(args, src_cmd=profile.command.split()) + self.log.info("Backup created successfully.") + + def _profile_for(self, name: str): + result = self.config.profiles.get(name) + if not result: + raise KeyError(f"Unknown backup profile: {name!r}") + return result diff --git a/src/lohup/cli.py b/src/lohup/cli.py new file mode 100644 index 0000000..176bb21 --- /dev/null +++ b/src/lohup/cli.py @@ -0,0 +1,101 @@ +import click +import humanize +from datetime import datetime + +from lohup.app import Lohup +from lohup import logger +from lohup.config import ConfigError + + +@click.group() +@click.option("--config", envvar="LOHUP_CONFIG", default="lohup.toml") +@click.pass_context +def cli(ctx, config): + log = logger.CliLogger(level=logger.LogLevel.DEBUG) + ctx.obj = Lohup(config_path=config, logger=log) + try: + ctx.obj.load() + except ConfigError as e: + log.error(e) + raise click.Abort() + + +@cli.command() +@click.option("--repo", help="Lohup repository name", required=True) +@click.argument("args", nargs=-1) +@click.pass_obj +def restic(obj: Lohup, repo, args: tuple[str, ...]): + """ + Pass command to restic + """ + obj.invoke_restic(repo, args) + + +@cli.command() +@click.argument("profile", required=True, nargs=1) +@click.pass_obj +def backup(obj: Lohup, profile: str): + obj.backup(profile=profile) + + +@cli.command() +@click.pass_obj +def backup_all(obj: Lohup): + obj.backup_all() + + +@cli.command() +@click.option("--repo", help="Lohup repository name", required=True) +@click.option("--raw", "raw_mode", is_flag=True) +@click.pass_obj +def snapshots(obj: Lohup, repo: str, raw_mode: bool): + result = obj.snapshots(repo=repo, is_json=not raw_mode) + if raw_mode: + return click.echo(result, nl=False) + # time, parent?, tree, paths, hostname, uid, gid + # tags, version, summary, id, short_id + snapshots = [] + for snap in result: + name = "-".join(snap["tags"]) + dt = datetime.fromisoformat(snap["time"]) + delta = humanize.naturaltime(datetime.now()) + summary = snap["summary"] + end = datetime.fromisoformat(summary["backup_end"]) + changes = summary["files_new"] + summary["files_changed"] + snapshots.append( + dict( + id=snap["short_id"], + name=name, + when_started=dt, + started_human=delta, + hostname=snap["hostname"], + duration=end - dt, + changes=changes, + changes_ratio=changes / summary["total_files_processed"], + size_comp=summary["data_added_packed"], + size_raw=summary["data_added"], + processed=summary["total_bytes_processed"], + ) + ) + snapshots.sort(key=lambda x: x["when_started"]) + for snap in snapshots: + name = click.style(snap["name"], fg="blue", italic=True) + click.echo(f"Snapshot {snap['id']} ({name}):") + started = humanize.naturaltime(snap["when_started"]) + text = click.style(started, fg="blue") + took = humanize.naturaldelta(snap["duration"]) + click.echo(f"\tStarted: {text} (took {took})") + ratio = snap["changes_ratio"] * 100 + ratio = click.style(f"{ratio:.1f}%", fg="blue") + click.echo(f"\tFiles changed (since previous): {snap['changes']} ({ratio})") + size = humanize.naturalsize(snap["size_raw"], binary=True) + compressed = humanize.naturalsize(snap["size_comp"], binary=True) + ratio = snap["size_raw"] / snap["processed"] + ratio = click.style(f"{ratio:.1f}%", fg="blue") + click.echo(f"\tDiff size: {compressed}, unpacked: {size} ({ratio})") + click.echo(f"\tHost: {snap['hostname']}") + click.echo() + + +if __name__ == "__main__": + cli() diff --git a/src/lohup/config.py b/src/lohup/config.py new file mode 100644 index 0000000..685fe13 --- /dev/null +++ b/src/lohup/config.py @@ -0,0 +1,274 @@ +import tomllib +from pathlib import Path +from dataclasses import dataclass + +from lohup.util import catch_errors, ensure_exists, CatcherError, Masked +from lohup.expander import VarExpander +from lohup.logger import LogLevel + + +class ConfigError(ValueError): + pass + + +@dataclass +class Settings: + backup_base_dir: Path + globalvars: dict[str, str] + + @staticmethod + def load(conf: dict): + with catch_errors() as catch: + basedir = conf.get("backup-base-dir") + basepath = Path(basedir) if basedir else Path.cwd() + settings = Settings( + backup_base_dir=basepath, globalvars=conf.get("globalvars") or {} + ) + settings.globalvars["BDIR"] = str(basepath) + for key, value in settings.globalvars.items(): + if "$" in value: + catch.error( + f"variable {key!r}: nested references in globals are not allowed" + ) + return settings + + +@dataclass +class LocalRepository: + name: str + path: str + repo_key_file: Masked + default: bool + + @staticmethod + def load(name: str, conf: dict, expander: VarExpander): + with catch_errors() as catcher: + repo = LocalRepository( + name=name, + path=expander.expand(conf.get("path")), + repo_key_file=Masked(expander.expand(conf.get("repo-key-file", ""))), + default=conf.get("default", False), + ) + if not repo.path: + catcher.error("field 'path' not set") + if msg := ensure_exists(repo.repo_key_file, field="repo-key-file"): + catcher.error(msg) + return repo + + +@dataclass +class S3Repository: + name: str + url: str + region: str + access_key_file: Masked + secret_key_file: Masked + repo_key_file: Masked + default: bool + + @staticmethod + def load(name: str, conf: dict, expander: VarExpander): + repo = S3Repository( + name=name, + url=expander.expand(conf.get("url", "")), + region=conf.get("region", ""), + access_key_file=Masked(expander.expand(conf.get("access-key-file", ""))), + secret_key_file=Masked(expander.expand(conf.get("secret-key-file", ""))), + repo_key_file=Masked(expander.expand(conf.get("repo-key-file", ""))), + default=conf.get("default", False), + ) + with catch_errors() as catcher: + if not repo.url: + catcher.error("field 'url': not set") + if value := repo.access_key_file.value: + if msg := ensure_exists(value, field="access-key-file"): + catcher.error(msg) + if value := repo.secret_key_file.value: + if msg := ensure_exists(value, field="secret-key-file"): + catcher.error(msg) + if msg := ensure_exists(repo.repo_key_file, field="repo-key-file"): + catcher.error(msg) + return repo + + +Repository = LocalRepository | S3Repository + + +@dataclass +class CommandHook: + command: str + hook_kind: str + + def load(conf: dict, kind: str, expander: VarExpander): + return CommandHook(command=expander.expand(conf.get("command")), hook_kind=kind) + + +@dataclass +class BtrfsHook: + action: str + subvolume: str + snapshot: str | None + hook_kind: str + + @staticmethod + def load(conf: dict, kind: str, expander: VarExpander): + return BtrfsHook( + action=conf.get("action"), + subvolume=expander.expand(conf.get("subvolume")), + snapshot=expander.expand(conf.get("snapshot")), + hook_kind=kind, + ) + + +Hook = CommandHook | BtrfsHook + + +@dataclass +class HookSet: + before_all: list[Hook] + after_all: list[Hook] + + @staticmethod + def load(conf: dict, expander: VarExpander): + with catch_errors() as catcher: + before_all = [] + after_all = [] + for hook in conf.get("before-all", []): + match kind := hook.get("kind"): + case "command": + catcher.catch( + lambda: before_all.append( + CommandHook.load(hook, kind=kind, expander=expander) + ), + prefix="before-all command:", + ) + case "btrfs": + catcher.catch( + lambda: before_all.append( + BtrfsHook.load(hook, kind=kind, expander=expander) + ), + prefix="before-all btrfs:", + ) + case _: + catcher.error(f"Unsupported before-all: {kind}") + for hook in conf.get("after-all", []): + match kind := hook.get("kind"): + case "command": + catcher.catch( + lambda: after_all.append( + CommandHook.load(hook, kind=kind, expander=expander) + ), + prefix="after-all command:", + ) + case "btrfs": + catcher.catch( + lambda: after_all.append( + BtrfsHook.load(hook, kind=kind, expander=expander) + ), + prefix="after-all btrfs:", + ) + case _: + catcher.error(f"after-all: unknown kind {kind!r}") + return HookSet(before_all=before_all, after_all=after_all) + + +@dataclass +class PathsProfile: + name: str + repo: str | None + paths: list[str] + exclude_paths: list[str] + + +@dataclass +class CommandProfile: + name: str + repo: str | None + command: str + + +Profile = PathsProfile | CommandProfile + + +@dataclass +class TomlConfig: + settings: Settings + repos: dict[str, Repository] + hooks: HookSet + expander: VarExpander + profiles: dict[str, Profile] + + @staticmethod + def from_file(name: str, logger): + path = Path(name) + try: + with catch_errors(failfast=logger.accepts(LogLevel.VERBOSE)) as catcher: + return TomlConfig._ffile_impl(name, path, catcher) + except CatcherError as e: + raise ConfigError(f"Failed to parse TOML config") from e + + @staticmethod + def _ffile_impl(name: str, path: Path, catcher): + if not path.exists(): + catcher.error(f"Config {name!r} does not exist") + return + with path.open("rb") as fp: + conf: dict = tomllib.load(fp) + settings = Settings.load({}) + if value := conf.get("settings"): + settings = catcher.catch(lambda: Settings.load(value), prefix="settings:") + if not settings: + return + expander = VarExpander.from_conf(settings) + repos = {} + for name, opts in conf.get("repos", {}).items(): + error_prefix = f"repo {name!r}:" + match kind := opts.get("kind"): + case "s3": + repos[name] = catcher.catch( + lambda: S3Repository.load(name, opts, expander=expander), + prefix=error_prefix, + ) + case "local": + repos[name] = catcher.catch( + lambda: LocalRepository.load(name, opts, expander=expander), + prefix=error_prefix, + ) + case None: + catcher.error(f"{error_prefix} repository type not set") + case _: + catcher.error(f"{error_prefix} unsupported kind: {kind}") + if not repos: + catcher.error("no repositories defined") + hooks = None + if hook_conf := conf.get("hooks"): + hooks = catcher.catch( + lambda: HookSet.load(hook_conf, expander=expander), prefix="hook:" + ) + profiles = {} + for name, opts in conf.get("profiles", {}).items(): + if cmd := opts.get("command"): + profiles[name] = CommandProfile( + name, repo=opts.get("repo"), command=cmd + ) + else: + paths = opts.get("paths") + if not paths: + catcher.error(f"profile {name!r}: no paths or command provided") + continue + profiles[name] = PathsProfile( + name, + repo=opts.get("repo"), + paths=[expander.expand(x) for x in paths], + exclude_paths=[ + expander.expand(x) for x in opts.get("exclude-paths", []) + ], + ) + toml = TomlConfig( + settings=settings, + repos=repos, + hooks=hooks, + expander=expander, + profiles=profiles, + ) + return toml diff --git a/src/lohup/expander.py b/src/lohup/expander.py new file mode 100644 index 0000000..f5f2c66 --- /dev/null +++ b/src/lohup/expander.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +import re + + +@dataclass +class VarExpander: + globalvars: dict[str, str] + _PATTERN = re.compile(r"(\$([\w_]+)[^\w_/$]*)+") + + def expand(self, text: str | None, extras: dict[str, str] | None = None): + if not text or "$" not in text: + return text + all_vars = self.globalvars.copy() + if extras: + all_vars.update(extras) + match = self._PATTERN.findall(text) + if not match: + return + for var, name in match: + value = all_vars.get(name) + if not value: + raise KeyError(f"variable {var} is not defined") + text = text.replace(var, value) + return text + + @staticmethod + def from_conf(settings): + return VarExpander(globalvars=settings.globalvars) diff --git a/src/lohup/logger.py b/src/lohup/logger.py new file mode 100644 index 0000000..90ee949 --- /dev/null +++ b/src/lohup/logger.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass, field +import click +import logging +import enum + + +class LogLevel(enum.IntEnum): + ERROR = 40 + INFO = 20 + VERBOSE = 15 + DEBUG = 10 + TRACE = 5 + + +@dataclass +class CliLogger: + level: LogLevel = field(default=LogLevel.INFO) + + def error(self, msg): + if self.level <= LogLevel.ERROR: + if isinstance(msg, Exception): + import traceback + + msg = "".join(traceback.format_exception(msg)) + click.echo(click.style(msg, fg="red")) + + def info(self, msg): + if self.level <= LogLevel.INFO: + click.echo(click.style(msg, fg="blue")) + + def debug(self, msg): + if self.level <= LogLevel.DEBUG: + click.echo(click.style(msg, fg="white")) + + def accepts(self, level): + return self.level <= level + + +class BasicLogger: + LEVEL_DEBUG = logging.DEBUG + LEVEL_INFO = logging.INFO + LEVEL_ERROR = logging.ERROR + + def __init__(self, level=LogLevel.DEBUG): + self.level = level + self.logger = logging.Logger("lohup", level=level.value) + + def error(self, msg): + if isinstance(msg, Exception): + self.logger.error("Got exception", exc_info=msg) + else: + self.logger.error(msg) + + def info(self, msg): + self.logger.info(msg) + + def debug(self, msg): + self.logger.debug(msg) + + def accepts(self, level): + return self.level <= level diff --git a/src/lohup/restic.py b/src/lohup/restic.py new file mode 100644 index 0000000..8ec4507 --- /dev/null +++ b/src/lohup/restic.py @@ -0,0 +1,58 @@ +from pathlib import Path +from dataclasses import dataclass, field +import os +import subprocess as procs +import json + +from lohup import config +from lohup.logger import CliLogger, BasicLogger + + +@dataclass +class Restic: + repo: config.Repository + log: CliLogger | BasicLogger + binary: str = field(default="restic") + + def environ(self): + env = os.environ.copy() + env["RESTIC_PASSWORD_FILE"] = self.repo.repo_key_file.value + match self.repo: + case config.S3Repository(): + if value := self.repo.region: + env["AWS_DEFAULT_REGION"] = value + if masked := self.repo.access_key_file: + env["AWS_ACCESS_KEY_ID"] = Path(masked.value).read_text().strip() + if masked := self.repo.secret_key_file: + env["AWS_SECRET_ACCESS_KEY"] = Path(masked.value).read_text().strip() + env["RESTIC_REPOSITORY"] = f"s3:{self.repo.url}" + case config.LocalRepository(): + env["RESTIC_REPOSITORY"] = self.repo.path + return env + + def run(self, args): + cmd, env = self._prepare(args) + procs.check_call(cmd, env=env) + + def pipe_stdout(self, args: list[str], src_cmd: list[str]): + cmd = [self.binary] + args + env = self.environ() + with procs.Popen(cmd, stdin=procs.PIPE, env=env) as restic: + procs.check_call(src_cmd, stdout=restic.stdin) + + def snapshots(self, format="text"): + cmd, env = self._prepare(["snapshots", "--compact"]) + if format == "json": + cmd.append("--json") + result = procs.check_output(cmd, env=env, encoding="utf-8") + if format == "json": + return json.loads(result) + return result + + def _prepare(self, args: list[str]): + if self.repo is None: + raise ValueError("repo not set") + cmd = [self.binary] + cmd.extend(args) + env = self.environ() + return cmd, env diff --git a/src/lohup/util.py b/src/lohup/util.py new file mode 100644 index 0000000..b10df03 --- /dev/null +++ b/src/lohup/util.py @@ -0,0 +1,100 @@ +from pathlib import Path +from dataclasses import dataclass, field +from contextlib import contextmanager + + +@dataclass(repr=False) +class Masked: + value: str + + def __str__(self): + return "[masked]" + + +@contextmanager +def catch_errors(sep="\n", prefix=None, failfast=False): + catcher = ErrorCatcher(prefix=prefix, failfast=failfast) + try: + yield catcher + except: + raise + catcher.verify(sep=sep) + + +@dataclass +class ErrorCatcher: + outer_prefix: str | None = field(default=None) + prefix: str | None = field(default=None) + errorlist: list = field(default_factory=list) + failfast: bool = field(default=False) + + def catch(self, func, prefix=None): + try: + return func() + except CatcherError as e: + if e.catcher.prefix: + e.catcher.outer_prefix = prefix + else: + e.catcher.prefix = prefix + self.errorlist.append(e) + except Exception as e: + if self.failfast: + raise + self.errorlist.append(e) + + def error(self, msg): + self.errorlist.append(msg) + + def verify(self, sep="\n"): + result = self.lines() + if result: + raise CatcherError( + sep.join(self._prefixed(x) for x in result), catcher=self + ) + + def lines(self): + out = [] + extras = [] + for err in self.errorlist: + match err: + case str(): + out.append(err) + case CatcherError(): + extras.extend(err.catcher.lines()) + case _: + extras.append(str(err)) + out.extend(extras) + return [self._prefixed(x) for x in out] + + def _prefixed(self, s): + if self.outer_prefix: + return f"{self.outer_prefix} {self.prefix} {s}" + return f"{self.prefix} {s}" if self.prefix else s + + +class CatcherError(ValueError): + catcher: ErrorCatcher + + def __init__(self, message, catcher=None): + self.catcher = catcher + super().__init__(message) + + +def ensure_exists(name, expect="file", field=None) -> str | None: + if not name: + if field is not None: + return f"field {field!r}: empty value" + return "empty value" + pth = Path(str(name)) + if isinstance(name, Masked): + pth = Path(name.value) + msg = None + if not pth.exists(): + msg = f"file {name} not found" + elif expect == "file" and not pth.is_file(): + msg = f"{name}: not a file" + elif expect == "dir" and not pth.is_dir(): + msg = f"{name}: not a directory" + if msg is not None and field is not None: + return f"field {field!r}: {msg}" + return msg diff --git a/tests/test_validate.py b/tests/test_validate.py new file mode 100644 index 0000000..0285582 --- /dev/null +++ b/tests/test_validate.py @@ -0,0 +1,58 @@ +from lohup.app import Lohup +from lohup.config import ConfigError +from lohup.logger import BasicLogger, LogLevel + +import pytest + +toml1 = """ +[settings.globals] +""" + +toml2 = """ +[[hooks.before-all]] +kind = "unknown" +foo = "bar" + +[repos.local] +kind = "local" +path = "{base}/repo" +repo-key-file = "{pwfile}" +""" + + +class RepoEnvironment: + def __init__(self, tmp_path): + self.basepath = tmp_path + self.log = BasicLogger(level=LogLevel.DEBUG) + + def write_password(self, value: str = "1"): + pth = self.basepath / "password.txt" + pth.write_text("1") + return pth + + +def test_error_message(tmp_path): + env = RepoEnvironment(tmp_path) + pwfile = env.write_password() + path = tmp_path / "lohup.toml" + path.write_text(toml1.format(base=tmp_path, pwfile=pwfile)) + app = Lohup(config_path=path, logger=env.log) + msg = "no repositories defined" + with pytest.raises(ConfigError, match="Failed to parse TOML config") as exc: + app.load() + env.log.error(exc.value) + assert str(exc.value.__cause__) == msg + + +def test_hook_error(tmp_path): + env = RepoEnvironment(tmp_path) + pwfile = env.write_password() + tmp_path.joinpath("repo").mkdir() + path = tmp_path / "lohup.toml" + path.write_text(toml2.format(base=tmp_path, pwfile=pwfile)) + app = Lohup(config_path=path, logger=env.log) + msg = "hook: Unsupported before-all: unknown" + with pytest.raises(ConfigError, match="Failed to parse TOML config") as exc: + app.load() + env.log.error(exc.value) + assert str(exc.value.__cause__) == msg diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..ecb40f1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,201 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "altgraph" +version = "0.17.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/a8/7145824cf0b9e3c28046520480f207df47e927df83aa9555fb47f8505922/altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406", size = 48418, upload-time = "2023-09-25T09:04:52.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/3f/3bc3f1d83f6e4a7fcb834d3720544ca597590425be5ba9db032b2bf322a2/altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff", size = 21212, upload-time = "2023-09-25T09:04:50.691Z" }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "humanize" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/1d/3062fcc89ee05a715c0b9bfe6490c00c576314f27ffee3a704122c6fd259/humanize-4.13.0.tar.gz", hash = "sha256:78f79e68f76f0b04d711c4e55d32bebef5be387148862cb1ef83d2b58e7935a0", size = 81884, upload-time = "2025-08-25T09:39:20.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/c7/316e7ca04d26695ef0635dc81683d628350810eb8e9b2299fc08ba49f366/humanize-4.13.0-py3-none-any.whl", hash = "sha256:b810820b31891813b1673e8fec7f1ed3312061eab2f26e3fa192c393d11ed25f", size = 128869, upload-time = "2025-08-25T09:39:18.54Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "lohup" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "humanize" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pyinstaller" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.2.1" }, + { name = "humanize", specifier = ">=4.13.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pyinstaller", specifier = ">=6.15.0" }, + { name = "pytest", specifier = ">=8.4.1" }, +] + +[[package]] +name = "macholib" +version = "1.16.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altgraph" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/ee/af1a3842bdd5902ce133bd246eb7ffd4375c38642aeb5dc0ae3a0329dfa2/macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30", size = 59309, upload-time = "2023-09-25T09:10:16.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/5d/c059c180c84f7962db0aeae7c3b9303ed1d73d76f2bfbc32bc231c8be314/macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c", size = 38094, upload-time = "2023-09-25T09:10:14.188Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pefile" +version = "2023.2.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/c5/3b3c62223f72e2360737fd2a57c30e5b2adecd85e70276879609a7403334/pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc", size = 74854, upload-time = "2023-02-07T12:23:55.958Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/26/d0ad8b448476d0a1e8d3ea5622dc77b916db84c6aa3cb1e1c0965af948fc/pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6", size = 71791, upload-time = "2023-02-07T12:28:36.678Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyinstaller" +version = "6.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altgraph" }, + { name = "macholib", marker = "sys_platform == 'darwin'" }, + { name = "packaging" }, + { name = "pefile", marker = "sys_platform == 'win32'" }, + { name = "pyinstaller-hooks-contrib" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/17/b2bb4de22650adbeef401fa82a1b43028976547a8728602e4d29735b455e/pyinstaller-6.15.0.tar.gz", hash = "sha256:a48fc4644ee4aa2aa2a35e7b51f496f8fbd7eecf6a2150646bbf1613ad07bc2d", size = 4331521, upload-time = "2025-08-03T18:33:35.709Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/dd/d5c8a127446adda954f68ea7fac22772f7ab8656ad4b06df396d82574ca9/pyinstaller-6.15.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:9f00c71c40148cd1e61695b2c6f1e086693d3bcf9bfa22ab513aa4254c3b966f", size = 1016981, upload-time = "2025-08-03T18:31:52.034Z" }, + { url = "https://files.pythonhosted.org/packages/2d/2a/7b50593b419db43e48d9bdeebaac0ff92a5fe035f3c30f87ca3e1650d7e2/pyinstaller-6.15.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:cbcc8eb77320c60722030ac875883b564e00768fe3ff1721c7ba3ad0e0a277e9", size = 726337, upload-time = "2025-08-03T18:31:57.592Z" }, + { url = "https://files.pythonhosted.org/packages/77/83/7f498fba0154c57eb5fc93eb9680a2dbadb9f780a3389fb85b8d79683378/pyinstaller-6.15.0-py3-none-manylinux2014_i686.whl", hash = "sha256:c33e6302bc53db2df1104ed5566bd980b3e0ee7f18416a6e3caa908c12a54542", size = 737539, upload-time = "2025-08-03T18:32:02.221Z" }, + { url = "https://files.pythonhosted.org/packages/09/d6/e4477feab7c8379fb49e7ec95c82d0a69ad88f6ccc247f76bef3cb0e3432/pyinstaller-6.15.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:eb902d0fed3bb1f8b7190dc4df5c11f3b59505767e0d56d1ed782b853938bbf3", size = 735426, upload-time = "2025-08-03T18:32:06.485Z" }, + { url = "https://files.pythonhosted.org/packages/32/7e/ff25648276f15e2e77fc563d36d8cfcd917e077bf2a172420df3588601b4/pyinstaller-6.15.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:b4df862adae7cf1f08eff53c43ace283822447f7f528f72e4f94749062712f15", size = 732210, upload-time = "2025-08-03T18:32:21.667Z" }, + { url = "https://files.pythonhosted.org/packages/db/3d/267a7dddd0647de95d260780050ccd8228ab29d2b9edea54ed1f56800967/pyinstaller-6.15.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:b9ebf16ed0f99016ae8ae5746dee4cb244848a12941539e62ce2eea1df5a3f95", size = 732194, upload-time = "2025-08-03T18:32:29.536Z" }, + { url = "https://files.pythonhosted.org/packages/4d/61/962b2eb79ef225233e2d6e04600e998935328011dfb2fa775b1dd16b943a/pyinstaller-6.15.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:22193489e6a22435417103f61e7950363bba600ef36ec3ab1487303668c81092", size = 731256, upload-time = "2025-08-03T18:32:36.069Z" }, + { url = "https://files.pythonhosted.org/packages/67/5e/4e20e1c0e5791b09b69bef3ac921fd0cd25551b56879324ad999b92fa045/pyinstaller-6.15.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:18f743069849dbaee3e10900385f35795a5743eabab55e99dcc42f204e40a0db", size = 731148, upload-time = "2025-08-03T18:32:41.269Z" }, + { url = "https://files.pythonhosted.org/packages/88/31/28956c534991f289e2f981c715730b6241e75dc6295737a8cbd050a0cc8c/pyinstaller-6.15.0-py3-none-win32.whl", hash = "sha256:60da8f1b5071766b45c0f607d8bc3d7e59ba2c3b262d08f2e4066ba65f3544a2", size = 1312297, upload-time = "2025-08-03T18:32:50.572Z" }, + { url = "https://files.pythonhosted.org/packages/09/ab/6a45186c7f8e34c422faecd72580116a67d068158c57faa2d2f6d01faa7f/pyinstaller-6.15.0-py3-none-win_amd64.whl", hash = "sha256:cbea297e16eeda30b41c300d6ec2fd2abea4dbd8d8a32650eeec36431c94fcd9", size = 1373091, upload-time = "2025-08-03T18:32:58.133Z" }, + { url = "https://files.pythonhosted.org/packages/5b/86/72159af032b9db36f2470a3b085f79277ec1c38e7e48f8c5dc1ed16dc4e1/pyinstaller-6.15.0-py3-none-win_arm64.whl", hash = "sha256:f43c035621742cf2d19b84308c60e4e44e72c94786d176b8f6adcde351b5bd98", size = 1314305, upload-time = "2025-08-03T18:33:05.557Z" }, +] + +[[package]] +name = "pyinstaller-hooks-contrib" +version = "2025.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/d6/e5b378b7d4add8c879295c531309b0320e9c07a70458665d091760ffdc87/pyinstaller_hooks_contrib-2025.8.tar.gz", hash = "sha256:3402ad41dfe9b5110af134422e37fc5d421ba342c6cb980bd67cb30b7415641c", size = 164214, upload-time = "2025-07-27T16:37:31.943Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/34/1d973d0dae849683e53fbcda84443ce016f315e6f4dc7605ede4f56a28c3/pyinstaller_hooks_contrib-2025.8-py3-none-any.whl", hash = "sha256:8d0b8cfa0cb689a619294ae200497374234bd4e3994b3ace2a4442274c899064", size = 442346, upload-time = "2025-07-27T16:37:30.268Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +]