From 0b05990776ae2c4e8c7c0b203755c305a952e4f7 Mon Sep 17 00:00:00 2001 From: Igor Ovsyannikov Date: Sun, 21 Sep 2025 15:29:05 +0300 Subject: [PATCH] feat: rustic support --- examples/complete.toml | 11 +++-- pyproject.toml | 3 +- src/lohup/__init__.py | 2 + src/lohup/app.py | 71 +++++++++++++++++--------------- src/lohup/cli.py | 9 +++-- src/lohup/config.py | 40 ++++++++++++++---- src/lohup/logger.py | 21 ++++++++-- src/lohup/restic.py | 29 ++++++++++++- src/lohup/rustic.py | 92 ++++++++++++++++++++++++++++++++++++++++++ uv.lock | 11 +++++ 10 files changed, 238 insertions(+), 51 deletions(-) create mode 100644 src/lohup/rustic.py diff --git a/examples/complete.toml b/examples/complete.toml index adef167..8279dce 100644 --- a/examples/complete.toml +++ b/examples/complete.toml @@ -1,24 +1,29 @@ [settings] backup-base-dir = "/mnt/snap1" +tmp-dir = "/tmp/rustic" +# backup engine to use, "restic" (default) or "rustic" +subsystem-name = "rustic" [settings.globalvars] +# also built-in BTRVOL = "snap1" CONF_BASE = "/home/osmium/backup" [repo.cloud] kind = "s3" -# restic by default expects path-style bucket URL -url = "https://s3.some-storage.localdomain//my-bucket/restic-backups/" +endpoint = "https://s3.some-storage.localdomain" region = "us-east-1" access-key-file = "$CONF_BASE/.s3/access.key" secret-key-file = "$CONF_BASE/.s3/secret.key" repo-key-file = "$CONF_BASE/repo-s3.password" +path = "/restic-backups" +bucket = "my-bucket" [repo.localdir] kind = "local" path = "$CONF_BASE/local-repo" -restic-key-file = "$CONF_BASE/repo-local.password" +repo-key-file = "$CONF_BASE/repo-local.password" default = true diff --git a/pyproject.toml b/pyproject.toml index 089d03c..5712fa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ maintainers = [ {name = "Igor Ovsyannikov"} ] license = "MIT" -keywords = ["restic", "backups"] +keywords = ["restic", "rustic", "backups"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", @@ -20,6 +20,7 @@ requires-python = ">=3.13" dependencies = [ "click>=8.2.1", "humanize>=4.13.0", + "tomlkit>=0.13.3", ] [project.urls] diff --git a/src/lohup/__init__.py b/src/lohup/__init__.py index 169fba9..ea3835d 100644 --- a/src/lohup/__init__.py +++ b/src/lohup/__init__.py @@ -1 +1,3 @@ from lohup.app import Lohup + +__all__ = ("Lohup",) diff --git a/src/lohup/app.py b/src/lohup/app.py index e1b31b7..95f4731 100644 --- a/src/lohup/app.py +++ b/src/lohup/app.py @@ -1,28 +1,31 @@ import subprocess as procs from lohup import config -from lohup.logger import BasicLogger, LogLevel +from lohup.logger import BasicLogger, LogLevel, LoggerProto from lohup.restic import Restic +from lohup.rustic import Rustic class Lohup: - def __init__(self, config_path: str | None, logger=None): + def __init__(self, config_path: str | None, logger: LoggerProto = None): self._config_path = config_path or "lohup.toml" self.config = None + self.subsystem = None self.log = logger or BasicLogger(level=LogLevel.INFO) def load(self): self.config = config.TomlConfig.from_file(self._config_path, logger=self.log) + self.subsystem = self.config.settings.subsystem + if self.subsystem not in ("restic", "rustic"): + raise KeyError(f"Invalid subsystem: {self.subsystem}") - def invoke_restic(self, repo: str, args: tuple[str, ...]): - spec = self.config.repos.get(repo) + def invoke_direct(self, repo: str, args: tuple[str, ...]): + spec = self.config.repos.get(repo) if repo else self._default_repo if spec is None: raise KeyError(f"Unknown repo: {repo}") - restic = Restic(spec, self.log) - try: - restic.run(args) - except procs.CalledProcessError as e: - self.log.error(e) + engine = self._engine_for(spec) + with engine: + engine.run(args) def _exechooks(self, hooks: list): for h in hooks: @@ -43,9 +46,13 @@ class Lohup: raise ValueError(f"Unknown btrfs action: {spec.action}") procs.check_call(cmd) - def _repo_for(self, profile: config.Profile): + @property + def _default_repo(self): default_list = list(filter(lambda x: x.default, self.config.repos.values())) - default = default_list.pop() if default_list else None + return default_list.pop() if default_list else None + + def _repo_for(self, profile: config.Profile): + default = self._default_repo if profile.repo is not None: if repo := self.config.repos.get(profile.repo): return repo @@ -55,52 +62,52 @@ class Lohup: raise KeyError(f"No repo attached to profile: {profile.name!r}") return default + def _engine_for(self, repo: config.Repository): + match self.subsystem: + case "rustic": + return Rustic( + repo, log=self.log, conf_dir=self.config.settings.build_dir + ) + case "restic": + return Restic(repo, log=self.log) + case x: + raise KeyError(f"Unknown subsystem: {x}") + def backup(self, profile: str): spec = self._profile_for(profile) repo = self._repo_for(spec) - restic = Restic(repo, log=self.log) + engine = self._engine_for(repo) self._exechooks(self.config.hooks.before_all) try: - self._invoke_profile(restic, profile=profile) + self._invoke_profile(engine, profile=spec) finally: self._exechooks(self.config.hooks.after_all) self.log.info("Finished!") def backup_all(self): - restics = {} + engines = {} for name, spec in self.config.profiles.items(): - restic = Restic(self._repo_for(spec), log=self.log) - restics[name] = restic - + engine = self._engine_for(self._repo_for(spec)) + engines[name] = engine self._exechooks(self.config.hooks.before_all) try: for name, spec in self.config.profiles.items(): - restic = restics[name] - self._invoke_profile(restic, profile=spec) + self._invoke_profile(engines[name], profile=spec) finally: self._exechooks(self.config.hooks.after_all) self.log.info("Finished!") def snapshots(self, repo: str, is_json=False): - spec = self.config.repos.get(repo) + spec = self.config.repos.get(repo) if repo else self._default_repo if spec is None: raise KeyError(f"Unknown repo: {repo}") restic = Restic(spec, log=self.log) format = "json" if is_json else "text" return restic.snapshots(format=format) - def _invoke_profile(self, restic: Restic, profile: config.Profile): - args = ["backup", "--tag", profile.name] - match profile: - case config.PathsProfile(): - args.append("--read-concurrency=6") - for pth in profile.exclude_paths: - args.extend(["-e", pth]) - args.extend(profile.paths) - restic.run(args) - case config.CommandProfile(): - args.append("--stdin") - restic.pipe_stdout(args, src_cmd=profile.command.split()) + def _invoke_profile(self, restic, profile: config.Profile): + with restic as engine: + engine.backup(profile) self.log.info("Backup created successfully.") def _profile_for(self, name: str): diff --git a/src/lohup/cli.py b/src/lohup/cli.py index 176bb21..f89a32f 100644 --- a/src/lohup/cli.py +++ b/src/lohup/cli.py @@ -28,7 +28,7 @@ def restic(obj: Lohup, repo, args: tuple[str, ...]): """ Pass command to restic """ - obj.invoke_restic(repo, args) + obj.invoke_direct(repo, args) @cli.command() @@ -62,6 +62,8 @@ def snapshots(obj: Lohup, repo: str, raw_mode: bool): summary = snap["summary"] end = datetime.fromisoformat(summary["backup_end"]) changes = summary["files_new"] + summary["files_changed"] + total_files = summary["total_files_processed"] + changes_ratio = changes / total_files if total_files else 0 snapshots.append( dict( id=snap["short_id"], @@ -71,7 +73,7 @@ def snapshots(obj: Lohup, repo: str, raw_mode: bool): hostname=snap["hostname"], duration=end - dt, changes=changes, - changes_ratio=changes / summary["total_files_processed"], + changes_ratio=changes_ratio, size_comp=summary["data_added_packed"], size_raw=summary["data_added"], processed=summary["total_bytes_processed"], @@ -90,7 +92,8 @@ def snapshots(obj: Lohup, repo: str, raw_mode: bool): click.echo(f"\tFiles changed (since previous): {snap['changes']} ({ratio})") size = humanize.naturalsize(snap["size_raw"], binary=True) compressed = humanize.naturalsize(snap["size_comp"], binary=True) - ratio = snap["size_raw"] / snap["processed"] + total_bytes = snap["processed"] + ratio = snap["size_raw"] / total_bytes if total_bytes else 0 ratio = click.style(f"{ratio:.1f}%", fg="blue") click.echo(f"\tDiff size: {compressed}, unpacked: {size} ({ratio})") click.echo(f"\tHost: {snap['hostname']}") diff --git a/src/lohup/config.py b/src/lohup/config.py index 685fe13..dc6385d 100644 --- a/src/lohup/config.py +++ b/src/lohup/config.py @@ -1,3 +1,4 @@ +import platform import tomllib from pathlib import Path from dataclasses import dataclass @@ -15,16 +16,29 @@ class ConfigError(ValueError): class Settings: backup_base_dir: Path globalvars: dict[str, str] + build_dir: Path + subsystem: str @staticmethod def load(conf: dict): with catch_errors() as catch: basedir = conf.get("backup-base-dir") basepath = Path(basedir) if basedir else Path.cwd() + tmpdir = conf.get("tmp-dir") + tmp_path = Path("/tmp/lohup") + if not tmpdir: + if platform.system() == "Windows": + tmp_path = Path.home().joinpath("AppData", "Local", "Temp", "lohup") + else: + tmp_path = Path(tmpdir) settings = Settings( - backup_base_dir=basepath, globalvars=conf.get("globalvars") or {} + backup_base_dir=basepath, + globalvars=conf.get("globalvars") or {}, + build_dir=tmp_path, + subsystem=conf.get("subsystem-name", "restic"), ) settings.globalvars["BDIR"] = str(basepath) + settings.globalvars["BUILDDIR"] = str(settings.build_dir) for key, value in settings.globalvars.items(): if "$" in value: catch.error( @@ -59,27 +73,33 @@ class LocalRepository: @dataclass class S3Repository: name: str - url: str + endpoint: str region: str access_key_file: Masked secret_key_file: Masked repo_key_file: Masked + bucket: str + path: str default: bool @staticmethod def load(name: str, conf: dict, expander: VarExpander): repo = S3Repository( name=name, - url=expander.expand(conf.get("url", "")), + endpoint=expander.expand(conf.get("endpoint", "https://s3.amazonaws.com")), region=conf.get("region", ""), access_key_file=Masked(expander.expand(conf.get("access-key-file", ""))), secret_key_file=Masked(expander.expand(conf.get("secret-key-file", ""))), repo_key_file=Masked(expander.expand(conf.get("repo-key-file", ""))), + bucket=expander.expand(conf.get("bucket", "")), + path=expander.expand(conf.get("path", "/")), default=conf.get("default", False), ) with catch_errors() as catcher: - if not repo.url: - catcher.error("field 'url': not set") + if not repo.endpoint: + catcher.error("field 'endpoint': not set") + if not repo.bucket: + catcher.error("field 'bucket': not set") if value := repo.access_key_file.value: if msg := ensure_exists(value, field="access-key-file"): catcher.error(msg) @@ -178,13 +198,15 @@ class PathsProfile: repo: str | None paths: list[str] exclude_paths: list[str] + cli_args: list[str] @dataclass class CommandProfile: name: str repo: str | None - command: str + command: str | list[str] + cli_args: list[str] Profile = PathsProfile | CommandProfile @@ -205,7 +227,7 @@ class TomlConfig: with catch_errors(failfast=logger.accepts(LogLevel.VERBOSE)) as catcher: return TomlConfig._ffile_impl(name, path, catcher) except CatcherError as e: - raise ConfigError(f"Failed to parse TOML config") from e + raise ConfigError("Failed to parse TOML config") from e @staticmethod def _ffile_impl(name: str, path: Path, catcher): @@ -247,9 +269,10 @@ class TomlConfig: ) profiles = {} for name, opts in conf.get("profiles", {}).items(): + args = opts.get("cli-args", []) if cmd := opts.get("command"): profiles[name] = CommandProfile( - name, repo=opts.get("repo"), command=cmd + name, repo=opts.get("repo"), command=cmd, cli_args=args ) else: paths = opts.get("paths") @@ -263,6 +286,7 @@ class TomlConfig: exclude_paths=[ expander.expand(x) for x in opts.get("exclude-paths", []) ], + cli_args=args, ) toml = TomlConfig( settings=settings, diff --git a/src/lohup/logger.py b/src/lohup/logger.py index 90ee949..67af8de 100644 --- a/src/lohup/logger.py +++ b/src/lohup/logger.py @@ -6,6 +6,7 @@ import enum class LogLevel(enum.IntEnum): ERROR = 40 + WARNING = 30 INFO = 20 VERBOSE = 15 DEBUG = 10 @@ -22,15 +23,23 @@ class CliLogger: import traceback msg = "".join(traceback.format_exception(msg)) - click.echo(click.style(msg, fg="red")) + msg = (click.style("[error]", fg="red"), msg) + click.echo(" ".join(msg)) + + def warning(self, msg): + if self.level <= LogLevel.WARNING: + msg = (click.style("[warn]", fg="orange"), msg) + click.echo(" ".join(msg)) def info(self, msg): if self.level <= LogLevel.INFO: - click.echo(click.style(msg, fg="blue")) + msg = (click.style("[info]", fg="blue"), msg) + click.echo(" ".join(msg)) def debug(self, msg): if self.level <= LogLevel.DEBUG: - click.echo(click.style(msg, fg="white")) + msg = (click.style("[debug]", fg="white"), msg) + click.echo(" ".join(msg)) def accepts(self, level): return self.level <= level @@ -51,6 +60,9 @@ class BasicLogger: else: self.logger.error(msg) + def warning(self, msg): + self.logger.warning(msg) + def info(self, msg): self.logger.info(msg) @@ -59,3 +71,6 @@ class BasicLogger: def accepts(self, level): return self.level <= level + + +LoggerProto = CliLogger | BasicLogger diff --git a/src/lohup/restic.py b/src/lohup/restic.py index 8ec4507..3569454 100644 --- a/src/lohup/restic.py +++ b/src/lohup/restic.py @@ -24,7 +24,9 @@ class Restic: if masked := self.repo.access_key_file: env["AWS_ACCESS_KEY_ID"] = Path(masked.value).read_text().strip() if masked := self.repo.secret_key_file: - env["AWS_SECRET_ACCESS_KEY"] = Path(masked.value).read_text().strip() + env["AWS_SECRET_ACCESS_KEY"] = ( + Path(masked.value).read_text().strip() + ) env["RESTIC_REPOSITORY"] = f"s3:{self.repo.url}" case config.LocalRepository(): env["RESTIC_REPOSITORY"] = self.repo.path @@ -34,6 +36,25 @@ class Restic: cmd, env = self._prepare(args) procs.check_call(cmd, env=env) + def backup(self, profile: config.Profile): + args = ["backup", "--tag", profile.name] + args.extend(profile.cli_args) + match profile: + case config.PathsProfile(): + for pth in profile.exclude_paths: + args.extend(["-e", pth]) + args.extend(profile.paths) + self.run(args) + case config.CommandProfile(): + args.append("--stdin") + match profile.command: + case str(x): + self.pipe_stdout(args, src_cmd=x.split()) + case list(x): + self.pipe_stdout(args, src_cmd=x) + case _: + raise NotImplementedError(profile.command) + def pipe_stdout(self, args: list[str], src_cmd: list[str]): cmd = [self.binary] + args env = self.environ() @@ -56,3 +77,9 @@ class Restic: cmd.extend(args) env = self.environ() return cmd, env + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + return self diff --git a/src/lohup/rustic.py b/src/lohup/rustic.py new file mode 100644 index 0000000..2007a39 --- /dev/null +++ b/src/lohup/rustic.py @@ -0,0 +1,92 @@ +from pathlib import Path +from dataclasses import dataclass, field +import subprocess as procs +import json +import tomlkit + +from lohup import config +from lohup.logger import LoggerProto + + +@dataclass +class Rustic: + repo: config.Repository + log: LoggerProto + conf_dir: Path + binary: str = field(default="rustic") + + @property + def conf_file(self) -> Path: + return self.conf_dir / "rustic.toml" + + def write_config(self) -> Path: + out = {} + repo_pass = Path(self.repo.repo_key_file.value).read_text().strip() + out["repository"] = {"password": repo_pass} + match self.repo: + case config.S3Repository(): + out["repository"]["repository"] = "opendal:s3" + opts = {} + if masked := self.repo.access_key_file: + opts["access_key_id"] = Path(masked.value).read_text().strip() + if masked := self.repo.secret_key_file: + opts["secret_access_key"] = Path(masked.value).read_text().strip() + opts["endpoint"] = self.repo.endpoint + opts["bucket"] = self.repo.bucket + opts["root"] = self.repo.path + opts["region"] = self.repo.region + out["repository"]["options"] = opts + case config.LocalRepository(): + out["repository"]["repository"] = self.repo.path + self.conf_dir.mkdir(exist_ok=True) + with self.conf_file.open("w") as f: + tomlkit.dump(out, f) + + def _cmdline(self): + out = [self.binary, "--log-level=warn", "-P", str(self.conf_dir / "rustic")] + return out + + def run(self, args): + cmd = self._cmdline() + cmd.extend(args) + procs.check_call(cmd) + + def backup(self, profile: config.Profile): + args = ["backup", "--tag", profile.name] + args.extend(profile.cli_args) + match profile: + case config.PathsProfile(): + for pth in profile.exclude_paths: + args.extend(["--glob", f"!{pth}"]) + args.extend(profile.paths) + self.run(args) + case config.CommandProfile(): + args.append("-") + match profile.command: + case str(x): + self.pipe_stdout(args, src_cmd=x.split()) + case list(x): + self.pipe_stdout(args, src_cmd=x) + + def pipe_stdout(self, args: list[str], src_cmd: list[str]): + cmd = self._cmdline() + cmd.extend(args) + with procs.Popen(cmd, stdin=procs.PIPE) as rustic: + procs.check_call(src_cmd, stdout=rustic.stdin) + + def snapshots(self, format="text"): + cmd = self._cmdline() + cmd.extend(["snapshots", "--compact"]) + if format == "json": + cmd.append("--json") + result = procs.check_output(cmd, encoding="utf-8") + if format == "json": + return json.loads(result) + return result + + def __enter__(self): + self.write_config() + return self + + def __exit__(self, type, value, traceback): + self.conf_file.unlink(True) diff --git a/uv.lock b/uv.lock index ecb40f1..f18d064 100644 --- a/uv.lock +++ b/uv.lock @@ -57,6 +57,7 @@ source = { editable = "." } dependencies = [ { name = "click" }, { name = "humanize" }, + { name = "tomlkit" }, ] [package.dev-dependencies] @@ -69,6 +70,7 @@ dev = [ requires-dist = [ { name = "click", specifier = ">=8.2.1" }, { name = "humanize", specifier = ">=4.13.0" }, + { name = "tomlkit", specifier = ">=0.13.3" }, ] [package.metadata.requires-dev] @@ -199,3 +201,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887 wheels = [ { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, ] + +[[package]] +name = "tomlkit" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" }, +]