cmdrosmium/lohup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: rustic support

Browse Source
This commit is contained in:
Igor Ovsyannikov
2025-09-21 15:29:05 +03:00
parent a2ff0ae91f
commit 0b05990776
10 changed files with 238 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/complete.toml
+8 -3
View File
@@ -1,24 +1,29 @@
[settings]
backup-base-dir = "/mnt/snap1"
tmp-dir = "/tmp/rustic"
# backup engine to use, "restic" (default) or "rustic"
subsystem-name = "rustic"
[settings.globalvars]
# also built-in
BTRVOL = "snap1"
CONF_BASE = "/home/osmium/backup"
[repo.cloud]
kind = "s3"
# restic by default expects path-style bucket URL
url = "https://s3.some-storage.localdomain//my-bucket/restic-backups/"
endpoint = "https://s3.some-storage.localdomain"
region = "us-east-1"
access-key-file = "$CONF_BASE/.s3/access.key"
secret-key-file = "$CONF_BASE/.s3/secret.key"
repo-key-file = "$CONF_BASE/repo-s3.password"
path = "/restic-backups"
bucket = "my-bucket"
[repo.localdir]
kind = "local"
path = "$CONF_BASE/local-repo"
restic-key-file = "$CONF_BASE/repo-local.password"
repo-key-file = "$CONF_BASE/repo-local.password"
default = true
pyproject.toml
+2 -1
View File
@@ -7,7 +7,7 @@ maintainers = [
{name = "Igor Ovsyannikov"}
]
license = "MIT"
keywords = ["restic", "backups"]
keywords = ["restic", "rustic", "backups"]
classifiers = [
"Development Status :: 3 - Alpha",
"Environment :: Console",
@@ -20,6 +20,7 @@ requires-python = ">=3.13"
dependencies = [
"click>=8.2.1",
"humanize>=4.13.0",
"tomlkit>=0.13.3",
]
[project.urls]
src/lohup/__init__.py
+2
View File
@@ -1 +1,3 @@
from lohup.app import Lohup
__all__ = ("Lohup",)
src/lohup/app.py
+39 -32
View File
@@ -1,28 +1,31 @@
import subprocess as procs
from lohup import config
from lohup.logger import BasicLogger, LogLevel
from lohup.logger import BasicLogger, LogLevel, LoggerProto
from lohup.restic import Restic
from lohup.rustic import Rustic
class Lohup:
def __init__(self, config_path: str | None, logger=None):
def __init__(self, config_path: str | None, logger: LoggerProto = None):
self._config_path = config_path or "lohup.toml"
self.config = None
self.subsystem = None
self.log = logger or BasicLogger(level=LogLevel.INFO)
def load(self):
self.config = config.TomlConfig.from_file(self._config_path, logger=self.log)
self.subsystem = self.config.settings.subsystem
if self.subsystem not in ("restic", "rustic"):
raise KeyError(f"Invalid subsystem: {self.subsystem}")
def invoke_restic(self, repo: str, args: tuple[str, ...]):
spec = self.config.repos.get(repo)
def invoke_direct(self, repo: str, args: tuple[str, ...]):
spec = self.config.repos.get(repo) if repo else self._default_repo
if spec is None:
raise KeyError(f"Unknown repo: {repo}")
restic = Restic(spec, self.log)
try:
restic.run(args)
except procs.CalledProcessError as e:
self.log.error(e)
engine = self._engine_for(spec)
with engine:
engine.run(args)
def _exechooks(self, hooks: list):
for h in hooks:
@@ -43,9 +46,13 @@ class Lohup:
raise ValueError(f"Unknown btrfs action: {spec.action}")
procs.check_call(cmd)
def _repo_for(self, profile: config.Profile):
@property
def _default_repo(self):
default_list = list(filter(lambda x: x.default, self.config.repos.values()))
default = default_list.pop() if default_list else None
return default_list.pop() if default_list else None
def _repo_for(self, profile: config.Profile):
default = self._default_repo
if profile.repo is not None:
if repo := self.config.repos.get(profile.repo):
return repo
@@ -55,52 +62,52 @@ class Lohup:
raise KeyError(f"No repo attached to profile: {profile.name!r}")
return default
def _engine_for(self, repo: config.Repository):
match self.subsystem:
case "rustic":
return Rustic(
repo, log=self.log, conf_dir=self.config.settings.build_dir
)
case "restic":
return Restic(repo, log=self.log)
case x:
raise KeyError(f"Unknown subsystem: {x}")
def backup(self, profile: str):
spec = self._profile_for(profile)
repo = self._repo_for(spec)
restic = Restic(repo, log=self.log)
engine = self._engine_for(repo)
self._exechooks(self.config.hooks.before_all)
try:
self._invoke_profile(restic, profile=profile)
self._invoke_profile(engine, profile=spec)
finally:
self._exechooks(self.config.hooks.after_all)
self.log.info("Finished!")
def backup_all(self):
restics = {}
engines = {}
for name, spec in self.config.profiles.items():
restic = Restic(self._repo_for(spec), log=self.log)
restics[name] = restic
engine = self._engine_for(self._repo_for(spec))
engines[name] = engine
self._exechooks(self.config.hooks.before_all)
try:
for name, spec in self.config.profiles.items():
restic = restics[name]
self._invoke_profile(restic, profile=spec)
self._invoke_profile(engines[name], profile=spec)
finally:
self._exechooks(self.config.hooks.after_all)
self.log.info("Finished!")
def snapshots(self, repo: str, is_json=False):
spec = self.config.repos.get(repo)
spec = self.config.repos.get(repo) if repo else self._default_repo
if spec is None:
raise KeyError(f"Unknown repo: {repo}")
restic = Restic(spec, log=self.log)
format = "json" if is_json else "text"
return restic.snapshots(format=format)
def _invoke_profile(self, restic: Restic, profile: config.Profile):
args = ["backup", "--tag", profile.name]
match profile:
case config.PathsProfile():
args.append("--read-concurrency=6")
for pth in profile.exclude_paths:
args.extend(["-e", pth])
args.extend(profile.paths)
restic.run(args)
case config.CommandProfile():
args.append("--stdin")
restic.pipe_stdout(args, src_cmd=profile.command.split())
def _invoke_profile(self, restic, profile: config.Profile):
with restic as engine:
engine.backup(profile)
self.log.info("Backup created successfully.")
def _profile_for(self, name: str):
src/lohup/cli.py
+6 -3
View File
@@ -28,7 +28,7 @@ def restic(obj: Lohup, repo, args: tuple[str, ...]):
"""
Pass command to restic
"""
obj.invoke_restic(repo, args)
obj.invoke_direct(repo, args)
@cli.command()
@@ -62,6 +62,8 @@ def snapshots(obj: Lohup, repo: str, raw_mode: bool):
summary = snap["summary"]
end = datetime.fromisoformat(summary["backup_end"])
changes = summary["files_new"] + summary["files_changed"]
total_files = summary["total_files_processed"]
changes_ratio = changes / total_files if total_files else 0
snapshots.append(
dict(
id=snap["short_id"],
@@ -71,7 +73,7 @@ def snapshots(obj: Lohup, repo: str, raw_mode: bool):
hostname=snap["hostname"],
duration=end - dt,
changes=changes,
changes_ratio=changes / summary["total_files_processed"],
changes_ratio=changes_ratio,
size_comp=summary["data_added_packed"],
size_raw=summary["data_added"],
processed=summary["total_bytes_processed"],
@@ -90,7 +92,8 @@ def snapshots(obj: Lohup, repo: str, raw_mode: bool):
click.echo(f"\tFiles changed (since previous): {snap['changes']} ({ratio})")
size = humanize.naturalsize(snap["size_raw"], binary=True)
compressed = humanize.naturalsize(snap["size_comp"], binary=True)
ratio = snap["size_raw"] / snap["processed"]
total_bytes = snap["processed"]
ratio = snap["size_raw"] / total_bytes if total_bytes else 0
ratio = click.style(f"{ratio:.1f}%", fg="blue")
click.echo(f"\tDiff size: {compressed}, unpacked: {size} ({ratio})")
click.echo(f"\tHost: {snap['hostname']}")
src/lohup/config.py
+32 -8
View File
@@ -1,3 +1,4 @@
import platform
import tomllib
from pathlib import Path
from dataclasses import dataclass
@@ -15,16 +16,29 @@ class ConfigError(ValueError):
class Settings:
backup_base_dir: Path
globalvars: dict[str, str]
build_dir: Path
subsystem: str
@staticmethod
def load(conf: dict):
with catch_errors() as catch:
basedir = conf.get("backup-base-dir")
basepath = Path(basedir) if basedir else Path.cwd()
tmpdir = conf.get("tmp-dir")
tmp_path = Path("/tmp/lohup")
if not tmpdir:
if platform.system() == "Windows":
tmp_path = Path.home().joinpath("AppData", "Local", "Temp", "lohup")
else:
tmp_path = Path(tmpdir)
settings = Settings(
backup_base_dir=basepath, globalvars=conf.get("globalvars") or {}
backup_base_dir=basepath,
globalvars=conf.get("globalvars") or {},
build_dir=tmp_path,
subsystem=conf.get("subsystem-name", "restic"),
)
settings.globalvars["BDIR"] = str(basepath)
settings.globalvars["BUILDDIR"] = str(settings.build_dir)
for key, value in settings.globalvars.items():
if "$" in value:
catch.error(
@@ -59,27 +73,33 @@ class LocalRepository:
@dataclass
class S3Repository:
name: str
url: str
endpoint: str
region: str
access_key_file: Masked
secret_key_file: Masked
repo_key_file: Masked
bucket: str
path: str
default: bool
@staticmethod
def load(name: str, conf: dict, expander: VarExpander):
repo = S3Repository(
name=name,
url=expander.expand(conf.get("url", "")),
endpoint=expander.expand(conf.get("endpoint", "https://s3.amazonaws.com")),
region=conf.get("region", ""),
access_key_file=Masked(expander.expand(conf.get("access-key-file", ""))),
secret_key_file=Masked(expander.expand(conf.get("secret-key-file", ""))),
repo_key_file=Masked(expander.expand(conf.get("repo-key-file", ""))),
bucket=expander.expand(conf.get("bucket", "")),
path=expander.expand(conf.get("path", "/")),
default=conf.get("default", False),
)
with catch_errors() as catcher:
if not repo.url:
catcher.error("field 'url': not set")
if not repo.endpoint:
catcher.error("field 'endpoint': not set")
if not repo.bucket:
catcher.error("field 'bucket': not set")
if value := repo.access_key_file.value:
if msg := ensure_exists(value, field="access-key-file"):
catcher.error(msg)
@@ -178,13 +198,15 @@ class PathsProfile:
repo: str | None
paths: list[str]
exclude_paths: list[str]
cli_args: list[str]
@dataclass
class CommandProfile:
name: str
repo: str | None
command: str
command: str | list[str]
cli_args: list[str]
Profile = PathsProfile | CommandProfile
@@ -205,7 +227,7 @@ class TomlConfig:
with catch_errors(failfast=logger.accepts(LogLevel.VERBOSE)) as catcher:
return TomlConfig._ffile_impl(name, path, catcher)
except CatcherError as e:
raise ConfigError(f"Failed to parse TOML config") from e
raise ConfigError("Failed to parse TOML config") from e
@staticmethod
def _ffile_impl(name: str, path: Path, catcher):
@@ -247,9 +269,10 @@ class TomlConfig:
)
profiles = {}
for name, opts in conf.get("profiles", {}).items():
args = opts.get("cli-args", [])
if cmd := opts.get("command"):
profiles[name] = CommandProfile(
name, repo=opts.get("repo"), command=cmd
name, repo=opts.get("repo"), command=cmd, cli_args=args
)
else:
paths = opts.get("paths")
@@ -263,6 +286,7 @@ class TomlConfig:
exclude_paths=[
expander.expand(x) for x in opts.get("exclude-paths", [])
],
cli_args=args,
)
toml = TomlConfig(
settings=settings,
src/lohup/logger.py
+18 -3
View File
@@ -6,6 +6,7 @@ import enum
class LogLevel(enum.IntEnum):
ERROR = 40
WARNING = 30
INFO = 20
VERBOSE = 15
DEBUG = 10
@@ -22,15 +23,23 @@ class CliLogger:
import traceback
msg = "".join(traceback.format_exception(msg))
click.echo(click.style(msg, fg="red"))
msg = (click.style("[error]", fg="red"), msg)
click.echo(" ".join(msg))
def warning(self, msg):
if self.level <= LogLevel.WARNING:
msg = (click.style("[warn]", fg="orange"), msg)
click.echo(" ".join(msg))
def info(self, msg):
if self.level <= LogLevel.INFO:
click.echo(click.style(msg, fg="blue"))
msg = (click.style("[info]", fg="blue"), msg)
click.echo(" ".join(msg))
def debug(self, msg):
if self.level <= LogLevel.DEBUG:
click.echo(click.style(msg, fg="white"))
msg = (click.style("[debug]", fg="white"), msg)
click.echo(" ".join(msg))
def accepts(self, level):
return self.level <= level
@@ -51,6 +60,9 @@ class BasicLogger:
else:
self.logger.error(msg)
def warning(self, msg):
self.logger.warning(msg)
def info(self, msg):
self.logger.info(msg)
@@ -59,3 +71,6 @@ class BasicLogger:
def accepts(self, level):
return self.level <= level
LoggerProto = CliLogger | BasicLogger
src/lohup/restic.py
+28 -1
View File
@@ -24,7 +24,9 @@ class Restic:
if masked := self.repo.access_key_file:
env["AWS_ACCESS_KEY_ID"] = Path(masked.value).read_text().strip()
if masked := self.repo.secret_key_file:
env["AWS_SECRET_ACCESS_KEY"] = Path(masked.value).read_text().strip()
env["AWS_SECRET_ACCESS_KEY"] = (
Path(masked.value).read_text().strip()
)
env["RESTIC_REPOSITORY"] = f"s3:{self.repo.url}"
case config.LocalRepository():
env["RESTIC_REPOSITORY"] = self.repo.path
@@ -34,6 +36,25 @@ class Restic:
cmd, env = self._prepare(args)
procs.check_call(cmd, env=env)
def backup(self, profile: config.Profile):
args = ["backup", "--tag", profile.name]
args.extend(profile.cli_args)
match profile:
case config.PathsProfile():
for pth in profile.exclude_paths:
args.extend(["-e", pth])
args.extend(profile.paths)
self.run(args)
case config.CommandProfile():
args.append("--stdin")
match profile.command:
case str(x):
self.pipe_stdout(args, src_cmd=x.split())
case list(x):
self.pipe_stdout(args, src_cmd=x)
case _:
raise NotImplementedError(profile.command)
def pipe_stdout(self, args: list[str], src_cmd: list[str]):
cmd = [self.binary] + args
env = self.environ()
@@ -56,3 +77,9 @@ class Restic:
cmd.extend(args)
env = self.environ()
return cmd, env
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
return self
src/lohup/rustic.py
+92
View File
@@ -0,0 +1,92 @@
from pathlib import Path
from dataclasses import dataclass, field
import subprocess as procs
import json
import tomlkit
from lohup import config
from lohup.logger import LoggerProto
@dataclass
class Rustic:
repo: config.Repository
log: LoggerProto
conf_dir: Path
binary: str = field(default="rustic")
@property
def conf_file(self) -> Path:
return self.conf_dir / "rustic.toml"
def write_config(self) -> Path:
out = {}
repo_pass = Path(self.repo.repo_key_file.value).read_text().strip()
out["repository"] = {"password": repo_pass}
match self.repo:
case config.S3Repository():
out["repository"]["repository"] = "opendal:s3"
opts = {}
if masked := self.repo.access_key_file:
opts["access_key_id"] = Path(masked.value).read_text().strip()
if masked := self.repo.secret_key_file:
opts["secret_access_key"] = Path(masked.value).read_text().strip()
opts["endpoint"] = self.repo.endpoint
opts["bucket"] = self.repo.bucket
opts["root"] = self.repo.path
opts["region"] = self.repo.region
out["repository"]["options"] = opts
case config.LocalRepository():
out["repository"]["repository"] = self.repo.path
self.conf_dir.mkdir(exist_ok=True)
with self.conf_file.open("w") as f:
tomlkit.dump(out, f)
def _cmdline(self):
out = [self.binary, "--log-level=warn", "-P", str(self.conf_dir / "rustic")]
return out
def run(self, args):
cmd = self._cmdline()
cmd.extend(args)
procs.check_call(cmd)
def backup(self, profile: config.Profile):
args = ["backup", "--tag", profile.name]
args.extend(profile.cli_args)
match profile:
case config.PathsProfile():
for pth in profile.exclude_paths:
args.extend(["--glob", f"!{pth}"])
args.extend(profile.paths)
self.run(args)
case config.CommandProfile():
args.append("-")
match profile.command:
case str(x):
self.pipe_stdout(args, src_cmd=x.split())
case list(x):
self.pipe_stdout(args, src_cmd=x)
def pipe_stdout(self, args: list[str], src_cmd: list[str]):
cmd = self._cmdline()
cmd.extend(args)
with procs.Popen(cmd, stdin=procs.PIPE) as rustic:
procs.check_call(src_cmd, stdout=rustic.stdin)
def snapshots(self, format="text"):
cmd = self._cmdline()
cmd.extend(["snapshots", "--compact"])
if format == "json":
cmd.append("--json")
result = procs.check_output(cmd, encoding="utf-8")
if format == "json":
return json.loads(result)
return result
def __enter__(self):
self.write_config()
return self
def __exit__(self, type, value, traceback):
self.conf_file.unlink(True)
uv.lock
Generated
+11
View File
@@ -57,6 +57,7 @@ source = { editable = "." }
dependencies = [
{ name = "click" },
{ name = "humanize" },
{ name = "tomlkit" },
]
[package.dev-dependencies]
@@ -69,6 +70,7 @@ dev = [
requires-dist = [
{ name = "click", specifier = ">=8.2.1" },
{ name = "humanize", specifier = ">=4.13.0" },
{ name = "tomlkit", specifier = ">=0.13.3" },
]
[package.metadata.requires-dev]
@@ -199,3 +201,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887
wheels = [
{ url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" },
]
[[package]]
name = "tomlkit"
version = "0.13.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" },
]