Compare commits

...
10 Commits
Author SHA1 Message Date
tdehaeze fb11ad022e Add --force-zero support for sync 2024-05-07 19:03:37 +02:00
tdehaeze df764ca47c Add gotify notification 2022-07-26 20:51:48 +02:00
Chronial 60b8b6731c Mark snapraid-runner.py as executable 2021-10-31 12:37:07 +01:00
Chronial 8ce2458e5e Fix support for non-numeric scrub plans (fixes #48) 2021-10-30 14:02:16 +02:00
Chronial 5524a86f7c Remove pointless division __future__ 2021-09-25 23:00:03 +02:00
Chronial 8acd9315dd Remove progress info from snapraid output 2021-09-25 22:58:37 +02:00
Chronial 4cb7e695af Use new scrub --plan, replacing --percentage
This gives us a dependency on snapraid being >= v9.0, which was released
in 2015.
2021-09-25 22:42:58 +02:00
Sean Ellingham 11affad8b7 Add argument to ignore delete threshold (#25) 2021-09-25 22:20:56 +02:00
Christian Fersch 6f8d170edc Update Changelog and explain scope 2021-02-26 23:02:10 +01:00
Pierre Toulouse 68a03ced91 Fix log rotation (#36) 2020-11-17 09:05:23 +01:00
3 changed files with 91 additions and 22 deletions

No files matched your search

+19 -2
View File
@@ -26,14 +26,31 @@ It supports Windows, Linux and macOS and requires at least python3.7.
* Can send notification emails after each run or only for failures. * Can send notification emails after each run or only for failures.
* Can run `scrub` after `sync` * Can run `scrub` after `sync`
## Scope of this project and contributions
Snapraid-runner is supposed to be a small tool with clear focus. It should not
have any dependencies to keep installation trivial. I always welcome bugfixes
and contributions, but be aware that I will not merge new features that I feel
do not fit the core purpose of this tool.
I keep the PRs for features I do not plan on merging open, so if there's a
feature you are missing, you can have a look
[at the open PRs](https://github.com/Chronial/snapraid-runner/pulls).
## Changelog ## Changelog
### Unreleased ### Unreleased
* Add --ignore-deletethreshold (by exterrestris, #25)
* Add support for scrub --plan, replacing --percentage (thanks to fmoledina)
* Remove snapraid progress output. Was accidentially introduced with python3
support.
### v0.5 (26 Feb 2021)
* Remove (broken) python2 support * Remove (broken) python2 support
* Fix snapraid output encoding handling (by hyyz17200, PR-31) * Fix snapraid output encoding handling (by hyyz17200, #31)
* Fix log rotation (by ptoulouse, #36)
### v0.4 (17 Aug 2019) ### v0.4 (17 Aug 2019)
* Add compatibility with python3 (by reed-jones) * Add compatibility with python3 (by reed-jones)
* Add support for running `snapraid touch` (by ShoGinn, PR-11) * Add support for running `snapraid touch` (by ShoGinn, #11)
* Add SMTP TLS support * Add SMTP TLS support
### v0.3 (20 Jul 2017) ### v0.3 (20 Jul 2017)
+16 -1
View File
@@ -25,6 +25,15 @@ to =
; maximum email size in KiB ; maximum email size in KiB
maxsize = 500 maxsize = 500
[gotify]
; when to send a gotify notification, comma-separated list of [success, error]
sendon = success,error
; url of the gotify server without trailing slash
url = https://gotify.server.com
; Gotify token
token =
[smtp] [smtp]
host = host =
; leave empty for default port ; leave empty for default port
@@ -38,5 +47,11 @@ password =
[scrub] [scrub]
; set to true to run scrub after sync ; set to true to run scrub after sync
enabled = false enabled = false
percentage = 12 ; scrub plan - either a percentage or one of [bad, new, full]
plan = 12
; minimum block age (in days) for scrubbing. Only used with percentage plans
older-than = 10 older-than = 10
[sync]
; set to true to force sync files with zero size
force-zero = false
Regular → Executable
+56 -19
View File
@@ -1,6 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from __future__ import division
import argparse import argparse
import configparser import configparser
import logging import logging
@@ -11,6 +9,7 @@ import sys
import threading import threading
import time import time
import traceback import traceback
import requests
from collections import Counter, defaultdict from collections import Counter, defaultdict
from io import StringIO from io import StringIO
@@ -26,11 +25,7 @@ def tee_log(infile, out_lines, log_level):
""" """
def tee_thread(): def tee_thread():
for line in iter(infile.readline, ""): for line in iter(infile.readline, ""):
line = line.strip() logging.log(log_level, line.rstrip())
# Do not log the progress display
if "\r" in line:
line = line.split("\r")[-1]
logging.log(log_level, line.strip())
out_lines.append(line) out_lines.append(line)
infile.close() infile.close()
t = threading.Thread(target=tee_thread) t = threading.Thread(target=tee_thread)
@@ -44,19 +39,20 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]):
Run snapraid command Run snapraid command
Raises subprocess.CalledProcessError if errorlevel != 0 Raises subprocess.CalledProcessError if errorlevel != 0
""" """
arguments = ["--conf", config["snapraid"]["config"]] arguments = ["--conf", config["snapraid"]["config"],
"--quiet"]
for (k, v) in args.items(): for (k, v) in args.items():
arguments.extend(["--" + k, str(v)]) arguments.append("--" + k)
if v != '':
arguments.append(str(v))
p = subprocess.Popen( p = subprocess.Popen(
[config["snapraid"]["executable"], command] + arguments, [config["snapraid"]["executable"], command] + arguments,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True,
# Snapraid always outputs utf-8 on windows. On linux, utf-8 # Snapraid always outputs utf-8 on windows. On linux, utf-8
# also seems a sensible assumption. # also seems a sensible assumption.
encoding="utf-8", encoding="utf-8",
errors="replace" errors="replace")
)
out = [] out = []
threads = [ threads = [
tee_log(p.stdout, out, logging.OUTPUT), tee_log(p.stdout, out, logging.OUTPUT),
@@ -71,6 +67,21 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]):
else: else:
raise subprocess.CalledProcessError(ret, "snapraid " + command) raise subprocess.CalledProcessError(ret, "snapraid " + command)
def send_gotify(success):
if success:
requests.post("{}/message?token={}".format(config["gotify"]["url"], config["gotify"]["token"]), json={
"message": "Success",
"priority": 2,
"title": "SnapRAID"
})
else:
requests.post("{}/message?token={}".format(config["gotify"]["url"], config["gotify"]["token"]), json={
"message": "Failed",
"priority": 2,
"title": "SnapRAID"
})
def send_email(success): def send_email(success):
import smtplib import smtplib
@@ -129,6 +140,11 @@ def finish(is_success):
send_email(is_success) send_email(is_success)
except Exception: except Exception:
logging.exception("Failed to send email") logging.exception("Failed to send email")
if ("error", "success")[is_success] in config["gotify"]["sendon"]:
try:
send_gotify(is_success)
except Exception:
logging.exception("Failed to send gotify notification")
if is_success: if is_success:
logging.info("Run finished successfully") logging.info("Run finished successfully")
else: else:
@@ -140,7 +156,7 @@ def load_config(args):
global config global config
parser = configparser.RawConfigParser() parser = configparser.RawConfigParser()
parser.read(args.conf) parser.read(args.conf)
sections = ["snapraid", "logging", "email", "smtp", "scrub"] sections = ["snapraid", "logging", "email", "gotify", "smtp", "scrub", "sync"]
config = dict((x, defaultdict(lambda: "")) for x in sections) config = dict((x, defaultdict(lambda: "")) for x in sections)
for section in parser.sections(): for section in parser.sections():
for (k, v) in parser.items(section): for (k, v) in parser.items(section):
@@ -148,7 +164,7 @@ def load_config(args):
int_options = [ int_options = [
("snapraid", "deletethreshold"), ("logging", "maxsize"), ("snapraid", "deletethreshold"), ("logging", "maxsize"),
("scrub", "percentage"), ("scrub", "older-than"), ("email", "maxsize"), ("scrub", "older-than"), ("email", "maxsize"),
] ]
for section, option in int_options: for section, option in int_options:
try: try:
@@ -161,10 +177,18 @@ def load_config(args):
config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true") config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true")
config["email"]["short"] = (config["email"]["short"].lower() == "true") config["email"]["short"] = (config["email"]["short"].lower() == "true")
config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true")
config["sync"]["force-zero"] = (config["sync"]["force-zero"].lower() == "true")
# Migration
if config["scrub"]["percentage"]:
config["scrub"]["plan"] = config["scrub"]["percentage"]
if args.scrub is not None: if args.scrub is not None:
config["scrub"]["enabled"] = args.scrub config["scrub"]["enabled"] = args.scrub
if args.ignore_deletethreshold:
config["snapraid"]["deletethreshold"] = -1
def setup_logger(): def setup_logger():
log_format = logging.Formatter( log_format = logging.Formatter(
@@ -180,7 +204,7 @@ def setup_logger():
root_logger.addHandler(console_logger) root_logger.addHandler(console_logger)
if config["logging"]["file"]: if config["logging"]["file"]:
max_log_size = min(config["logging"]["maxsize"], 0) * 1024 max_log_size = max(config["logging"]["maxsize"], 0) * 1024
file_logger = logging.handlers.RotatingFileHandler( file_logger = logging.handlers.RotatingFileHandler(
config["logging"]["file"], config["logging"]["file"],
maxBytes=max_log_size, maxBytes=max_log_size,
@@ -208,6 +232,8 @@ def main():
parser.add_argument("--no-scrub", action='store_false', parser.add_argument("--no-scrub", action='store_false',
dest='scrub', default=None, dest='scrub', default=None,
help="Do not scrub (overrides config)") help="Do not scrub (overrides config)")
parser.add_argument("--ignore-deletethreshold", action='store_true',
help="Sync even if configured delete threshold is exceeded")
args = parser.parse_args() args = parser.parse_args()
if not os.path.exists(args.conf): if not os.path.exists(args.conf):
@@ -271,6 +297,7 @@ def run():
logging.error( logging.error(
"Deleted files exceed delete threshold of {}, aborting".format( "Deleted files exceed delete threshold of {}, aborting".format(
config["snapraid"]["deletethreshold"])) config["snapraid"]["deletethreshold"]))
logging.error("Run again with --ignore-deletethreshold to sync anyways")
finish(False) finish(False)
if (diff_results["remove"] + diff_results["add"] + diff_results["move"] + if (diff_results["remove"] + diff_results["add"] + diff_results["move"] +
@@ -278,8 +305,11 @@ def run():
logging.info("No changes detected, no sync required") logging.info("No changes detected, no sync required")
else: else:
logging.info("Running sync...") logging.info("Running sync...")
sync_args: dict[str, str] = {}
if config["sync"]["force-zero"]:
sync_args["force-zero"] = ""
try: try:
snapraid_command("sync") snapraid_command("sync", sync_args)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
logging.error(e) logging.error(e)
finish(False) finish(False)
@@ -288,10 +318,17 @@ def run():
if config["scrub"]["enabled"]: if config["scrub"]["enabled"]:
logging.info("Running scrub...") logging.info("Running scrub...")
try: try:
snapraid_command("scrub", { # Check if a percentage plan was given
"percentage": config["scrub"]["percentage"], int(config["scrub"]["plan"])
except ValueError:
scrub_args = {"plan": config["scrub"]["plan"]}
else:
scrub_args = {
"plan": config["scrub"]["plan"],
"older-than": config["scrub"]["older-than"], "older-than": config["scrub"]["older-than"],
}) }
try:
snapraid_command("scrub", scrub_args)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
logging.error(e) logging.error(e)
finish(False) finish(False)