Compare commits
10
Commits
76af6f739d
...
fb11ad022e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fb11ad022e | ||
|
|
df764ca47c | ||
|
|
60b8b6731c | ||
|
|
8ce2458e5e | ||
|
|
5524a86f7c | ||
|
|
8acd9315dd | ||
|
|
4cb7e695af | ||
|
|
11affad8b7 | ||
|
|
6f8d170edc | ||
|
|
68a03ced91 |
No files matched your search
@@ -26,14 +26,31 @@ It supports Windows, Linux and macOS and requires at least python3.7.
|
|||||||
* Can send notification emails after each run or only for failures.
|
* Can send notification emails after each run or only for failures.
|
||||||
* Can run `scrub` after `sync`
|
* Can run `scrub` after `sync`
|
||||||
|
|
||||||
|
## Scope of this project and contributions
|
||||||
|
Snapraid-runner is supposed to be a small tool with clear focus. It should not
|
||||||
|
have any dependencies to keep installation trivial. I always welcome bugfixes
|
||||||
|
and contributions, but be aware that I will not merge new features that I feel
|
||||||
|
do not fit the core purpose of this tool.
|
||||||
|
|
||||||
|
I keep the PRs for features I do not plan on merging open, so if there's a
|
||||||
|
feature you are missing, you can have a look
|
||||||
|
[at the open PRs](https://github.com/Chronial/snapraid-runner/pulls).
|
||||||
|
|
||||||
## Changelog
|
## Changelog
|
||||||
### Unreleased
|
### Unreleased
|
||||||
|
* Add --ignore-deletethreshold (by exterrestris, #25)
|
||||||
|
* Add support for scrub --plan, replacing --percentage (thanks to fmoledina)
|
||||||
|
* Remove snapraid progress output. Was accidentially introduced with python3
|
||||||
|
support.
|
||||||
|
|
||||||
|
### v0.5 (26 Feb 2021)
|
||||||
* Remove (broken) python2 support
|
* Remove (broken) python2 support
|
||||||
* Fix snapraid output encoding handling (by hyyz17200, PR-31)
|
* Fix snapraid output encoding handling (by hyyz17200, #31)
|
||||||
|
* Fix log rotation (by ptoulouse, #36)
|
||||||
|
|
||||||
### v0.4 (17 Aug 2019)
|
### v0.4 (17 Aug 2019)
|
||||||
* Add compatibility with python3 (by reed-jones)
|
* Add compatibility with python3 (by reed-jones)
|
||||||
* Add support for running `snapraid touch` (by ShoGinn, PR-11)
|
* Add support for running `snapraid touch` (by ShoGinn, #11)
|
||||||
* Add SMTP TLS support
|
* Add SMTP TLS support
|
||||||
|
|
||||||
### v0.3 (20 Jul 2017)
|
### v0.3 (20 Jul 2017)
|
||||||
|
|||||||
@@ -25,6 +25,15 @@ to =
|
|||||||
; maximum email size in KiB
|
; maximum email size in KiB
|
||||||
maxsize = 500
|
maxsize = 500
|
||||||
|
|
||||||
|
[gotify]
|
||||||
|
; when to send a gotify notification, comma-separated list of [success, error]
|
||||||
|
sendon = success,error
|
||||||
|
; url of the gotify server without trailing slash
|
||||||
|
url = https://gotify.server.com
|
||||||
|
; Gotify token
|
||||||
|
token =
|
||||||
|
|
||||||
|
|
||||||
[smtp]
|
[smtp]
|
||||||
host =
|
host =
|
||||||
; leave empty for default port
|
; leave empty for default port
|
||||||
@@ -38,5 +47,11 @@ password =
|
|||||||
[scrub]
|
[scrub]
|
||||||
; set to true to run scrub after sync
|
; set to true to run scrub after sync
|
||||||
enabled = false
|
enabled = false
|
||||||
percentage = 12
|
; scrub plan - either a percentage or one of [bad, new, full]
|
||||||
|
plan = 12
|
||||||
|
; minimum block age (in days) for scrubbing. Only used with percentage plans
|
||||||
older-than = 10
|
older-than = 10
|
||||||
|
|
||||||
|
[sync]
|
||||||
|
; set to true to force sync files with zero size
|
||||||
|
force-zero = false
|
||||||
Regular → Executable
+56
-19
@@ -1,6 +1,4 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
from __future__ import division
|
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import configparser
|
import configparser
|
||||||
import logging
|
import logging
|
||||||
@@ -11,6 +9,7 @@ import sys
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
|
import requests
|
||||||
from collections import Counter, defaultdict
|
from collections import Counter, defaultdict
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
|
|
||||||
@@ -26,11 +25,7 @@ def tee_log(infile, out_lines, log_level):
|
|||||||
"""
|
"""
|
||||||
def tee_thread():
|
def tee_thread():
|
||||||
for line in iter(infile.readline, ""):
|
for line in iter(infile.readline, ""):
|
||||||
line = line.strip()
|
logging.log(log_level, line.rstrip())
|
||||||
# Do not log the progress display
|
|
||||||
if "\r" in line:
|
|
||||||
line = line.split("\r")[-1]
|
|
||||||
logging.log(log_level, line.strip())
|
|
||||||
out_lines.append(line)
|
out_lines.append(line)
|
||||||
infile.close()
|
infile.close()
|
||||||
t = threading.Thread(target=tee_thread)
|
t = threading.Thread(target=tee_thread)
|
||||||
@@ -44,19 +39,20 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]):
|
|||||||
Run snapraid command
|
Run snapraid command
|
||||||
Raises subprocess.CalledProcessError if errorlevel != 0
|
Raises subprocess.CalledProcessError if errorlevel != 0
|
||||||
"""
|
"""
|
||||||
arguments = ["--conf", config["snapraid"]["config"]]
|
arguments = ["--conf", config["snapraid"]["config"],
|
||||||
|
"--quiet"]
|
||||||
for (k, v) in args.items():
|
for (k, v) in args.items():
|
||||||
arguments.extend(["--" + k, str(v)])
|
arguments.append("--" + k)
|
||||||
|
if v != '':
|
||||||
|
arguments.append(str(v))
|
||||||
p = subprocess.Popen(
|
p = subprocess.Popen(
|
||||||
[config["snapraid"]["executable"], command] + arguments,
|
[config["snapraid"]["executable"], command] + arguments,
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE,
|
stderr=subprocess.PIPE,
|
||||||
universal_newlines=True,
|
|
||||||
# Snapraid always outputs utf-8 on windows. On linux, utf-8
|
# Snapraid always outputs utf-8 on windows. On linux, utf-8
|
||||||
# also seems a sensible assumption.
|
# also seems a sensible assumption.
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
errors="replace"
|
errors="replace")
|
||||||
)
|
|
||||||
out = []
|
out = []
|
||||||
threads = [
|
threads = [
|
||||||
tee_log(p.stdout, out, logging.OUTPUT),
|
tee_log(p.stdout, out, logging.OUTPUT),
|
||||||
@@ -71,6 +67,21 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]):
|
|||||||
else:
|
else:
|
||||||
raise subprocess.CalledProcessError(ret, "snapraid " + command)
|
raise subprocess.CalledProcessError(ret, "snapraid " + command)
|
||||||
|
|
||||||
|
def send_gotify(success):
|
||||||
|
|
||||||
|
if success:
|
||||||
|
requests.post("{}/message?token={}".format(config["gotify"]["url"], config["gotify"]["token"]), json={
|
||||||
|
"message": "Success",
|
||||||
|
"priority": 2,
|
||||||
|
"title": "SnapRAID"
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
requests.post("{}/message?token={}".format(config["gotify"]["url"], config["gotify"]["token"]), json={
|
||||||
|
"message": "Failed",
|
||||||
|
"priority": 2,
|
||||||
|
"title": "SnapRAID"
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
def send_email(success):
|
def send_email(success):
|
||||||
import smtplib
|
import smtplib
|
||||||
@@ -129,6 +140,11 @@ def finish(is_success):
|
|||||||
send_email(is_success)
|
send_email(is_success)
|
||||||
except Exception:
|
except Exception:
|
||||||
logging.exception("Failed to send email")
|
logging.exception("Failed to send email")
|
||||||
|
if ("error", "success")[is_success] in config["gotify"]["sendon"]:
|
||||||
|
try:
|
||||||
|
send_gotify(is_success)
|
||||||
|
except Exception:
|
||||||
|
logging.exception("Failed to send gotify notification")
|
||||||
if is_success:
|
if is_success:
|
||||||
logging.info("Run finished successfully")
|
logging.info("Run finished successfully")
|
||||||
else:
|
else:
|
||||||
@@ -140,7 +156,7 @@ def load_config(args):
|
|||||||
global config
|
global config
|
||||||
parser = configparser.RawConfigParser()
|
parser = configparser.RawConfigParser()
|
||||||
parser.read(args.conf)
|
parser.read(args.conf)
|
||||||
sections = ["snapraid", "logging", "email", "smtp", "scrub"]
|
sections = ["snapraid", "logging", "email", "gotify", "smtp", "scrub", "sync"]
|
||||||
config = dict((x, defaultdict(lambda: "")) for x in sections)
|
config = dict((x, defaultdict(lambda: "")) for x in sections)
|
||||||
for section in parser.sections():
|
for section in parser.sections():
|
||||||
for (k, v) in parser.items(section):
|
for (k, v) in parser.items(section):
|
||||||
@@ -148,7 +164,7 @@ def load_config(args):
|
|||||||
|
|
||||||
int_options = [
|
int_options = [
|
||||||
("snapraid", "deletethreshold"), ("logging", "maxsize"),
|
("snapraid", "deletethreshold"), ("logging", "maxsize"),
|
||||||
("scrub", "percentage"), ("scrub", "older-than"), ("email", "maxsize"),
|
("scrub", "older-than"), ("email", "maxsize"),
|
||||||
]
|
]
|
||||||
for section, option in int_options:
|
for section, option in int_options:
|
||||||
try:
|
try:
|
||||||
@@ -161,10 +177,18 @@ def load_config(args):
|
|||||||
config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true")
|
config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true")
|
||||||
config["email"]["short"] = (config["email"]["short"].lower() == "true")
|
config["email"]["short"] = (config["email"]["short"].lower() == "true")
|
||||||
config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true")
|
config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true")
|
||||||
|
config["sync"]["force-zero"] = (config["sync"]["force-zero"].lower() == "true")
|
||||||
|
|
||||||
|
# Migration
|
||||||
|
if config["scrub"]["percentage"]:
|
||||||
|
config["scrub"]["plan"] = config["scrub"]["percentage"]
|
||||||
|
|
||||||
if args.scrub is not None:
|
if args.scrub is not None:
|
||||||
config["scrub"]["enabled"] = args.scrub
|
config["scrub"]["enabled"] = args.scrub
|
||||||
|
|
||||||
|
if args.ignore_deletethreshold:
|
||||||
|
config["snapraid"]["deletethreshold"] = -1
|
||||||
|
|
||||||
|
|
||||||
def setup_logger():
|
def setup_logger():
|
||||||
log_format = logging.Formatter(
|
log_format = logging.Formatter(
|
||||||
@@ -180,7 +204,7 @@ def setup_logger():
|
|||||||
root_logger.addHandler(console_logger)
|
root_logger.addHandler(console_logger)
|
||||||
|
|
||||||
if config["logging"]["file"]:
|
if config["logging"]["file"]:
|
||||||
max_log_size = min(config["logging"]["maxsize"], 0) * 1024
|
max_log_size = max(config["logging"]["maxsize"], 0) * 1024
|
||||||
file_logger = logging.handlers.RotatingFileHandler(
|
file_logger = logging.handlers.RotatingFileHandler(
|
||||||
config["logging"]["file"],
|
config["logging"]["file"],
|
||||||
maxBytes=max_log_size,
|
maxBytes=max_log_size,
|
||||||
@@ -208,6 +232,8 @@ def main():
|
|||||||
parser.add_argument("--no-scrub", action='store_false',
|
parser.add_argument("--no-scrub", action='store_false',
|
||||||
dest='scrub', default=None,
|
dest='scrub', default=None,
|
||||||
help="Do not scrub (overrides config)")
|
help="Do not scrub (overrides config)")
|
||||||
|
parser.add_argument("--ignore-deletethreshold", action='store_true',
|
||||||
|
help="Sync even if configured delete threshold is exceeded")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if not os.path.exists(args.conf):
|
if not os.path.exists(args.conf):
|
||||||
@@ -271,6 +297,7 @@ def run():
|
|||||||
logging.error(
|
logging.error(
|
||||||
"Deleted files exceed delete threshold of {}, aborting".format(
|
"Deleted files exceed delete threshold of {}, aborting".format(
|
||||||
config["snapraid"]["deletethreshold"]))
|
config["snapraid"]["deletethreshold"]))
|
||||||
|
logging.error("Run again with --ignore-deletethreshold to sync anyways")
|
||||||
finish(False)
|
finish(False)
|
||||||
|
|
||||||
if (diff_results["remove"] + diff_results["add"] + diff_results["move"] +
|
if (diff_results["remove"] + diff_results["add"] + diff_results["move"] +
|
||||||
@@ -278,8 +305,11 @@ def run():
|
|||||||
logging.info("No changes detected, no sync required")
|
logging.info("No changes detected, no sync required")
|
||||||
else:
|
else:
|
||||||
logging.info("Running sync...")
|
logging.info("Running sync...")
|
||||||
|
sync_args: dict[str, str] = {}
|
||||||
|
if config["sync"]["force-zero"]:
|
||||||
|
sync_args["force-zero"] = ""
|
||||||
try:
|
try:
|
||||||
snapraid_command("sync")
|
snapraid_command("sync", sync_args)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
logging.error(e)
|
logging.error(e)
|
||||||
finish(False)
|
finish(False)
|
||||||
@@ -288,10 +318,17 @@ def run():
|
|||||||
if config["scrub"]["enabled"]:
|
if config["scrub"]["enabled"]:
|
||||||
logging.info("Running scrub...")
|
logging.info("Running scrub...")
|
||||||
try:
|
try:
|
||||||
snapraid_command("scrub", {
|
# Check if a percentage plan was given
|
||||||
"percentage": config["scrub"]["percentage"],
|
int(config["scrub"]["plan"])
|
||||||
|
except ValueError:
|
||||||
|
scrub_args = {"plan": config["scrub"]["plan"]}
|
||||||
|
else:
|
||||||
|
scrub_args = {
|
||||||
|
"plan": config["scrub"]["plan"],
|
||||||
"older-than": config["scrub"]["older-than"],
|
"older-than": config["scrub"]["older-than"],
|
||||||
})
|
}
|
||||||
|
try:
|
||||||
|
snapraid_command("scrub", scrub_args)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
logging.error(e)
|
logging.error(e)
|
||||||
finish(False)
|
finish(False)
|
||||||
|
|||||||
Reference in new issue
Block a user