Add uploader scripts and docs
This commit is contained in:
Maxx Cherevko
Executable
+463
@@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Downloads Auto-Uploader
|
||||
=======================
|
||||
Scans the Downloads folder, fetches the remote file tree via SSH,
|
||||
and uploads — file by file — only what is missing or has changed.
|
||||
|
||||
Config file : downloads_uploader_config.json (same folder as this script)
|
||||
State file : .uploader_state.json (audit log of uploads)
|
||||
Log file : uploader_log.txt
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# ── Safety: never scan these paths ────────────────────────────────────────────
|
||||
FORBIDDEN_SCAN_ROOTS = {
|
||||
"/", "/private", "/private/var",
|
||||
"/usr", "/bin", "/sbin", "/etc", "/var",
|
||||
"/System", "/Library", "/Applications",
|
||||
"/home", "/opt", "/cores",
|
||||
}
|
||||
|
||||
# ── Fixed paths (relative to this script's directory) ─────────────────────────
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
SCAN_DEFAULT = SCRIPT_DIR.parent # one level up: Downloads/
|
||||
CONFIG_FILE = SCRIPT_DIR / "downloads_uploader_config.json"
|
||||
STATE_FILE = SCRIPT_DIR / ".uploader_state.json"
|
||||
LOG_FILE = SCRIPT_DIR / "uploader_log.txt"
|
||||
|
||||
# ── Defaults ───────────────────────────────────────────────────────────────────
|
||||
DEFAULT_CONFIG = {
|
||||
"remote_destination": "filez@10.10.8.1:/mnt/ssd2t/shared/torrent/Downloads/",
|
||||
"ssh_key_path": "", # empty = use ~/.ssh/id_rsa (system default)
|
||||
"ssh_port": 22,
|
||||
# Empty = auto-detect (parent of the uploader/ folder = Downloads/).
|
||||
# Set an absolute path to watch a different directory.
|
||||
"scan_directory": "",
|
||||
"skip_patterns": [
|
||||
".*", # hidden / dot-files
|
||||
"*.crdownload", # incomplete Chrome downloads
|
||||
"*.part", # incomplete Firefox downloads
|
||||
"*.tmp",
|
||||
"uploader", # this tool's own folder
|
||||
"ORGANIZER_LOG.txt",
|
||||
"organizer_log*.txt",
|
||||
"__pycache__",
|
||||
],
|
||||
"min_file_age_seconds": 30, # ignore files newer than this (still being written)
|
||||
"timeout_seconds": 7200, # per-file rsync timeout (2 h)
|
||||
"remote_scan_timeout": 120, # SSH find timeout
|
||||
"dry_run": False, # True = log only, no actual transfer
|
||||
}
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Config / State helpers
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def load_config() -> dict:
|
||||
if CONFIG_FILE.exists():
|
||||
with open(CONFIG_FILE, "r", encoding="utf-8") as f:
|
||||
user_cfg = json.load(f)
|
||||
return {**DEFAULT_CONFIG, **user_cfg}
|
||||
cfg = DEFAULT_CONFIG.copy()
|
||||
save_config(cfg)
|
||||
log("INFO", f"Created default config: {CONFIG_FILE}")
|
||||
return cfg
|
||||
|
||||
|
||||
def save_config(cfg: dict):
|
||||
with open(CONFIG_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def load_state() -> dict:
|
||||
if STATE_FILE.exists():
|
||||
with open(STATE_FILE, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
return {"uploaded": {}}
|
||||
|
||||
|
||||
def save_state(state: dict):
|
||||
with open(STATE_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Logging
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def log(level: str, message: str):
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"[{ts}] {level:7s} {message}"
|
||||
print(line)
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Utilities
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def matches_skip_pattern(name: str, patterns: list) -> bool:
|
||||
import fnmatch
|
||||
return any(fnmatch.fnmatch(name, p) for p in patterns)
|
||||
|
||||
|
||||
def _ssh_opts(cfg: dict) -> list[str]:
|
||||
"""Return ssh CLI option list from config."""
|
||||
opts = [
|
||||
"-p", str(cfg["ssh_port"]),
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "BatchMode=yes", # never prompt for a password
|
||||
"-o", "ConnectTimeout=15",
|
||||
]
|
||||
if cfg.get("ssh_key_path"):
|
||||
opts += ["-i", cfg["ssh_key_path"]]
|
||||
return opts
|
||||
|
||||
|
||||
def _parse_remote(destination: str) -> tuple[str, str]:
|
||||
"""Split 'user@host:/path' into ('user@host', '/path')."""
|
||||
host, rpath = destination.split(":", 1)
|
||||
return host, rpath.rstrip("/")
|
||||
|
||||
|
||||
def format_bytes(n: int) -> str:
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if n < 1024:
|
||||
return f"{n:.1f} {unit}"
|
||||
n /= 1024
|
||||
return f"{n:.1f} TB"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Scan directory validation
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def resolve_scan_dir(cfg: dict) -> Path:
|
||||
raw = cfg.get("scan_directory", "").strip()
|
||||
scan_dir = Path(raw).resolve() if raw else SCAN_DEFAULT.resolve()
|
||||
|
||||
log("INFO", f"Script dir : {SCRIPT_DIR}")
|
||||
log("INFO", f"Scan dir : {scan_dir}")
|
||||
|
||||
if str(scan_dir) in FORBIDDEN_SCAN_ROOTS or scan_dir == Path("/"):
|
||||
log("FATAL",
|
||||
f"Refusing to scan a system/root path: {scan_dir}\n"
|
||||
" → Set 'scan_directory' in downloads_uploader_config.json\n"
|
||||
" → Or re-run: ./downloads_uploader_install.sh install")
|
||||
sys.exit(1)
|
||||
|
||||
if not scan_dir.is_dir():
|
||||
log("FATAL", f"scan_directory does not exist: {scan_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
return scan_dir
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Remote file tree
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def get_remote_file_tree(cfg: dict) -> dict[str, int]:
|
||||
"""
|
||||
SSH into the remote host, run `find`, and return a dict of
|
||||
{ relative_path : size_in_bytes } for every file under remote_destination.
|
||||
|
||||
Uses GNU find's -printf '%P\\t%s\\n' (Linux servers have this).
|
||||
If the remote directory doesn't exist yet, returns an empty dict.
|
||||
On any error, logs a warning and returns {} (safe: everything gets uploaded).
|
||||
"""
|
||||
remote = cfg["remote_destination"]
|
||||
host, rpath = remote.split(":", 1)
|
||||
rpath = rpath.rstrip("/")
|
||||
timeout = cfg.get("remote_scan_timeout", 120)
|
||||
|
||||
# find with -printf (GNU / Linux); || true so SSH exits 0 even if dir missing
|
||||
find_cmd = (
|
||||
f'find "{rpath}" -type f -printf "%P\\t%s\\n" 2>/dev/null || true'
|
||||
)
|
||||
cmd = ["ssh"] + _ssh_opts(cfg) + [host, find_cmd]
|
||||
|
||||
log("INFO", f"Fetching remote file tree: {host}:{rpath} ...")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
stderr = result.stderr.decode("utf-8", errors="replace").strip()
|
||||
if result.returncode != 0 and stderr:
|
||||
log("WARN", f"SSH stderr: {stderr.splitlines()[0]}")
|
||||
|
||||
remote_files: dict[str, int] = {}
|
||||
for line in result.stdout.decode("utf-8", errors="replace").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("\t", 1)
|
||||
if len(parts) == 2:
|
||||
rel, size_str = parts
|
||||
try:
|
||||
remote_files[rel] = int(size_str)
|
||||
except ValueError:
|
||||
remote_files[rel] = -1 # unknown size → will compare by name only
|
||||
else:
|
||||
remote_files[line] = -1
|
||||
|
||||
log("INFO", f"Remote tree: {len(remote_files):,} file(s) indexed")
|
||||
return remote_files
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
log("WARN",
|
||||
f"Timeout ({timeout}s) fetching remote tree. "
|
||||
"Treating remote as empty — all local files will be uploaded.")
|
||||
return {}
|
||||
except Exception as exc:
|
||||
log("WARN", f"Could not fetch remote tree: {exc}. Treating remote as empty.")
|
||||
return {}
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Local ↔ Remote diff
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def collect_files_to_upload(
|
||||
scan_dir: Path,
|
||||
remote_files: dict[str, int],
|
||||
cfg: dict,
|
||||
) -> list[tuple[Path, str]]:
|
||||
"""
|
||||
Walk every candidate under scan_dir and compare with remote_files.
|
||||
Returns [(local_abs_path, remote_relative_path), ...] for files to upload.
|
||||
remote_relative_path uses forward slashes, relative to scan_dir.
|
||||
"""
|
||||
patterns = cfg["skip_patterns"]
|
||||
min_age = cfg["min_file_age_seconds"]
|
||||
now = datetime.now().timestamp()
|
||||
to_upload = []
|
||||
synced_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
for top in sorted(scan_dir.iterdir()):
|
||||
if matches_skip_pattern(top.name, patterns):
|
||||
continue
|
||||
|
||||
try:
|
||||
age = now - top.stat().st_mtime
|
||||
if age < min_age:
|
||||
log("SKIP", f"{top.name} (age={age:.0f}s, threshold={min_age}s)")
|
||||
skipped_count += 1
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# Enumerate all files: a single file, or every file inside a directory
|
||||
if top.is_file():
|
||||
local_files = [top]
|
||||
elif top.is_dir():
|
||||
local_files = sorted(f for f in top.rglob("*") if f.is_file())
|
||||
else:
|
||||
continue
|
||||
|
||||
for lf in local_files:
|
||||
rel = lf.relative_to(scan_dir).as_posix() # forward-slash path
|
||||
local_size = lf.stat().st_size
|
||||
remote_size = remote_files.get(rel)
|
||||
|
||||
if remote_size is None:
|
||||
# File is not on remote at all
|
||||
to_upload.append((lf, rel))
|
||||
elif remote_size == -1:
|
||||
# Remote size unknown (find didn't return it) — upload to be safe
|
||||
to_upload.append((lf, rel))
|
||||
elif remote_size != local_size:
|
||||
log("DIFF",
|
||||
f"{rel} "
|
||||
f"local={format_bytes(local_size)} "
|
||||
f"remote={format_bytes(remote_size)} → will re-upload")
|
||||
to_upload.append((lf, rel))
|
||||
else:
|
||||
synced_count += 1 # identical size → skip
|
||||
|
||||
log("INFO",
|
||||
f"Diff complete: "
|
||||
f"{len(to_upload)} to upload, "
|
||||
f"{synced_count} already in sync, "
|
||||
f"{skipped_count} skipped (too recent)")
|
||||
return to_upload
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Per-file upload
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _ensure_remote_dir(remote_dir: str, host: str, cfg: dict,
|
||||
created_dirs: set) -> bool:
|
||||
"""
|
||||
Create *remote_dir* on *host* via SSH mkdir -p (once per session per dir).
|
||||
Returns True on success, False on error.
|
||||
"""
|
||||
if remote_dir in created_dirs:
|
||||
return True
|
||||
cmd = ["ssh"] + _ssh_opts(cfg) + [host, f"mkdir -p '{remote_dir}'"]
|
||||
try:
|
||||
result = subprocess.run(cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, timeout=30)
|
||||
if result.returncode == 0:
|
||||
created_dirs.add(remote_dir)
|
||||
return True
|
||||
err = result.stderr.decode("utf-8", errors="replace").strip()
|
||||
log("ERROR", f"mkdir -p failed for {remote_dir}: {err}")
|
||||
return False
|
||||
except Exception as exc:
|
||||
log("ERROR", f"mkdir -p exception: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def upload_file(
|
||||
local_path: Path,
|
||||
remote_rel: str,
|
||||
cfg: dict,
|
||||
created_dirs: set,
|
||||
) -> bool:
|
||||
"""
|
||||
Upload a single file to the exact remote path, preserving the
|
||||
relative directory structure.
|
||||
|
||||
Strategy (no --relative, no ./ anchor tricks):
|
||||
1. SSH mkdir -p the remote parent directory (cached per session).
|
||||
2. rsync the file into that directory.
|
||||
|
||||
This avoids the macOS rsync 2.6.9 bug where --relative recreates the
|
||||
entire absolute local path (/Users/mcer323/Downloads/…) on the remote.
|
||||
"""
|
||||
host, rbase = _parse_remote(cfg["remote_destination"])
|
||||
timeout = cfg.get("timeout_seconds", 7200)
|
||||
ssh_args = " ".join(_ssh_opts(cfg))
|
||||
|
||||
# Compute the remote directory for this file
|
||||
parent = Path(remote_rel).parent
|
||||
if str(parent) == ".":
|
||||
remote_dir = rbase # file sits directly in Downloads
|
||||
else:
|
||||
remote_dir = rbase + "/" + parent.as_posix()
|
||||
|
||||
remote_target = f"{host}:{remote_dir}/" # trailing / = place file inside dir
|
||||
|
||||
log("UPLOAD", f"[{remote_rel}] ({format_bytes(local_path.stat().st_size)})")
|
||||
|
||||
# Step 1 — ensure the remote directory exists
|
||||
if not cfg.get("dry_run"):
|
||||
if not _ensure_remote_dir(remote_dir, host, cfg, created_dirs):
|
||||
return False
|
||||
|
||||
# Step 2 — rsync the file (no --relative needed)
|
||||
cmd = [
|
||||
"rsync",
|
||||
"--archive", # preserve perms / timestamps
|
||||
"--compress", # compress during transfer
|
||||
"--partial", # resume if interrupted
|
||||
"-e", f"ssh {ssh_args}",
|
||||
]
|
||||
if cfg.get("dry_run"):
|
||||
cmd.append("--dry-run")
|
||||
cmd += [str(local_path), remote_target]
|
||||
|
||||
log("CMD", " ".join(cmd))
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
stderr = result.stderr.decode("latin-1", errors="replace").strip()
|
||||
|
||||
if result.returncode == 0:
|
||||
log("OK", f"✓ {remote_rel}")
|
||||
return True
|
||||
else:
|
||||
log("ERROR", f"✗ {remote_rel} (rc={result.returncode})")
|
||||
if stderr:
|
||||
log("ERROR", f" {stderr.splitlines()[0]}")
|
||||
return False
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
log("ERROR",
|
||||
f"Timeout ({timeout}s) uploading: {remote_rel}. "
|
||||
"Increase 'timeout_seconds' in config for large files.")
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
log("ERROR", "rsync not found. Install: brew install rsync")
|
||||
return False
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# Main
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def main():
|
||||
log("─" * 64, "")
|
||||
log("START", "Downloads uploader scan triggered")
|
||||
|
||||
cfg = load_config()
|
||||
state = load_state()
|
||||
|
||||
if cfg.get("dry_run"):
|
||||
log("INFO", "DRY RUN — no files will actually be transferred")
|
||||
|
||||
scan_dir = resolve_scan_dir(cfg)
|
||||
|
||||
# ── 1. Fetch remote file tree ──────────────────────────────────────────────
|
||||
remote_files = get_remote_file_tree(cfg)
|
||||
|
||||
# ── 2. Diff local vs remote ────────────────────────────────────────────────
|
||||
to_upload = collect_files_to_upload(scan_dir, remote_files, cfg)
|
||||
|
||||
if not to_upload:
|
||||
log("DONE", "Everything is already in sync. Nothing to upload.")
|
||||
return
|
||||
|
||||
log("INFO",
|
||||
f"Starting upload of {len(to_upload)} file(s) "
|
||||
f"({format_bytes(sum(p.stat().st_size for p, _ in to_upload))} total)")
|
||||
|
||||
# ── 3. Upload file by file ─────────────────────────────────────────────────
|
||||
uploaded = 0
|
||||
failed = 0
|
||||
created_dirs: set[str] = set() # remote dirs created this session (cache)
|
||||
|
||||
for idx, (local_path, remote_rel) in enumerate(to_upload, start=1):
|
||||
log("INFO", f"── [{idx}/{len(to_upload)}] {remote_rel}")
|
||||
|
||||
success = upload_file(local_path, remote_rel, cfg, created_dirs)
|
||||
|
||||
if success:
|
||||
uploaded += 1
|
||||
state["uploaded"][remote_rel] = {
|
||||
"size": local_path.stat().st_size,
|
||||
"mtime": local_path.stat().st_mtime,
|
||||
"uploaded_at": datetime.now().isoformat(timespec="seconds"),
|
||||
}
|
||||
save_state(state) # persist after every successful file
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
log("DONE",
|
||||
f"Session complete — "
|
||||
f"uploaded: {uploaded}, failed: {failed}, total: {len(to_upload)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user