downloads_uploader/downloads_uploader.py

#!/usr/bin/env python3
"""
Downloads Auto-Uploader
=======================
Scans the Downloads folder, fetches the remote file tree via SSH,
and uploads — file by file — only what is missing or has changed.

Config file : downloads_uploader_config.json  (same folder as this script)
State file  : .uploader_state.json            (audit log of uploads)
Log file    : uploader_log.txt
"""

import os
import sys
import json
import subprocess
from datetime import datetime
from pathlib import Path

# ── Safety: never scan these paths ────────────────────────────────────────────
FORBIDDEN_SCAN_ROOTS = {
    "/", "/private", "/private/var",
    "/usr", "/bin", "/sbin", "/etc", "/var",
    "/System", "/Library", "/Applications",
    "/home", "/opt", "/cores",
}

# ── Fixed paths (relative to this script's directory) ─────────────────────────
SCRIPT_DIR   = Path(__file__).parent.resolve()
SCAN_DEFAULT = SCRIPT_DIR.parent          # one level up: Downloads/
CONFIG_FILE  = SCRIPT_DIR / "downloads_uploader_config.json"
STATE_FILE   = SCRIPT_DIR / ".uploader_state.json"
LOG_FILE     = SCRIPT_DIR / "uploader_log.txt"

# ── Defaults ───────────────────────────────────────────────────────────────────
DEFAULT_CONFIG = {
    "remote_destination": "filez@10.10.8.1:/mnt/ssd2t/shared/torrent/Downloads/",
    "ssh_key_path": "",       # empty = use ~/.ssh/id_rsa (system default)
    "ssh_port": 22,
    # Empty = auto-detect (parent of the uploader/ folder = Downloads/).
    # Set an absolute path to watch a different directory.
    "scan_directory": "",
    "skip_patterns": [
        ".*",            # hidden / dot-files
        "*.crdownload",  # incomplete Chrome downloads
        "*.part",        # incomplete Firefox downloads
        "*.tmp",
        "uploader",      # this tool's own folder
        "ORGANIZER_LOG.txt",
        "organizer_log*.txt",
        "__pycache__",
    ],
    "min_file_age_seconds": 30,     # ignore files newer than this (still being written)
    "timeout_seconds": 7200,        # per-file rsync timeout (2 h)
    "remote_scan_timeout": 120,     # SSH find timeout
    "dry_run": False,               # True = log only, no actual transfer
}


# ══════════════════════════════════════════════════════════════════════════════
# Config / State helpers
# ══════════════════════════════════════════════════════════════════════════════

def load_config() -> dict:
    if CONFIG_FILE.exists():
        with open(CONFIG_FILE, "r", encoding="utf-8") as f:
            user_cfg = json.load(f)
        return {**DEFAULT_CONFIG, **user_cfg}
    cfg = DEFAULT_CONFIG.copy()
    save_config(cfg)
    log("INFO", f"Created default config: {CONFIG_FILE}")
    return cfg


def save_config(cfg: dict):
    with open(CONFIG_FILE, "w", encoding="utf-8") as f:
        json.dump(cfg, f, indent=2, ensure_ascii=False)


def load_state() -> dict:
    if STATE_FILE.exists():
        with open(STATE_FILE, "r", encoding="utf-8") as f:
            return json.load(f)
    return {"uploaded": {}}


def save_state(state: dict):
    with open(STATE_FILE, "w", encoding="utf-8") as f:
        json.dump(state, f, indent=2, ensure_ascii=False)


# ══════════════════════════════════════════════════════════════════════════════
# Logging
# ══════════════════════════════════════════════════════════════════════════════

def log(level: str, message: str):
    ts   = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    line = f"[{ts}] {level:7s}  {message}"
    print(line)
    with open(LOG_FILE, "a", encoding="utf-8") as f:
        f.write(line + "\n")


# ══════════════════════════════════════════════════════════════════════════════
# Utilities
# ══════════════════════════════════════════════════════════════════════════════

def matches_skip_pattern(name: str, patterns: list) -> bool:
    import fnmatch
    return any(fnmatch.fnmatch(name, p) for p in patterns)


def _ssh_opts(cfg: dict) -> list[str]:
    """Return ssh CLI option list from config."""
    opts = [
        "-p", str(cfg["ssh_port"]),
        "-o", "StrictHostKeyChecking=accept-new",
        "-o", "BatchMode=yes",           # never prompt for a password
        "-o", "ConnectTimeout=15",
    ]
    if cfg.get("ssh_key_path"):
        opts += ["-i", cfg["ssh_key_path"]]
    return opts


def _parse_remote(destination: str) -> tuple[str, str]:
    """Split 'user@host:/path' into ('user@host', '/path')."""
    host, rpath = destination.split(":", 1)
    return host, rpath.rstrip("/")


def format_bytes(n: int) -> str:
    for unit in ("B", "KB", "MB", "GB"):
        if n < 1024:
            return f"{n:.1f} {unit}"
        n /= 1024
    return f"{n:.1f} TB"


# ══════════════════════════════════════════════════════════════════════════════
# Scan directory validation
# ══════════════════════════════════════════════════════════════════════════════

def resolve_scan_dir(cfg: dict) -> Path:
    raw      = cfg.get("scan_directory", "").strip()
    scan_dir = Path(raw).resolve() if raw else SCAN_DEFAULT.resolve()

    log("INFO", f"Script dir : {SCRIPT_DIR}")
    log("INFO", f"Scan dir   : {scan_dir}")

    if str(scan_dir) in FORBIDDEN_SCAN_ROOTS or scan_dir == Path("/"):
        log("FATAL",
            f"Refusing to scan a system/root path: {scan_dir}\n"
            "  → Set 'scan_directory' in downloads_uploader_config.json\n"
            "  → Or re-run: ./downloads_uploader_install.sh install")
        sys.exit(1)

    if not scan_dir.is_dir():
        log("FATAL", f"scan_directory does not exist: {scan_dir}")
        sys.exit(1)

    return scan_dir


# ══════════════════════════════════════════════════════════════════════════════
# Remote file tree
# ══════════════════════════════════════════════════════════════════════════════

def get_remote_file_tree(cfg: dict) -> dict[str, int]:
    """
    SSH into the remote host, run `find`, and return a dict of
    { relative_path : size_in_bytes } for every file under remote_destination.

    Uses GNU find's -printf '%P\\t%s\\n' (Linux servers have this).
    If the remote directory doesn't exist yet, returns an empty dict.
    On any error, logs a warning and returns {} (safe: everything gets uploaded).
    """
    remote      = cfg["remote_destination"]
    host, rpath = remote.split(":", 1)
    rpath       = rpath.rstrip("/")
    timeout     = cfg.get("remote_scan_timeout", 120)

    # find with -printf (GNU / Linux); || true so SSH exits 0 even if dir missing
    find_cmd = (
        f'find "{rpath}" -type f -printf "%P\\t%s\\n" 2>/dev/null || true'
    )
    cmd = ["ssh"] + _ssh_opts(cfg) + [host, find_cmd]

    log("INFO", f"Fetching remote file tree: {host}:{rpath} ...")

    try:
        result = subprocess.run(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            timeout=timeout,
        )
        stderr = result.stderr.decode("utf-8", errors="replace").strip()
        if result.returncode != 0 and stderr:
            log("WARN", f"SSH stderr: {stderr.splitlines()[0]}")

        remote_files: dict[str, int] = {}
        for line in result.stdout.decode("utf-8", errors="replace").splitlines():
            line = line.strip()
            if not line:
                continue
            parts = line.split("\t", 1)
            if len(parts) == 2:
                rel, size_str = parts
                try:
                    remote_files[rel] = int(size_str)
                except ValueError:
                    remote_files[rel] = -1   # unknown size → will compare by name only
            else:
                remote_files[line] = -1

        log("INFO", f"Remote tree: {len(remote_files):,} file(s) indexed")
        return remote_files

    except subprocess.TimeoutExpired:
        log("WARN",
            f"Timeout ({timeout}s) fetching remote tree. "
            "Treating remote as empty — all local files will be uploaded.")
        return {}
    except Exception as exc:
        log("WARN", f"Could not fetch remote tree: {exc}. Treating remote as empty.")
        return {}


# ══════════════════════════════════════════════════════════════════════════════
# Local ↔ Remote diff
# ══════════════════════════════════════════════════════════════════════════════

def collect_files_to_upload(
    scan_dir: Path,
    remote_files: dict[str, int],
    cfg: dict,
) -> list[tuple[Path, str]]:
    """
    Walk every candidate under scan_dir and compare with remote_files.
    Returns [(local_abs_path, remote_relative_path), ...] for files to upload.
    remote_relative_path uses forward slashes, relative to scan_dir.
    """
    patterns      = cfg["skip_patterns"]
    min_age       = cfg["min_file_age_seconds"]
    now           = datetime.now().timestamp()
    to_upload     = []
    synced_count  = 0
    skipped_count = 0

    for top in sorted(scan_dir.iterdir()):
        if matches_skip_pattern(top.name, patterns):
            continue

        try:
            age = now - top.stat().st_mtime
            if age < min_age:
                log("SKIP", f"{top.name}  (age={age:.0f}s, threshold={min_age}s)")
                skipped_count += 1
                continue
        except OSError:
            continue

        # Enumerate all files: a single file, or every file inside a directory
        if top.is_file():
            local_files = [top]
        elif top.is_dir():
            local_files = sorted(f for f in top.rglob("*") if f.is_file())
        else:
            continue

        for lf in local_files:
            rel         = lf.relative_to(scan_dir).as_posix()   # forward-slash path
            local_size  = lf.stat().st_size
            remote_size = remote_files.get(rel)

            if remote_size is None:
                # File is not on remote at all
                to_upload.append((lf, rel))
            elif remote_size == -1:
                # Remote size unknown (find didn't return it) — upload to be safe
                to_upload.append((lf, rel))
            elif remote_size != local_size:
                log("DIFF",
                    f"{rel}  "
                    f"local={format_bytes(local_size)}  "
                    f"remote={format_bytes(remote_size)}  → will re-upload")
                to_upload.append((lf, rel))
            else:
                synced_count += 1   # identical size → skip

    log("INFO",
        f"Diff complete: "
        f"{len(to_upload)} to upload, "
        f"{synced_count} already in sync, "
        f"{skipped_count} skipped (too recent)")
    return to_upload


# ══════════════════════════════════════════════════════════════════════════════
# Per-file upload
# ══════════════════════════════════════════════════════════════════════════════

def _ensure_remote_dir(remote_dir: str, host: str, cfg: dict,
                        created_dirs: set) -> bool:
    """
    Create *remote_dir* on *host* via SSH mkdir -p (once per session per dir).
    Returns True on success, False on error.
    """
    if remote_dir in created_dirs:
        return True
    cmd = ["ssh"] + _ssh_opts(cfg) + [host, f"mkdir -p '{remote_dir}'"]
    try:
        result = subprocess.run(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE, timeout=30)
        if result.returncode == 0:
            created_dirs.add(remote_dir)
            return True
        err = result.stderr.decode("utf-8", errors="replace").strip()
        log("ERROR", f"mkdir -p failed for {remote_dir}: {err}")
        return False
    except Exception as exc:
        log("ERROR", f"mkdir -p exception: {exc}")
        return False


def upload_file(
    local_path: Path,
    remote_rel: str,
    cfg: dict,
    created_dirs: set,
) -> bool:
    """
    Upload a single file to the exact remote path, preserving the
    relative directory structure.

    Strategy (no --relative, no ./ anchor tricks):
      1. SSH mkdir -p  the remote parent directory (cached per session).
      2. rsync the file into that directory.

    This avoids the macOS rsync 2.6.9 bug where --relative recreates the
    entire absolute local path (/Users/mcer323/Downloads/…) on the remote.
    """
    host, rbase = _parse_remote(cfg["remote_destination"])
    timeout     = cfg.get("timeout_seconds", 7200)
    ssh_args    = " ".join(_ssh_opts(cfg))

    # Compute the remote directory for this file
    parent = Path(remote_rel).parent
    if str(parent) == ".":
        remote_dir = rbase                    # file sits directly in Downloads
    else:
        remote_dir = rbase + "/" + parent.as_posix()

    remote_target = f"{host}:{remote_dir}/"   # trailing / = place file inside dir

    log("UPLOAD", f"[{remote_rel}]  ({format_bytes(local_path.stat().st_size)})")

    # Step 1 — ensure the remote directory exists
    if not cfg.get("dry_run"):
        if not _ensure_remote_dir(remote_dir, host, cfg, created_dirs):
            return False

    # Step 2 — rsync the file (no --relative needed)
    cmd = [
        "rsync",
        "--archive",    # preserve perms / timestamps
        "--compress",   # compress during transfer
        "--partial",    # resume if interrupted
        "-e", f"ssh {ssh_args}",
    ]
    if cfg.get("dry_run"):
        cmd.append("--dry-run")
    cmd += [str(local_path), remote_target]

    log("CMD", " ".join(cmd))

    try:
        result = subprocess.run(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            timeout=timeout,
        )
        stderr = result.stderr.decode("latin-1", errors="replace").strip()

        if result.returncode == 0:
            log("OK", f"✓  {remote_rel}")
            return True
        else:
            log("ERROR", f"✗  {remote_rel}  (rc={result.returncode})")
            if stderr:
                log("ERROR", f"   {stderr.splitlines()[0]}")
            return False

    except subprocess.TimeoutExpired:
        log("ERROR",
            f"Timeout ({timeout}s) uploading: {remote_rel}. "
            "Increase 'timeout_seconds' in config for large files.")
        return False
    except FileNotFoundError:
        log("ERROR", "rsync not found. Install: brew install rsync")
        return False


# ══════════════════════════════════════════════════════════════════════════════
# Main
# ══════════════════════════════════════════════════════════════════════════════

def main():
    log("─" * 64, "")
    log("START", "Downloads uploader scan triggered")

    cfg   = load_config()
    state = load_state()

    if cfg.get("dry_run"):
        log("INFO", "DRY RUN — no files will actually be transferred")

    scan_dir = resolve_scan_dir(cfg)

    # ── 1. Fetch remote file tree ──────────────────────────────────────────────
    remote_files = get_remote_file_tree(cfg)

    # ── 2. Diff local vs remote ────────────────────────────────────────────────
    to_upload = collect_files_to_upload(scan_dir, remote_files, cfg)

    if not to_upload:
        log("DONE", "Everything is already in sync. Nothing to upload.")
        return

    log("INFO",
        f"Starting upload of {len(to_upload)} file(s) "
        f"({format_bytes(sum(p.stat().st_size for p, _ in to_upload))} total)")

    # ── 3. Upload file by file ─────────────────────────────────────────────────
    uploaded     = 0
    failed       = 0
    created_dirs: set[str] = set()   # remote dirs created this session (cache)

    for idx, (local_path, remote_rel) in enumerate(to_upload, start=1):
        log("INFO", f"── [{idx}/{len(to_upload)}] {remote_rel}")

        success = upload_file(local_path, remote_rel, cfg, created_dirs)

        if success:
            uploaded += 1
            state["uploaded"][remote_rel] = {
                "size":        local_path.stat().st_size,
                "mtime":       local_path.stat().st_mtime,
                "uploaded_at": datetime.now().isoformat(timespec="seconds"),
            }
            save_state(state)   # persist after every successful file
        else:
            failed += 1

    log("DONE",
        f"Session complete — "
        f"uploaded: {uploaded}, failed: {failed}, total: {len(to_upload)}")


if __name__ == "__main__":
    main()