#!/usr/bin/env bash
#
# kimi k3 spent 60k tokens writing this
# date: 2026-10-31
#
# runs well, continuation support is implemented well, suited for running
# in a supercomputer environment
#
# ============================================================================
#  script.sh - incremental 60 fps AV1 timelapse builder from archive-*.zip
# ============================================================================
#
#  For every input archive (processed ONE at a time, in sorted order):
#    1. extract it into TMPDIR
#    2. hash every .jpg with md5sum, drop duplicates (within the archive,
#       across archives, and across script restarts - the hash DB lives
#       permanently in TMPDIR2)
#    3. encode the new unique frames (sorted by date from the filename)
#       ONCE with libsvtav1 into a small per-archive segment file
#    4. update the final mp4 by CONCATENATING all segments with stream copy
#       (=> the video is updated WITHOUT re-encoding old frames)
#    5. append the .html files to one big html archive (zip) in TMPDIR2
#    6. delete the extracted jpgs and htmls from TMPDIR
#
#  Results kept in TMPDIR2:
#    timelapse.mp4           - the growing 60 fps AV1 timelapse
#    html-archive.zip        - the big html archive
#    progress.log            - human readable log with timestamps (mirrors stdout)
#    processed-archives.txt  - names of finished archives (resume support)
#    seen-hashes.txt         - md5 hash DB of every kept picture
#    per-archive-stats.tsv   - per-archive picture counters
#    ffmpeg.log              - raw ffmpeg output (kept out of the human log)
#    segments/               - per-archive AV1 segments (the encoded source
#                              of timelapse.mp4; do NOT delete, needed to
#                              update the video without re-encoding)
#
#  Resume: just re-run the script. Archives listed in processed-archives.txt
#  are skipped. Counters and the hash DB are rebuilt/healed from the files in
#  TMPDIR2, so an interruption at (almost) any point is safe.
#
#  The script uses no paths other than the CONFIGURATION block below and can
#  be run from any working directory.
#
# ============================================================================

set -uo pipefail

# ============================ CONFIGURATION ================================
SRC_DIR="/mnt/mystorage/meteo-padochov"   # directory containing the input archives
ARCHIVE_PATTERN="archive-*.zip"           # glob for the input archives in SRC_DIR

TMPDIR="/mnt/storage/timelapse-work"      # scratch space: extraction + frame staging
                                          # (contents deleted after every archive)
TMPDIR2="/mnt/storage/timelapse-out"      # output: mp4, html zip, logs, segments

FPS=60            # timelapse frame rate
SVT_PRESET=8      # libsvtav1 preset: 0 = slow/best ... 13 = fastest (7-9 is a good spot)
SVT_CRF=32        # libsvtav1 quality: lower = better/bigger (30-35 sane for webcams)
# ===========================================================================

# ---- derived paths (no need to touch) ----
VIDEO="$TMPDIR2/timelapse.mp4"
HTML_ZIP="$TMPDIR2/html-archive.zip"
SEGDIR="$TMPDIR2/segments"
LOG_FILE="$TMPDIR2/progress.log"
PROCESSED_FILE="$TMPDIR2/processed-archives.txt"
HASH_FILE="$TMPDIR2/seen-hashes.txt"
STATS_FILE="$TMPDIR2/per-archive-stats.tsv"
FFMPEG_LOG="$TMPDIR2/ffmpeg.log"

WE="$TMPDIR/extract"      # extraction area
WF="$TMPDIR/frames"       # staged frames of the current archive

# ---------------------------------------------------------------------------
log() {  # timestamped message to stdout AND the human log file
    local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
    printf '%s\n' "$msg" | tee -a "$LOG_FILE"
}

die() { log "FATAL: $*"; exit 1; }

on_interrupt() {
    log "INTERRUPTED - partial state is safe. Just re-run the script to resume."
    exit 130
}
trap on_interrupt INT TERM

# ---------------------------------------------------------------------------
# Rebuild timelapse.mp4 from all per-archive segments (stream copy = no re-encode)
rebuild_video() {
    local list="$SEGDIR/.concat_list.txt" tmp_out="$SEGDIR/.timelapse_tmp.mp4"
    local count=0 f
    : > "$list"
    while IFS= read -r f; do
        printf "file '%s'\n" "$f" >> "$list"
        count=$((count + 1))
    done < <(find "$SEGDIR" -maxdepth 1 -name 'seg_*.mp4' -printf '%f\n' | LC_ALL=C sort)

    (( count > 0 )) || return 0

    log "    updating timelapse.mp4 - concatenating $count segments (stream copy, no re-encode)..."
    if ( cd "$SEGDIR" && ffmpeg -hide_banner -loglevel error -y \
            -f concat -safe 0 -i "$list" \
            -c copy -an "$tmp_out" >> "$FFMPEG_LOG" 2>&1 ); then
        mv -f -- "$tmp_out" "$VIDEO"
        log "    timelapse.mp4 updated"
    else
        rm -f -- "$tmp_out"
        log "    WARNING: video update failed (see ffmpeg.log); will retry on next run"
    fi
}

# ---------------------------------------------------------------------------
# Process a single archive. Returns non-zero on failure (archive is then NOT
# marked as processed and will be retried on the next run).
process_one() {
    local base="$1"
    local stem="${base%.zip}"
    local tmp_seg="$SEGDIR/.tmp_seg_${stem}.mp4"
    local hash_tmp="$TMPDIR/.hashes_new.txt"

    # clean the workspace (also wipes leftovers from an interrupted run)
    rm -rf -- "$WE" "$WF"
    mkdir -p -- "$WE" "$WF"
    rm -f -- "$tmp_seg"
    : > "$hash_tmp"

    # -- 1. extract --
    unzip -q -- "$SRC_DIR/$base" -d "$WE"
    local rc=$?
    if (( rc > 1 )); then   # unzip: 0=ok, 1=warnings only
        log "    ERROR: unzip failed (rc=$rc)"
        return 1
    fi

    # -- 2. md5 all jpgs, drop duplicates (archive-local DB + global DB) --
    local arch_total=0 arch_new=0 arch_dup=0
    local -A LOCAL=()
    local -a NEWFRAMES=()
    local line h f
    while IFS= read -r -d '' line; do
        h=${line:0:32}          # "md5sum -z" line: <32 hex><space><space><path>\0
        f=${line:34}
        arch_total=$((arch_total + 1))
        if [[ -n "${SEEN[$h]:-}" || -n "${LOCAL[$h]:-}" ]]; then
            arch_dup=$((arch_dup + 1))
        else
            LOCAL[$h]=1
            arch_new=$((arch_new + 1))
            NEWFRAMES+=("$f")
            printf '%s  %s  [%s]\n' "$h" "${f##*/}" "$base" >> "$hash_tmp"
        fi
    done < <(find "$WE" -type f -iname '*.jpg' -print0 \
             | LC_ALL=C sort -z \
             | xargs -0 -r md5sum -z --)

    log "    pictures: $arch_total total, $arch_new new, $arch_dup duplicate(s) removed"

    # -- 3. encode the new unique frames into a per-archive segment --
    if (( arch_new > 0 )); then
        local n=0 seq
        while IFS= read -r -d '' f; do
            n=$((n + 1))
            printf -v seq 'seq_%06d.jpg' "$n"
            ln -s -- "$f" "$WF/$seq"
        done < <(printf '%s\0' "${NEWFRAMES[@]}" | LC_ALL=C sort -z)  # sorted by date in filename

        log "    encoding $arch_new new frame(s) at ${FPS} fps with libsvtav1 (preset $SVT_PRESET, crf $SVT_CRF)..."
        if ffmpeg -hide_banner -loglevel error -y \
                -framerate "$FPS" -i "$WF/seq_%06d.jpg" \
                -c:v libsvtav1 -preset "$SVT_PRESET" -crf "$SVT_CRF" \
                -pix_fmt yuv420p -video_track_timescale 15360 -an \
                "$tmp_seg" >> "$FFMPEG_LOG" 2>&1; then
            mv -f -- "$tmp_seg" "$SEGDIR/seg_${stem}.mp4"
        else
            rm -f -- "$tmp_seg"
            log "    ERROR: ffmpeg encoding failed (see ffmpeg.log)"
            return 1
        fi
    else
        log "    nothing to encode - all pictures were duplicates"
    fi

    # -- 4. append html files to the big html zip --
    local -a HTMLS=()
    while IFS= read -r -d '' f; do
        HTMLS+=("$f")
    done < <(find "$WE" -type f -iname '*.html' -print0)
    if (( ${#HTMLS[@]} > 0 )); then
        if zip -q -j "$HTML_ZIP" -- "${HTMLS[@]}"; then
            log "    added ${#HTMLS[@]} html file(s) to $(basename "$HTML_ZIP")"
        else
            log "    ERROR: updating the html zip failed"
            return 1
        fi
    fi

    # -- commit: only now touch the permanent state --
    # (stats go first: if we crash between stats and hash/marker writes, a
    #  re-run finds the entry and reuses its numbers instead of recounting)
    if [[ -n "${STAT_T[$base]:-}" ]]; then
        arch_total=${STAT_T[$base]}
        arch_new=${STAT_N[$base]}
        arch_dup=${STAT_D[$base]}
    else
        printf '%s\t%d\t%d\t%d\n' "$base" "$arch_total" "$arch_new" "$arch_dup" >> "$STATS_FILE"
        STAT_T[$base]=$arch_total; STAT_N[$base]=$arch_new; STAT_D[$base]=$arch_dup
    fi
    cat "$hash_tmp" >> "$HASH_FILE"
    for h in "${!LOCAL[@]}"; do SEEN[$h]=1; done
    TOTAL_ALL=$((TOTAL_ALL + arch_total))
    UNIQUE_ALL=$((UNIQUE_ALL + arch_new))
    DUPS_ALL=$((DUPS_ALL + arch_dup))
    printf '%s\n' "$base" >> "$PROCESSED_FILE"
    PROCD[$base]=1

    # -- 5. update the final video (remux only) --
    (( arch_new > 0 )) && rebuild_video

    # -- 6. delete extracted jpgs/htmls from TMPDIR --
    rm -rf -- "$WE" "$WF"
    mkdir -p -- "$WF"

    log "    cumulative: $TOTAL_ALL pictures seen, $UNIQUE_ALL unique kept, $DUPS_ALL duplicates removed"
    return 0
}

# ===========================================================================
#  main
# ===========================================================================
mkdir -p -- "$TMPDIR" "$TMPDIR2" "$SEGDIR" "$WF" \
    || die "cannot create TMPDIR/TMPDIR2 - check the configuration paths"
[[ "$TMPDIR" == / || "$TMPDIR2" == / ]] && die "cowardly refusing to use / as TMPDIR/TMPDIR2"
[[ -d "$SRC_DIR" ]] || die "SRC_DIR '$SRC_DIR' does not exist"
touch "$LOG_FILE" "$PROCESSED_FILE" "$HASH_FILE" "$STATS_FILE" "$FFMPEG_LOG"

for tool in unzip zip ffmpeg md5sum awk find sort xargs; do
    command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool"
done
ENCODERS="$(ffmpeg -hide_banner -encoders 2>/dev/null)"
grep -q 'libsvtav1' <<< "$ENCODERS" \
    || die "this ffmpeg build has no libsvtav1 encoder"

AV1HOSTNAME=$(hostname)
log "=================================================================="
log "timelapse builder starting"
log "  SRC_DIR = $SRC_DIR   (pattern: $ARCHIVE_PATTERN)"
log "  TMPDIR  = $TMPDIR    (scratch, cleaned per archive)"
log "  TMPDIR2 = $TMPDIR2   (outputs + logs)"
log "  video   = ${FPS} fps, libsvtav1, preset $SVT_PRESET, crf $SVT_CRF"
log "  host    = $AV1HOSTNAME"

# ---- load permanent state (resume support) ----
declare -A PROCD=()
declare -A SEEN=()
while IFS= read -r a; do [[ -n "$a" ]] && PROCD[$a]=1; done < "$PROCESSED_FILE"
while read -r h _; do [[ -n "$h" ]] && SEEN[$h]=1; done < "$HASH_FILE"

# heal + load counters from the per-archive stats (dedup keeps first entry)
TOTAL_ALL=0; UNIQUE_ALL=0; DUPS_ALL=0
declare -A STAT_T=() STAT_N=() STAT_D=()
if [[ -s "$STATS_FILE" ]]; then
    awk -F'\t' '!s[$1]++' "$STATS_FILE" > "$STATS_FILE.tmp" \
        && mv -f "$STATS_FILE.tmp" "$STATS_FILE"
    while IFS=$'\t' read -r a t u d; do
        [[ -n "$a" ]] || continue
        STAT_T[$a]=$t; STAT_N[$a]=$u; STAT_D[$a]=$d
        # only archives that were fully committed (marker written) count;
        # the rest will be reprocessed and their stored numbers reused
        if [[ -n "${PROCD[$a]:-}" ]]; then
            TOTAL_ALL=$((TOTAL_ALL + t))
            UNIQUE_ALL=$((UNIQUE_ALL + u))
            DUPS_ALL=$((DUPS_ALL + d))
        fi
    done < "$STATS_FILE"
fi
log "  resume state: ${#PROCD[@]} archive(s) already processed, ${#SEEN[@]} known picture hashes, counters: $TOTAL_ALL seen / $UNIQUE_ALL unique / $DUPS_ALL dupes"

# ---- self-heal: segment(s) committed but video rebuild was interrupted ----
if compgen -G "$SEGDIR/seg_*.mp4" > /dev/null \
   && { [[ ! -f "$VIDEO" ]] \
        || [[ -n "$(find "$SEGDIR" -maxdepth 1 -name 'seg_*.mp4' -newer "$VIDEO" -print -quit)" ]]; }; then
    log "  detected segments newer than timelapse.mp4 - rebuilding video first"
    rebuild_video
fi

# ---- gather archives ----
mapfile -t ARCHIVES < <(find "$SRC_DIR" -maxdepth 1 -type f \
                        -name "$ARCHIVE_PATTERN" -printf '%f\n' | LC_ALL=C sort)
N=${#ARCHIVES[@]}
(( N > 0 )) || die "no files matching '$ARCHIVE_PATTERN' in $SRC_DIR"
log "found $N archive(s)"

IDX=0; SKIPPED=0; DONE_ONE=0; FAILED=0
for BASE in "${ARCHIVES[@]}"; do
    IDX=$((IDX + 1))
    if [[ -n "${PROCD[$BASE]:-}" ]]; then
        log "Processing archive $IDX of $N: $BASE - already processed, skipping"
        SKIPPED=$((SKIPPED + 1))
        continue
    fi
    log "Processing archive $IDX of $N: $BASE"
    if process_one "$BASE"; then
        DONE_ONE=$((DONE_ONE + 1))
    else
        FAILED=$((FAILED + 1))
        log "Processing archive $IDX of $N: $BASE - FAILED (left unprocessed, will be retried next run)"
    fi
done

rm -rf -- "$WE" "$WF"   # make sure TMPDIR holds no jpgs/htmls at the end

log "=================================================================="
log "all done: $DONE_ONE archive(s) processed, $SKIPPED skipped, $FAILED failed (out of $N)"
log "RESULT: $TOTAL_ALL pictures found in total"
log "RESULT: $UNIQUE_ALL unique pictures (frames in the video)"
log "RESULT: $DUPS_ALL duplicates removed"
log "outputs: $VIDEO"
log "         $HTML_ZIP"
exit 0