vita/av1 encoding

av1-script.sh · 13 KiB · Bash Raw

#!/usr/bin/env bash
#
# kimi k3 spent 60k tokens writing this
# date: 2026-10-31
#
# runs well, continuation support is implemented well, suited for running
# in a supercomputer environment
#
# ============================================================================
#  script.sh - incremental 60 fps AV1 timelapse builder from archive-*.zip
# ============================================================================
#
#  For every input archive (processed ONE at a time, in sorted order):
#    1. extract it into TMPDIR
#    2. hash every .jpg with md5sum, drop duplicates (within the archive,
#       across archives, and across script restarts - the hash DB lives
#       permanently in TMPDIR2)
#    3. encode the new unique frames (sorted by date from the filename)
#       ONCE with libsvtav1 into a small per-archive segment file
#    4. update the final mp4 by CONCATENATING all segments with stream copy
#       (=> the video is updated WITHOUT re-encoding old frames)
#    5. append the .html files to one big html archive (zip) in TMPDIR2
#    6. delete the extracted jpgs and htmls from TMPDIR
#
#  Results kept in TMPDIR2:
#    timelapse.mp4           - the growing 60 fps AV1 timelapse
#    html-archive.zip        - the big html archive
#    progress.log            - human readable log with timestamps (mirrors stdout)
#    processed-archives.txt  - names of finished archives (resume support)
#    seen-hashes.txt         - md5 hash DB of every kept picture
#    per-archive-stats.tsv   - per-archive picture counters
#    ffmpeg.log              - raw ffmpeg output (kept out of the human log)
#    segments/               - per-archive AV1 segments (the encoded source
#                              of timelapse.mp4; do NOT delete, needed to
#                              update the video without re-encoding)
#
#  Resume: just re-run the script. Archives listed in processed-archives.txt
#  are skipped. Counters and the hash DB are rebuilt/healed from the files in
#  TMPDIR2, so an interruption at (almost) any point is safe.
#
#  The script uses no paths other than the CONFIGURATION block below and can
#  be run from any working directory.
#
# ============================================================================

set -uo pipefail

# ============================ CONFIGURATION ================================
SRC_DIR="/mnt/mystorage/meteo-padochov"   # directory containing the input archives
ARCHIVE_PATTERN="archive-*.zip"           # glob for the input archives in SRC_DIR

TMPDIR="/mnt/storage/timelapse-work"      # scratch space: extraction + frame staging
                                          # (contents deleted after every archive)
TMPDIR2="/mnt/storage/timelapse-out"      # output: mp4, html zip, logs, segments

FPS=60            # timelapse frame rate
SVT_PRESET=8      # libsvtav1 preset: 0 = slow/best ... 13 = fastest (7-9 is a good spot)
SVT_CRF=32        # libsvtav1 quality: lower = better/bigger (30-35 sane for webcams)
# ===========================================================================

# ---- derived paths (no need to touch) ----
VIDEO="$TMPDIR2/timelapse.mp4"
HTML_ZIP="$TMPDIR2/html-archive.zip"
SEGDIR="$TMPDIR2/segments"
LOG_FILE="$TMPDIR2/progress.log"
PROCESSED_FILE="$TMPDIR2/processed-archives.txt"
HASH_FILE="$TMPDIR2/seen-hashes.txt"
STATS_FILE="$TMPDIR2/per-archive-stats.tsv"
FFMPEG_LOG="$TMPDIR2/ffmpeg.log"

WE="$TMPDIR/extract"      # extraction area
WF="$TMPDIR/frames"       # staged frames of the current archive

# ---------------------------------------------------------------------------
log() {  # timestamped message to stdout AND the human log file
    local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
    printf '%s\n' "$msg" | tee -a "$LOG_FILE"
}

die() { log "FATAL: $*"; exit 1; }

on_interrupt() {
    log "INTERRUPTED - partial state is safe. Just re-run the script to resume."
    exit 130
}
trap on_interrupt INT TERM

# ---------------------------------------------------------------------------
# Rebuild timelapse.mp4 from all per-archive segments (stream copy = no re-encode)
rebuild_video() {
    local list="$SEGDIR/.concat_list.txt" tmp_out="$SEGDIR/.timelapse_tmp.mp4"
    local count=0 f
    : > "$list"
    while IFS= read -r f; do
        printf "file '%s'\n" "$f" >> "$list"
        count=$((count + 1))
    done < <(find "$SEGDIR" -maxdepth 1 -name 'seg_*.mp4' -printf '%f\n' | LC_ALL=C sort)

(( count > 0 )) || return 0

log "    updating timelapse.mp4 - concatenating $count segments (stream copy, no re-encode)..."
    if ( cd "$SEGDIR" && ffmpeg -hide_banner -loglevel error -y \
            -f concat -safe 0 -i "$list" \
            -c copy -an "$tmp_out" >> "$FFMPEG_LOG" 2>&1 ); then
        mv -f -- "$tmp_out" "$VIDEO"
        log "    timelapse.mp4 updated"
    else
        rm -f -- "$tmp_out"
        log "    WARNING: video update failed (see ffmpeg.log); will retry on next run"
    fi
}

# ---------------------------------------------------------------------------
# Process a single archive. Returns non-zero on failure (archive is then NOT
# marked as processed and will be retried on the next run).
process_one() {
    local base="$1"
    local stem="${base%.zip}"
    local tmp_seg="$SEGDIR/.tmp_seg_${stem}.mp4"
    local hash_tmp="$TMPDIR/.hashes_new.txt"

# clean the workspace (also wipes leftovers from an interrupted run)
    rm -rf -- "$WE" "$WF"
    mkdir -p -- "$WE" "$WF"
    rm -f -- "$tmp_seg"
    : > "$hash_tmp"

# -- 1. extract --
    unzip -q -- "$SRC_DIR/$base" -d "$WE"
    local rc=$?
    if (( rc > 1 )); then   # unzip: 0=ok, 1=warnings only
        log "    ERROR: unzip failed (rc=$rc)"
        return 1
    fi

# -- 2. md5 all jpgs, drop duplicates (archive-local DB + global DB) --
    local arch_total=0 arch_new=0 arch_dup=0
    local -A LOCAL=()
    local -a NEWFRAMES=()
    local line h f
    while IFS= read -r -d '' line; do
        h=${line:0:32}          # "md5sum -z" line: <32 hex><space><space><path>\0
        f=${line:34}
        arch_total=$((arch_total + 1))
        if [[ -n "${SEEN[$h]:-}" || -n "${LOCAL[$h]:-}" ]]; then
            arch_dup=$((arch_dup + 1))
        else
            LOCAL[$h]=1
            arch_new=$((arch_new + 1))
            NEWFRAMES+=("$f")
            printf '%s  %s  [%s]\n' "$h" "${f##*/}" "$base" >> "$hash_tmp"
        fi
    done < <(find "$WE" -type f -iname '*.jpg' -print0 \
             | LC_ALL=C sort -z \
             | xargs -0 -r md5sum -z --)

log "    pictures: $arch_total total, $arch_new new, $arch_dup duplicate(s) removed"

# -- 3. encode the new unique frames into a per-archive segment --
    if (( arch_new > 0 )); then
        local n=0 seq
        while IFS= read -r -d '' f; do
            n=$((n + 1))
            printf -v seq 'seq_%06d.jpg' "$n"
            ln -s -- "$f" "$WF/$seq"
        done < <(printf '%s\0' "${NEWFRAMES[@]}" | LC_ALL=C sort -z)  # sorted by date in filename

log "    encoding $arch_new new frame(s) at ${FPS} fps with libsvtav1 (preset $SVT_PRESET, crf $SVT_CRF)..."
        if ffmpeg -hide_banner -loglevel error -y \
                -framerate "$FPS" -i "$WF/seq_%06d.jpg" \
                -c:v libsvtav1 -preset "$SVT_PRESET" -crf "$SVT_CRF" \
                -pix_fmt yuv420p -video_track_timescale 15360 -an \
                "$tmp_seg" >> "$FFMPEG_LOG" 2>&1; then
            mv -f -- "$tmp_seg" "$SEGDIR/seg_${stem}.mp4"
        else
            rm -f -- "$tmp_seg"
            log "    ERROR: ffmpeg encoding failed (see ffmpeg.log)"
            return 1
        fi
    else
        log "    nothing to encode - all pictures were duplicates"
    fi

# -- 4. append html files to the big html zip --
    local -a HTMLS=()
    while IFS= read -r -d '' f; do
        HTMLS+=("$f")
    done < <(find "$WE" -type f -iname '*.html' -print0)
    if (( ${#HTMLS[@]} > 0 )); then
        if zip -q -j "$HTML_ZIP" -- "${HTMLS[@]}"; then
            log "    added ${#HTMLS[@]} html file(s) to $(basename "$HTML_ZIP")"
        else
            log "    ERROR: updating the html zip failed"
            return 1
        fi
    fi

# -- commit: only now touch the permanent state --
    # (stats go first: if we crash between stats and hash/marker writes, a
    #  re-run finds the entry and reuses its numbers instead of recounting)
    if [[ -n "${STAT_T[$base]:-}" ]]; then
        arch_total=${STAT_T[$base]}
        arch_new=${STAT_N[$base]}
        arch_dup=${STAT_D[$base]}
    else
        printf '%s\t%d\t%d\t%d\n' "$base" "$arch_total" "$arch_new" "$arch_dup" >> "$STATS_FILE"
        STAT_T[$base]=$arch_total; STAT_N[$base]=$arch_new; STAT_D[$base]=$arch_dup
    fi
    cat "$hash_tmp" >> "$HASH_FILE"
    for h in "${!LOCAL[@]}"; do SEEN[$h]=1; done
    TOTAL_ALL=$((TOTAL_ALL + arch_total))
    UNIQUE_ALL=$((UNIQUE_ALL + arch_new))
    DUPS_ALL=$((DUPS_ALL + arch_dup))
    printf '%s\n' "$base" >> "$PROCESSED_FILE"
    PROCD[$base]=1

# -- 5. update the final video (remux only) --
    (( arch_new > 0 )) && rebuild_video

# -- 6. delete extracted jpgs/htmls from TMPDIR --
    rm -rf -- "$WE" "$WF"
    mkdir -p -- "$WF"

log "    cumulative: $TOTAL_ALL pictures seen, $UNIQUE_ALL unique kept, $DUPS_ALL duplicates removed"
    return 0
}

# ===========================================================================
#  main
# ===========================================================================
mkdir -p -- "$TMPDIR" "$TMPDIR2" "$SEGDIR" "$WF" \
    || die "cannot create TMPDIR/TMPDIR2 - check the configuration paths"
[[ "$TMPDIR" == / || "$TMPDIR2" == / ]] && die "cowardly refusing to use / as TMPDIR/TMPDIR2"
[[ -d "$SRC_DIR" ]] || die "SRC_DIR '$SRC_DIR' does not exist"
touch "$LOG_FILE" "$PROCESSED_FILE" "$HASH_FILE" "$STATS_FILE" "$FFMPEG_LOG"

for tool in unzip zip ffmpeg md5sum awk find sort xargs; do
    command -v "$tool" >/dev/null 2>&1 || die "required tool not found: $tool"
done
ENCODERS="$(ffmpeg -hide_banner -encoders 2>/dev/null)"
grep -q 'libsvtav1' <<< "$ENCODERS" \
    || die "this ffmpeg build has no libsvtav1 encoder"

AV1HOSTNAME=$(hostname)
log "=================================================================="
log "timelapse builder starting"
log "  SRC_DIR = $SRC_DIR   (pattern: $ARCHIVE_PATTERN)"
log "  TMPDIR  = $TMPDIR    (scratch, cleaned per archive)"
log "  TMPDIR2 = $TMPDIR2   (outputs + logs)"
log "  video   = ${FPS} fps, libsvtav1, preset $SVT_PRESET, crf $SVT_CRF"
log "  host    = $AV1HOSTNAME"

# ---- load permanent state (resume support) ----
declare -A PROCD=()
declare -A SEEN=()
while IFS= read -r a; do [[ -n "$a" ]] && PROCD[$a]=1; done < "$PROCESSED_FILE"
while read -r h _; do [[ -n "$h" ]] && SEEN[$h]=1; done < "$HASH_FILE"

# heal + load counters from the per-archive stats (dedup keeps first entry)
TOTAL_ALL=0; UNIQUE_ALL=0; DUPS_ALL=0
declare -A STAT_T=() STAT_N=() STAT_D=()
if [[ -s "$STATS_FILE" ]]; then
    awk -F'\t' '!s[$1]++' "$STATS_FILE" > "$STATS_FILE.tmp" \
        && mv -f "$STATS_FILE.tmp" "$STATS_FILE"
    while IFS=$'\t' read -r a t u d; do
        [[ -n "$a" ]] || continue
        STAT_T[$a]=$t; STAT_N[$a]=$u; STAT_D[$a]=$d
        # only archives that were fully committed (marker written) count;
        # the rest will be reprocessed and their stored numbers reused
        if [[ -n "${PROCD[$a]:-}" ]]; then
            TOTAL_ALL=$((TOTAL_ALL + t))
            UNIQUE_ALL=$((UNIQUE_ALL + u))
            DUPS_ALL=$((DUPS_ALL + d))
        fi
    done < "$STATS_FILE"
fi
log "  resume state: ${#PROCD[@]} archive(s) already processed, ${#SEEN[@]} known picture hashes, counters: $TOTAL_ALL seen / $UNIQUE_ALL unique / $DUPS_ALL dupes"

# ---- self-heal: segment(s) committed but video rebuild was interrupted ----
if compgen -G "$SEGDIR/seg_*.mp4" > /dev/null \
   && { [[ ! -f "$VIDEO" ]] \
        || [[ -n "$(find "$SEGDIR" -maxdepth 1 -name 'seg_*.mp4' -newer "$VIDEO" -print -quit)" ]]; }; then
    log "  detected segments newer than timelapse.mp4 - rebuilding video first"
    rebuild_video
fi

# ---- gather archives ----
mapfile -t ARCHIVES < <(find "$SRC_DIR" -maxdepth 1 -type f \
                        -name "$ARCHIVE_PATTERN" -printf '%f\n' | LC_ALL=C sort)
N=${#ARCHIVES[@]}
(( N > 0 )) || die "no files matching '$ARCHIVE_PATTERN' in $SRC_DIR"
log "found $N archive(s)"

IDX=0; SKIPPED=0; DONE_ONE=0; FAILED=0
for BASE in "${ARCHIVES[@]}"; do
    IDX=$((IDX + 1))
    if [[ -n "${PROCD[$BASE]:-}" ]]; then
        log "Processing archive $IDX of $N: $BASE - already processed, skipping"
        SKIPPED=$((SKIPPED + 1))
        continue
    fi
    log "Processing archive $IDX of $N: $BASE"
    if process_one "$BASE"; then
        DONE_ONE=$((DONE_ONE + 1))
    else
        FAILED=$((FAILED + 1))
        log "Processing archive $IDX of $N: $BASE - FAILED (left unprocessed, will be retried next run)"
    fi
done

rm -rf -- "$WE" "$WF"   # make sure TMPDIR holds no jpgs/htmls at the end

log "=================================================================="
log "all done: $DONE_ONE archive(s) processed, $SKIPPED skipped, $FAILED failed (out of $N)"
log "RESULT: $TOTAL_ALL pictures found in total"
log "RESULT: $UNIQUE_ALL unique pictures (frames in the video)"
log "RESULT: $DUPS_ALL duplicates removed"
log "outputs: $VIDEO"
log "         $HTML_ZIP"
exit 0

1	#!/usr/bin/env bash
2	#
3	# kimi k3 spent 60k tokens writing this
4	# date: 2026-10-31
5	#
6	# runs well, continuation support is implemented well, suited for running
7	# in a supercomputer environment
8	#
9	# ============================================================================
10	# script.sh - incremental 60 fps AV1 timelapse builder from archive-*.zip
11	# ============================================================================
12	#
13	# For every input archive (processed ONE at a time, in sorted order):
14	# 1. extract it into TMPDIR
15	# 2. hash every .jpg with md5sum, drop duplicates (within the archive,
16	# across archives, and across script restarts - the hash DB lives
17	# permanently in TMPDIR2)
18	# 3. encode the new unique frames (sorted by date from the filename)
19	# ONCE with libsvtav1 into a small per-archive segment file
20	# 4. update the final mp4 by CONCATENATING all segments with stream copy
21	# (=> the video is updated WITHOUT re-encoding old frames)
22	# 5. append the .html files to one big html archive (zip) in TMPDIR2
23	# 6. delete the extracted jpgs and htmls from TMPDIR
24	#
25	# Results kept in TMPDIR2:
26	# timelapse.mp4 - the growing 60 fps AV1 timelapse
27	# html-archive.zip - the big html archive
28	# progress.log - human readable log with timestamps (mirrors stdout)
29	# processed-archives.txt - names of finished archives (resume support)
30	# seen-hashes.txt - md5 hash DB of every kept picture
31	# per-archive-stats.tsv - per-archive picture counters
32	# ffmpeg.log - raw ffmpeg output (kept out of the human log)
33	# segments/ - per-archive AV1 segments (the encoded source
34	# of timelapse.mp4; do NOT delete, needed to
35	# update the video without re-encoding)
36	#
37	# Resume: just re-run the script. Archives listed in processed-archives.txt
38	# are skipped. Counters and the hash DB are rebuilt/healed from the files in
39	# TMPDIR2, so an interruption at (almost) any point is safe.
40	#
41	# The script uses no paths other than the CONFIGURATION block below and can
42	# be run from any working directory.
43	#
44	# ============================================================================
45
46	set -uo pipefail
47
48	# ============================ CONFIGURATION ================================
49	SRC_DIR="/mnt/mystorage/meteo-padochov" # directory containing the input archives
50	ARCHIVE_PATTERN="archive-*.zip" # glob for the input archives in SRC_DIR
51
52	TMPDIR="/mnt/storage/timelapse-work" # scratch space: extraction + frame staging
53	# (contents deleted after every archive)
54	TMPDIR2="/mnt/storage/timelapse-out" # output: mp4, html zip, logs, segments
55
56	FPS=60 # timelapse frame rate
57	SVT_PRESET=8 # libsvtav1 preset: 0 = slow/best ... 13 = fastest (7-9 is a good spot)
58	SVT_CRF=32 # libsvtav1 quality: lower = better/bigger (30-35 sane for webcams)
59	# ===========================================================================
60
61	# ---- derived paths (no need to touch) ----
62	VIDEO="$TMPDIR2/timelapse.mp4"
63	HTML_ZIP="$TMPDIR2/html-archive.zip"
64	SEGDIR="$TMPDIR2/segments"
65	LOG_FILE="$TMPDIR2/progress.log"
66	PROCESSED_FILE="$TMPDIR2/processed-archives.txt"
67	HASH_FILE="$TMPDIR2/seen-hashes.txt"
68	STATS_FILE="$TMPDIR2/per-archive-stats.tsv"
69	FFMPEG_LOG="$TMPDIR2/ffmpeg.log"
70
71	WE="$TMPDIR/extract" # extraction area
72	WF="$TMPDIR/frames" # staged frames of the current archive
73
74	# ---------------------------------------------------------------------------
75	log() { # timestamped message to stdout AND the human log file
76	local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
77	printf '%s\n' "$msg" \| tee -a "$LOG_FILE"
78	}
79
80	die() { log "FATAL: $*"; exit 1; }
81
82	on_interrupt() {
83	log "INTERRUPTED - partial state is safe. Just re-run the script to resume."
84	exit 130
85	}
86	trap on_interrupt INT TERM
87
88	# ---------------------------------------------------------------------------
89	# Rebuild timelapse.mp4 from all per-archive segments (stream copy = no re-encode)
90	rebuild_video() {
91	local list="$SEGDIR/.concat_list.txt" tmp_out="$SEGDIR/.timelapse_tmp.mp4"
92	local count=0 f
93	: > "$list"
94	while IFS= read -r f; do
95	printf "file '%s'\n" "$f" >> "$list"
96	count=$((count + 1))
97	done < <(find "$SEGDIR" -maxdepth 1 -name 'seg_*.mp4' -printf '%f\n' \| LC_ALL=C sort)
98
99	(( count > 0 )) \|\| return 0
100
101	log " updating timelapse.mp4 - concatenating $count segments (stream copy, no re-encode)..."
102	if ( cd "$SEGDIR" && ffmpeg -hide_banner -loglevel error -y \
103	-f concat -safe 0 -i "$list" \
104	-c copy -an "$tmp_out" >> "$FFMPEG_LOG" 2>&1 ); then
105	mv -f -- "$tmp_out" "$VIDEO"
106	log " timelapse.mp4 updated"
107	else
108	rm -f -- "$tmp_out"
109	log " WARNING: video update failed (see ffmpeg.log); will retry on next run"
110	fi
111	}
112
113	# ---------------------------------------------------------------------------
114	# Process a single archive. Returns non-zero on failure (archive is then NOT
115	# marked as processed and will be retried on the next run).
116	process_one() {
117	local base="$1"
118	local stem="${base%.zip}"
119	local tmp_seg="$SEGDIR/.tmp_seg_${stem}.mp4"
120	local hash_tmp="$TMPDIR/.hashes_new.txt"
121
122	# clean the workspace (also wipes leftovers from an interrupted run)
123	rm -rf -- "$WE" "$WF"
124	mkdir -p -- "$WE" "$WF"
125	rm -f -- "$tmp_seg"
126	: > "$hash_tmp"
127
128	# -- 1. extract --
129	unzip -q -- "$SRC_DIR/$base" -d "$WE"
130	local rc=$?
131	if (( rc > 1 )); then # unzip: 0=ok, 1=warnings only
132	log " ERROR: unzip failed (rc=$rc)"
133	return 1
134	fi
135
136	# -- 2. md5 all jpgs, drop duplicates (archive-local DB + global DB) --
137	local arch_total=0 arch_new=0 arch_dup=0
138	local -A LOCAL=()
139	local -a NEWFRAMES=()
140	local line h f
141	while IFS= read -r -d '' line; do
142	h=${line:0:32} # "md5sum -z" line: <32 hex><space><space><path>\0
143	f=${line:34}
144	arch_total=$((arch_total + 1))
145	if [[ -n "${SEEN[$h]:-}" \|\| -n "${LOCAL[$h]:-}" ]]; then
146	arch_dup=$((arch_dup + 1))
147	else
148	LOCAL[$h]=1
149	arch_new=$((arch_new + 1))
150	NEWFRAMES+=("$f")
151	printf '%s %s [%s]\n' "$h" "${f##*/}" "$base" >> "$hash_tmp"
152	fi
153	done < <(find "$WE" -type f -iname '*.jpg' -print0 \
154	\| LC_ALL=C sort -z \
155	\| xargs -0 -r md5sum -z --)
156
157	log " pictures: $arch_total total, $arch_new new, $arch_dup duplicate(s) removed"
158
159	# -- 3. encode the new unique frames into a per-archive segment --
160	if (( arch_new > 0 )); then
161	local n=0 seq
162	while IFS= read -r -d '' f; do
163	n=$((n + 1))
164	printf -v seq 'seq_%06d.jpg' "$n"
165	ln -s -- "$f" "$WF/$seq"
166	done < <(printf '%s\0' "${NEWFRAMES[@]}" \| LC_ALL=C sort -z) # sorted by date in filename
167
168	log " encoding $arch_new new frame(s) at ${FPS} fps with libsvtav1 (preset $SVT_PRESET, crf $SVT_CRF)..."
169	if ffmpeg -hide_banner -loglevel error -y \
170	-framerate "$FPS" -i "$WF/seq_%06d.jpg" \
171	-c:v libsvtav1 -preset "$SVT_PRESET" -crf "$SVT_CRF" \
172	-pix_fmt yuv420p -video_track_timescale 15360 -an \
173	"$tmp_seg" >> "$FFMPEG_LOG" 2>&1; then
174	mv -f -- "$tmp_seg" "$SEGDIR/seg_${stem}.mp4"
175	else
176	rm -f -- "$tmp_seg"
177	log " ERROR: ffmpeg encoding failed (see ffmpeg.log)"
178	return 1
179	fi
180	else
181	log " nothing to encode - all pictures were duplicates"
182	fi
183
184	# -- 4. append html files to the big html zip --
185	local -a HTMLS=()
186	while IFS= read -r -d '' f; do
187	HTMLS+=("$f")
188	done < <(find "$WE" -type f -iname '*.html' -print0)
189	if (( ${#HTMLS[@]} > 0 )); then
190	if zip -q -j "$HTML_ZIP" -- "${HTMLS[@]}"; then
191	log " added ${#HTMLS[@]} html file(s) to $(basename "$HTML_ZIP")"
192	else
193	log " ERROR: updating the html zip failed"
194	return 1
195	fi
196	fi
197
198	# -- commit: only now touch the permanent state --
199	# (stats go first: if we crash between stats and hash/marker writes, a
200	# re-run finds the entry and reuses its numbers instead of recounting)
201	if [[ -n "${STAT_T[$base]:-}" ]]; then
202	arch_total=${STAT_T[$base]}
203	arch_new=${STAT_N[$base]}
204	arch_dup=${STAT_D[$base]}
205	else
206	printf '%s\t%d\t%d\t%d\n' "$base" "$arch_total" "$arch_new" "$arch_dup" >> "$STATS_FILE"
207	STAT_T[$base]=$arch_total; STAT_N[$base]=$arch_new; STAT_D[$base]=$arch_dup
208	fi
209	cat "$hash_tmp" >> "$HASH_FILE"
210	for h in "${!LOCAL[@]}"; do SEEN[$h]=1; done
211	TOTAL_ALL=$((TOTAL_ALL + arch_total))
212	UNIQUE_ALL=$((UNIQUE_ALL + arch_new))
213	DUPS_ALL=$((DUPS_ALL + arch_dup))
214	printf '%s\n' "$base" >> "$PROCESSED_FILE"
215	PROCD[$base]=1
216
217	# -- 5. update the final video (remux only) --
218	(( arch_new > 0 )) && rebuild_video
219
220	# -- 6. delete extracted jpgs/htmls from TMPDIR --
221	rm -rf -- "$WE" "$WF"
222	mkdir -p -- "$WF"
223
224	log " cumulative: $TOTAL_ALL pictures seen, $UNIQUE_ALL unique kept, $DUPS_ALL duplicates removed"
225	return 0
226	}
227
228	# ===========================================================================
229	# main
230	# ===========================================================================
231	mkdir -p -- "$TMPDIR" "$TMPDIR2" "$SEGDIR" "$WF" \
232	\|\| die "cannot create TMPDIR/TMPDIR2 - check the configuration paths"
233	[[ "$TMPDIR" == / \|\| "$TMPDIR2" == / ]] && die "cowardly refusing to use / as TMPDIR/TMPDIR2"
234	[[ -d "$SRC_DIR" ]] \|\| die "SRC_DIR '$SRC_DIR' does not exist"
235	touch "$LOG_FILE" "$PROCESSED_FILE" "$HASH_FILE" "$STATS_FILE" "$FFMPEG_LOG"
236
237	for tool in unzip zip ffmpeg md5sum awk find sort xargs; do
238	command -v "$tool" >/dev/null 2>&1 \|\| die "required tool not found: $tool"
239	done
240	ENCODERS="$(ffmpeg -hide_banner -encoders 2>/dev/null)"
241	grep -q 'libsvtav1' <<< "$ENCODERS" \
242	\|\| die "this ffmpeg build has no libsvtav1 encoder"
243
244	AV1HOSTNAME=$(hostname)
245	log "=================================================================="
246	log "timelapse builder starting"
247	log " SRC_DIR = $SRC_DIR (pattern: $ARCHIVE_PATTERN)"
248	log " TMPDIR = $TMPDIR (scratch, cleaned per archive)"
249	log " TMPDIR2 = $TMPDIR2 (outputs + logs)"
250	log " video = ${FPS} fps, libsvtav1, preset $SVT_PRESET, crf $SVT_CRF"
251	log " host = $AV1HOSTNAME"
252
253	# ---- load permanent state (resume support) ----
254	declare -A PROCD=()
255	declare -A SEEN=()
256	while IFS= read -r a; do [[ -n "$a" ]] && PROCD[$a]=1; done < "$PROCESSED_FILE"
257	while read -r h _; do [[ -n "$h" ]] && SEEN[$h]=1; done < "$HASH_FILE"
258
259	# heal + load counters from the per-archive stats (dedup keeps first entry)
260	TOTAL_ALL=0; UNIQUE_ALL=0; DUPS_ALL=0
261	declare -A STAT_T=() STAT_N=() STAT_D=()
262	if [[ -s "$STATS_FILE" ]]; then
263	awk -F'\t' '!s[$1]++' "$STATS_FILE" > "$STATS_FILE.tmp" \
264	&& mv -f "$STATS_FILE.tmp" "$STATS_FILE"
265	while IFS=$'\t' read -r a t u d; do
266	[[ -n "$a" ]] \|\| continue
267	STAT_T[$a]=$t; STAT_N[$a]=$u; STAT_D[$a]=$d
268	# only archives that were fully committed (marker written) count;
269	# the rest will be reprocessed and their stored numbers reused
270	if [[ -n "${PROCD[$a]:-}" ]]; then
271	TOTAL_ALL=$((TOTAL_ALL + t))
272	UNIQUE_ALL=$((UNIQUE_ALL + u))
273	DUPS_ALL=$((DUPS_ALL + d))
274	fi
275	done < "$STATS_FILE"
276	fi
277	log " resume state: ${#PROCD[@]} archive(s) already processed, ${#SEEN[@]} known picture hashes, counters: $TOTAL_ALL seen / $UNIQUE_ALL unique / $DUPS_ALL dupes"
278
279	# ---- self-heal: segment(s) committed but video rebuild was interrupted ----
280	if compgen -G "$SEGDIR/seg_*.mp4" > /dev/null \
281	&& { [[ ! -f "$VIDEO" ]] \
282	\|\| [[ -n "$(find "$SEGDIR" -maxdepth 1 -name 'seg_*.mp4' -newer "$VIDEO" -print -quit)" ]]; }; then
283	log " detected segments newer than timelapse.mp4 - rebuilding video first"
284	rebuild_video
285	fi
286
287	# ---- gather archives ----
288	mapfile -t ARCHIVES < <(find "$SRC_DIR" -maxdepth 1 -type f \
289	-name "$ARCHIVE_PATTERN" -printf '%f\n' \| LC_ALL=C sort)
290	N=${#ARCHIVES[@]}
291	(( N > 0 )) \|\| die "no files matching '$ARCHIVE_PATTERN' in $SRC_DIR"
292	log "found $N archive(s)"
293
294	IDX=0; SKIPPED=0; DONE_ONE=0; FAILED=0
295	for BASE in "${ARCHIVES[@]}"; do
296	IDX=$((IDX + 1))
297	if [[ -n "${PROCD[$BASE]:-}" ]]; then
298	log "Processing archive $IDX of $N: $BASE - already processed, skipping"
299	SKIPPED=$((SKIPPED + 1))
300	continue
301	fi
302	log "Processing archive $IDX of $N: $BASE"
303	if process_one "$BASE"; then
304	DONE_ONE=$((DONE_ONE + 1))
305	else
306	FAILED=$((FAILED + 1))
307	log "Processing archive $IDX of $N: $BASE - FAILED (left unprocessed, will be retried next run)"
308	fi
309	done
310
311	rm -rf -- "$WE" "$WF" # make sure TMPDIR holds no jpgs/htmls at the end
312
313	log "=================================================================="
314	log "all done: $DONE_ONE archive(s) processed, $SKIPPED skipped, $FAILED failed (out of $N)"
315	log "RESULT: $TOTAL_ALL pictures found in total"
316	log "RESULT: $UNIQUE_ALL unique pictures (frames in the video)"
317	log "RESULT: $DUPS_ALL duplicates removed"
318	log "outputs: $VIDEO"
319	log " $HTML_ZIP"
320	exit 0