#!/usr/bin/env bash
set -Eeuo pipefail

ROOT="/mnt/bigstor/www/usecases"
VENV="$ROOT/.venv"
LOG_DIR="$ROOT/logs"
LOCK_FILE="$ROOT/.collect_daily.lock"
STAMP="$(date '+%Y%m%d-%H%M%S')"
LOG_FILE="$LOG_DIR/daily-$STAMP.log"
STATUS_FILE="$ROOT/site/update_status.json"

mkdir -p "$LOG_DIR"
cd "$ROOT"

exec 9>"$LOCK_FILE"
if ! flock -n 9; then
    echo "[$(date '+%F %T')] ERROR daily collection already running" >> "$LOG_FILE"
    exit 75
fi

exec >> >(tee -a "$LOG_FILE") 2>&1

trap 'rc=$?; echo "[$(date "+%F %T")] ERROR line=$LINENO command=$BASH_COMMAND exit=$rc"; exit $rc' ERR

echo "============================================================"
echo "[$(date '+%F %T')] DAILY COLLECTION START"
echo "Root: $ROOT"
echo "Log:  $LOG_FILE"
echo "============================================================"

if [ -f "$VENV/bin/activate" ]; then
    # shellcheck disable=SC1091
    source "$VENV/bin/activate"
fi

run_step() {
    local name="$1"
    shift
    echo
    echo "[$(date '+%F %T')] START $name"
    "$@"
    local rc=$?
    echo "[$(date '+%F %T')] DONE  $name exit=$rc"
}

# Daily collectors. These are the collector names already used in the project.
run_step "CVE / zero-day collection" python3 zero_day_intel.py
run_step "Paper collection" python3 papers_intel.py
run_step "GitHub collection" python3 github_intel.py --force

# Signals/incidents are collected when their collector exists. This keeps this
# script compatible with the current tree without inventing a missing command.
if [ -f signal_detector.py ]; then
    run_step "Signal collection" python3 signal_detector.py
else
    echo "[$(date '+%F %T')] SKIP signal_detector.py not present"
fi

if [ -f incidents_intel.py ]; then
    run_step "Incident collection" python3 incidents_intel.py
else
    echo "[$(date '+%F %T')] SKIP incidents_intel.py not present"
fi

# Validate the core outputs that the weekly pipeline depends on.
for json_file in \
    zero_day_threats/zero_day.json \
    papers_threats/papers.json \
    github/github.json
do
    echo "[$(date '+%F %T')] VALIDATE $json_file"
    python3 -m json.tool "$json_file" >/dev/null
done

for json_file in signals/signals.json incidents/incidents.json narratives/narratives.json; do
    if [ -f "$json_file" ]; then
        echo "[$(date '+%F %T')] VALIDATE $json_file"
        python3 -m json.tool "$json_file" >/dev/null
    fi
done

# Update status only after the entire daily collection and validation succeeded.
mkdir -p "$(dirname "$STATUS_FILE")"
python3 - "$STATUS_FILE" <<'PY_STATUS'
import json
import os
import sys
import tempfile
from datetime import datetime
from pathlib import Path

path = Path(sys.argv[1])
data = {
    "collection_schedule": "Daily",
    "correlation_schedule": "Weekly",
    "last_collection": None,
    "last_correlation": None,
    "last_publication": None,
}
if path.exists():
    try:
        current = json.loads(path.read_text(encoding="utf-8"))
        if isinstance(current, dict):
            data.update(current)
    except (OSError, json.JSONDecodeError):
        pass

data["collection_schedule"] = "Daily"
data["correlation_schedule"] = "Weekly"
data["last_collection"] = datetime.now().astimezone().isoformat(timespec="seconds")

fd, tmp_name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=path.parent)
try:
    with os.fdopen(fd, "w", encoding="utf-8") as handle:
        json.dump(data, handle, indent=2, ensure_ascii=False)
        handle.write("\n")
        handle.flush()
        os.fsync(handle.fileno())
    os.replace(tmp_name, path)
except Exception:
    try:
        os.unlink(tmp_name)
    except FileNotFoundError:
        pass
    raise
PY_STATUS
python3 -m json.tool "$STATUS_FILE" >/dev/null
echo "[$(date '+%F %T')] UPDATED $STATUS_FILE"

echo
echo "[$(date '+%F %T')] DAILY COLLECTION COMPLETE"
