#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
github_intel.py

Build a normalized GitHub repository evidence dataset for the NCRC
Automotive Cybersecurity Threat Register.

Input
-----
github_probe.json

The input is produced by github_probe.py and contains discovered GitHub
repositories with deterministic relevance classifications.

Selection
---------
Only repositories with relevance levels:

    direct
    strong

are selected by default.

Processing
----------
For each selected repository:

1. Refresh repository metadata through the GitHub REST API.
2. Retrieve the repository README as raw text.
3. Preserve discovery classification and relevance information.
4. Normalize the result into a paper-like evidence structure.
5. Write checkpoint data during processing.
6. Write the final dataset to github/github.json.

The script does NOT use AI or LLM analysis.

Authentication
--------------
Set a GitHub personal access token:

    export GITHUB_TOKEN="github_pat_..."

The token is never written to output.

Examples
--------

Standard run:

    python3 github_intel.py -v

Resume an interrupted run:

    python3 github_intel.py --resume -v

Force a clean rebuild:

    python3 github_intel.py --force -v

Test only 10 repositories:

    python3 github_intel.py --limit 10 -v

Select contextual repositories too:

    python3 github_intel.py \
        --levels direct strong contextual \
        -v

Validate input without API retrieval:

    python3 github_intel.py --validate-only

Show detailed logging:

    python3 github_intel.py -vv
"""

from __future__ import annotations

import argparse
import hashlib
import json
import logging
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


# =============================================================================
# CONFIGURATION
# =============================================================================

SCRIPT_VERSION = "1.0.0"
SCHEMA_VERSION = "1.0"

GITHUB_API = "https://api.github.com"

DEFAULT_INPUT = Path("github_probe.json")
DEFAULT_OUTPUT = Path("github/github.json")
DEFAULT_CHECKPOINT = Path("github/github_checkpoint.json")

DEFAULT_LEVELS = [
    "direct",
    "strong",
]

DEFAULT_README_MAX_CHARS = 80_000
DEFAULT_CHECKPOINT_INTERVAL = 10
DEFAULT_REQUEST_DELAY = 0.20
DEFAULT_TIMEOUT = 30
DEFAULT_RETRIES = 6

LOG = logging.getLogger("github_intel")


# =============================================================================
# GENERAL HELPERS
# =============================================================================

def utc_now() -> str:
    return (
        datetime.now(timezone.utc)
        .replace(microsecond=0)
        .isoformat()
        .replace("+00:00", "Z")
    )


def configure_logging(verbose: int) -> None:
    if verbose >= 2:
        level = logging.DEBUG
    elif verbose == 1:
        level = logging.INFO
    else:
        level = logging.WARNING

    logging.basicConfig(
        level=level,
        format="%(asctime)s %(levelname)-8s %(message)s",
    )


def clean_text(value: Any) -> str:
    if value is None:
        return ""

    return " ".join(
        str(value)
        .replace("\r", " ")
        .replace("\n", " ")
        .split()
    )


def normalize_multiline_text(value: Any) -> str:
    if value is None:
        return ""

    text = str(value).replace("\r\n", "\n").replace("\r", "\n")

    text = text.replace("\x00", "")

    lines = [
        line.rstrip()
        for line in text.splitlines()
    ]

    while lines and not lines[0].strip():
        lines.pop(0)

    while lines and not lines[-1].strip():
        lines.pop()

    return "\n".join(lines)


def safe_int(value: Any) -> int:
    try:
        return int(value or 0)
    except (TypeError, ValueError):
        return 0


def safe_float(value: Any) -> float:
    try:
        return float(value or 0)
    except (TypeError, ValueError):
        return 0.0


def stable_hash(value: str) -> str:
    return hashlib.sha256(
        value.encode(
            "utf-8",
            errors="replace",
        )
    ).hexdigest()


def compact_hash(value: str, length: int = 16) -> str:
    return stable_hash(value)[:length]


def atomic_write_json(
    path: Path,
    payload: Any,
) -> None:
    path.parent.mkdir(
        parents=True,
        exist_ok=True,
    )

    temporary = path.with_suffix(
        path.suffix + ".tmp"
    )

    with temporary.open(
        "w",
        encoding="utf-8",
    ) as handle:
        json.dump(
            payload,
            handle,
            indent=2,
            ensure_ascii=False,
        )
        handle.write("\n")

    temporary.replace(path)


def load_json(path: Path) -> Any:
    with path.open(
        "r",
        encoding="utf-8",
    ) as handle:
        return json.load(handle)


# =============================================================================
# GITHUB IDENTITY HELPERS
# =============================================================================

def normalize_full_name(value: Any) -> str:
    text = clean_text(value).strip("/")

    if text.lower().startswith("github:"):
        text = text[7:]

    if text.startswith("https://github.com/"):
        text = text[len("https://github.com/"):]

    if text.startswith("http://github.com/"):
        text = text[len("http://github.com/"):]

    pieces = [
        piece
        for piece in text.split("/")
        if piece
    ]

    if len(pieces) < 2:
        return ""

    owner = pieces[0]
    repository = pieces[1]

    if repository.endswith(".git"):
        repository = repository[:-4]

    if not owner or not repository:
        return ""

    return f"{owner}/{repository}"


def split_full_name(
    full_name: str,
) -> tuple[str, str]:
    normalized = normalize_full_name(
        full_name
    )

    if not normalized:
        raise ValueError(
            f"Invalid GitHub repository identity: {full_name!r}"
        )

    owner, repository = normalized.split(
        "/",
        1,
    )

    return owner, repository


def repository_api_url(
    full_name: str,
) -> str:
    owner, repository = split_full_name(
        full_name
    )

    return (
        f"{GITHUB_API}/repos/"
        f"{urllib.parse.quote(owner, safe='')}/"
        f"{urllib.parse.quote(repository, safe='')}"
    )


def readme_api_url(
    full_name: str,
) -> str:
    return repository_api_url(full_name) + "/readme"


# =============================================================================
# GITHUB API HELPERS
# =============================================================================

def build_headers(
    token: str | None,
    accept: str = "application/vnd.github+json",
) -> dict[str, str]:
    headers = {
        "Accept": accept,
        "User-Agent": f"NCRC-GitHub-Intel/{SCRIPT_VERSION}",
        "X-GitHub-Api-Version": "2022-11-28",
    }

    if token:
        headers["Authorization"] = f"Bearer {token}"

    return headers


def parse_rate_headers(
    headers: Any,
) -> dict[str, Any]:
    result: dict[str, Any] = {}

    mappings = {
        "limit": "X-RateLimit-Limit",
        "remaining": "X-RateLimit-Remaining",
        "used": "X-RateLimit-Used",
        "reset": "X-RateLimit-Reset",
        "resource": "X-RateLimit-Resource",
    }

    for key, header_name in mappings.items():
        value = headers.get(header_name)

        if value is None:
            continue

        if key in {
            "limit",
            "remaining",
            "used",
            "reset",
        }:
            try:
                result[key] = int(value)
            except (TypeError, ValueError):
                result[key] = value
        else:
            result[key] = value

    return result


def calculate_rate_wait(
    rate: dict[str, Any],
    retry_after: str | None,
    attempt: int,
) -> int:
    if retry_after:
        try:
            return max(
                1,
                int(retry_after),
            )
        except (TypeError, ValueError):
            pass

    reset = rate.get("reset")

    if reset:
        try:
            now = int(time.time())

            return max(
                1,
                int(reset) - now + 2,
            )
        except (TypeError, ValueError):
            pass

    return min(
        60 * (attempt + 1),
        300,
    )


def github_request(
    url: str,
    token: str | None,
    accept: str,
    timeout: int,
    retries: int,
) -> tuple[
    bytes,
    dict[str, str],
    dict[str, Any],
]:
    last_error: Exception | None = None

    for attempt in range(retries + 1):
        headers = build_headers(
            token=token,
            accept=accept,
        )

        request = urllib.request.Request(
            url=url,
            headers=headers,
            method="GET",
        )

        try:
            with urllib.request.urlopen(
                request,
                timeout=timeout,
            ) as response:
                body = response.read()

                response_headers = {
                    key: value
                    for key, value
                    in response.headers.items()
                }

                rate = parse_rate_headers(
                    response.headers
                )

                LOG.debug(
                    "GitHub request succeeded: %s "
                    "(remaining=%s resource=%s)",
                    url,
                    rate.get("remaining", "?"),
                    rate.get("resource", "?"),
                )

                return (
                    body,
                    response_headers,
                    rate,
                )

        except urllib.error.HTTPError as exc:
            last_error = exc

            rate = parse_rate_headers(
                exc.headers
            )

            try:
                error_body = exc.read().decode(
                    "utf-8",
                    errors="replace",
                )
            except Exception:
                error_body = ""

            error_message = ""

            if error_body:
                try:
                    error_payload = json.loads(
                        error_body
                    )

                    error_message = clean_text(
                        error_payload.get(
                            "message",
                            error_body,
                        )
                    )
                except Exception:
                    error_message = clean_text(
                        error_body
                    )

            if exc.code == 404:
                raise FileNotFoundError(
                    f"GitHub resource not found: {url}"
                ) from exc

            if exc.code in {403, 429}:
                if attempt >= retries:
                    raise RuntimeError(
                        "GitHub rate limit or retry budget exhausted"
                    ) from exc

                wait_for = calculate_rate_wait(
                    rate=rate,
                    retry_after=exc.headers.get(
                        "Retry-After"
                    ),
                    attempt=attempt,
                )

                LOG.warning(
                    "GitHub HTTP %d for %s: %s",
                    exc.code,
                    url,
                    error_message
                    or "rate limit or secondary limit",
                )

                LOG.warning(
                    "Waiting %d seconds before retry "
                    "(attempt %d/%d)",
                    wait_for,
                    attempt + 1,
                    retries,
                )

                time.sleep(wait_for)

                LOG.info(
                    "Retrying GitHub request after wait"
                )

                continue

            if 500 <= exc.code < 600:
                if attempt >= retries:
                    raise

                wait_for = min(
                    2 ** attempt,
                    30,
                )

                LOG.warning(
                    "GitHub server error HTTP %d; "
                    "retrying in %d seconds",
                    exc.code,
                    wait_for,
                )

                time.sleep(wait_for)
                continue

            raise RuntimeError(
                f"GitHub HTTP {exc.code}: "
                f"{error_message or url}"
            ) from exc

        except (
            urllib.error.URLError,
            TimeoutError,
        ) as exc:
            last_error = exc

            if attempt >= retries:
                raise

            wait_for = min(
                2 ** attempt,
                30,
            )

            LOG.warning(
                "GitHub request failed: %s; "
                "retrying in %d seconds",
                exc,
                wait_for,
            )

            time.sleep(wait_for)

    if last_error:
        raise last_error

    raise RuntimeError(
        "GitHub request failed without an exception"
    )


def github_get_json(
    url: str,
    token: str | None,
    timeout: int,
    retries: int,
) -> tuple[
    dict[str, Any],
    dict[str, Any],
]:
    body, _, rate = github_request(
        url=url,
        token=token,
        accept="application/vnd.github+json",
        timeout=timeout,
        retries=retries,
    )

    try:
        payload = json.loads(
            body.decode(
                "utf-8",
                errors="replace",
            )
        )
    except json.JSONDecodeError as exc:
        raise RuntimeError(
            f"Invalid JSON returned by GitHub for {url}"
        ) from exc

    if not isinstance(payload, dict):
        raise RuntimeError(
            f"Unexpected GitHub JSON response for {url}"
        )

    return payload, rate


def github_get_raw_text(
    url: str,
    token: str | None,
    timeout: int,
    retries: int,
) -> tuple[
    str,
    dict[str, str],
    dict[str, Any],
]:
    body, headers, rate = github_request(
        url=url,
        token=token,
        accept="application/vnd.github.raw+json",
        timeout=timeout,
        retries=retries,
    )

    content_type = headers.get(
        "Content-Type",
        "",
    )

    charset = "utf-8"

    match = re.search(
        r"charset=([A-Za-z0-9._-]+)",
        content_type,
        flags=re.IGNORECASE,
    )

    if match:
        charset = match.group(1)

    try:
        text = body.decode(
            charset,
            errors="replace",
        )
    except LookupError:
        text = body.decode(
            "utf-8",
            errors="replace",
        )

    return (
        normalize_multiline_text(text),
        headers,
        rate,
    )


# =============================================================================
# INPUT SELECTION
# =============================================================================

def get_probe_repositories(
    payload: Any,
) -> list[dict[str, Any]]:
    if not isinstance(payload, dict):
        raise ValueError(
            "Probe input must be a JSON object"
        )

    repositories = payload.get(
        "repositories"
    )

    if not isinstance(repositories, list):
        raise ValueError(
            "Probe input must contain a 'repositories' list"
        )

    return [
        repository
        for repository in repositories
        if isinstance(repository, dict)
    ]


def repository_relevance_level(
    repository: dict[str, Any],
) -> str:
    relevance = repository.get(
        "relevance"
    )

    if not isinstance(relevance, dict):
        return ""

    return clean_text(
        relevance.get("level")
    ).lower()


def repository_relevance_score(
    repository: dict[str, Any],
) -> float:
    relevance = repository.get(
        "relevance"
    )

    if not isinstance(relevance, dict):
        return 0.0

    return safe_float(
        relevance.get("score")
    )


def select_repositories(
    repositories: list[dict[str, Any]],
    levels: set[str],
    limit: int | None,
) -> list[dict[str, Any]]:
    selected: list[dict[str, Any]] = []

    seen: set[str] = set()

    for repository in repositories:
        level = repository_relevance_level(
            repository
        )

        if level not in levels:
            continue

        full_name = normalize_full_name(
            repository.get("full_name")
            or repository.get("id")
            or repository.get("url")
        )

        if not full_name:
            LOG.warning(
                "Skipping repository without valid full_name: %s",
                repository.get("name")
                or repository.get("id")
                or "unknown",
            )
            continue

        key = full_name.lower()

        if key in seen:
            continue

        repository = dict(repository)
        repository["full_name"] = full_name

        selected.append(repository)
        seen.add(key)

    selected.sort(
        key=lambda repository: (
            repository_relevance_score(
                repository
            ),
            safe_int(
                repository.get("stars")
            ),
            repository.get(
                "full_name",
                "",
            ).lower(),
        ),
        reverse=True,
    )

    if limit is not None:
        selected = selected[:limit]

    return selected


# =============================================================================
# CHECKPOINT HANDLING
# =============================================================================

def checkpoint_template(
    input_file: Path,
    selected_count: int,
    levels: list[str],
) -> dict[str, Any]:
    return {
        "schema_version": SCHEMA_VERSION,
        "script_version": SCRIPT_VERSION,
        "created_at": utc_now(),
        "updated_at": utc_now(),
        "input_file": str(input_file),
        "selected_count": selected_count,
        "selected_levels": levels,
        "completed": {},
        "errors": {},
    }


def load_checkpoint(
    path: Path,
) -> dict[str, Any]:
    payload = load_json(path)

    if not isinstance(payload, dict):
        raise ValueError(
            f"Checkpoint must be a JSON object: {path}"
        )

    completed = payload.get(
        "completed"
    )

    if not isinstance(completed, dict):
        payload["completed"] = {}

    errors = payload.get(
        "errors"
    )

    if not isinstance(errors, dict):
        payload["errors"] = {}

    return payload


def save_checkpoint(
    path: Path,
    checkpoint: dict[str, Any],
) -> None:
    checkpoint["updated_at"] = utc_now()

    atomic_write_json(
        path,
        checkpoint,
    )


# =============================================================================
# METADATA NORMALIZATION
# =============================================================================

def normalize_topics(
    value: Any,
) -> list[str]:
    if not isinstance(value, list):
        return []

    topics = {
        clean_text(topic).lower()
        for topic in value
        if clean_text(topic)
    }

    return sorted(topics)

def normalize_repository_metadata(
    metadata: dict[str, Any],
) -> dict[str, Any]:
    owner = metadata.get("owner")

    if not isinstance(owner, dict):
        owner = {}

    license_data = metadata.get("license")

    if not isinstance(license_data, dict):
        license_data = {}

    full_name = normalize_full_name(
        metadata.get("full_name")
    )

    return {
        "github_id": safe_int(
            metadata.get("id")
        ),
        "node_id": clean_text(
            metadata.get("node_id")
        ) or None,
        "full_name": full_name,
        "name": clean_text(
            metadata.get("name")
        ),
        "owner": clean_text(
            owner.get("login")
        ),
        "owner_type": clean_text(
            owner.get("type")
        ) or None,
        "url": clean_text(
            metadata.get("html_url")
        ),
        "api_url": clean_text(
            metadata.get("url")
        ),
        "description": clean_text(
            metadata.get("description")
        ),
        "homepage": clean_text(
            metadata.get("homepage")
        ) or None,
        "topics": normalize_topics(
            metadata.get("topics")
        ),
        "language": clean_text(
            metadata.get("language")
        ) or None,
        "default_branch": clean_text(
            metadata.get("default_branch")
        ) or None,
        "stars": safe_int(
            metadata.get("stargazers_count")
        ),
        "forks": safe_int(
            metadata.get("forks_count")
        ),
        "watchers": safe_int(
            metadata.get("subscribers_count")
            or metadata.get("watchers_count")
        ),
        "open_issues": safe_int(
            metadata.get("open_issues_count")
        ),
        "size_kb": safe_int(
            metadata.get("size")
        ),
        "fork": bool(
            metadata.get("fork")
        ),
        "archived": bool(
            metadata.get("archived")
        ),
        "disabled": bool(
            metadata.get("disabled")
        ),
        "is_template": bool(
            metadata.get("is_template")
        ),
        "has_issues": bool(
            metadata.get("has_issues")
        ),
        "has_projects": bool(
            metadata.get("has_projects")
        ),
        "has_wiki": bool(
            metadata.get("has_wiki")
        ),
        "has_pages": bool(
            metadata.get("has_pages")
        ),
        "has_discussions": bool(
            metadata.get("has_discussions")
        ),
        "license": (
            clean_text(
                license_data.get("spdx_id")
            )
            or clean_text(
                license_data.get("name")
            )
            or None
        ),
        "created_at": metadata.get(
            "created_at"
        ),
        "updated_at": metadata.get(
            "updated_at"
        ),
        "pushed_at": metadata.get(
            "pushed_at"
        ),
    }


def truncate_readme(
    text: str,
    maximum_chars: int,
) -> tuple[str, bool]:
    if maximum_chars <= 0:
        return "", bool(text)

    if len(text) <= maximum_chars:
        return text, False

    suffix = (
        "\n\n"
        "[README truncated by github_intel.py]"
    )

    allowed = max(
        0,
        maximum_chars - len(suffix),
    )

    return text[:allowed] + suffix, True


def readme_title(
    readme: str,
) -> str | None:
    for line in readme.splitlines():
        stripped = line.strip()

        if stripped.startswith("#"):
            title = stripped.lstrip(
                "#"
            ).strip()

            if title:
                return clean_text(title)

    return None


def combine_search_text(
    metadata: dict[str, Any],
    readme: str,
) -> str:
    values = [
        metadata.get("full_name"),
        metadata.get("name"),
        metadata.get("description"),
        " ".join(
            metadata.get("topics") or []
        ),
        readme,
    ]

    return "\n".join(
        str(value)
        for value in values
        if value
    )


def build_record(
    probe_repository: dict[str, Any],
    metadata: dict[str, Any],
    readme: str,
    readme_available: bool,
    readme_truncated: bool,
    readme_error: str | None,
    retrieved_at: str,
) -> dict[str, Any]:
    normalized = normalize_repository_metadata(
        metadata
    )

    full_name = (
        normalized.get("full_name")
        or normalize_full_name(
            probe_repository.get(
                "full_name"
            )
        )
    )

    if not full_name:
        raise ValueError(
            "Unable to determine normalized repository full_name"
        )

    description = (
        normalized.get("description")
        or clean_text(
            probe_repository.get(
                "description"
            )
        )
    )

    topics = sorted(
        set(
            normalized.get("topics") or []
        )
        | set(
            normalize_topics(
                probe_repository.get(
                    "topics"
                )
            )
        )
    )

    normalized["topics"] = topics
    normalized["description"] = description

    relevance = probe_repository.get(
        "relevance"
    )

    if not isinstance(relevance, dict):
        relevance = {
            "score": 0.0,
            "level": "unknown",
            "reasons": [],
        }

    classification = probe_repository.get(
        "classification"
    )

    if not isinstance(classification, dict):
        classification = {}

    discovery = probe_repository.get(
        "discovery"
    )

    if not isinstance(discovery, dict):
        discovery = {}

    search_text = combine_search_text(
        metadata=normalized,
        readme=readme,
    )

    repository_name = (
        normalized.get("name")
        or full_name.split(
            "/",
            1,
        )[1]
    )

    display_title = (
        readme_title(readme)
        or repository_name
    )

    return {
        "id": f"github:{full_name.lower()}",
        "evidence_type": "repository",
        "record_type": "repository",

        "title": display_title,
        "full_name": full_name,
        "name": repository_name,
        "owner": normalized.get("owner"),

        "url": (
            normalized.get("url")
            or f"https://github.com/{full_name}"
        ),

        "description": description,
        "readme": readme,

        "topics": topics,
        "language": normalized.get(
            "language"
        ),
        "license": normalized.get(
            "license"
        ),

        "stars": normalized.get(
            "stars",
            0,
        ),
        "forks": normalized.get(
            "forks",
            0,
        ),
        "watchers": normalized.get(
            "watchers",
            0,
        ),
        "open_issues": normalized.get(
            "open_issues",
            0,
        ),
        "size_kb": normalized.get(
            "size_kb",
            0,
        ),

        "fork": normalized.get(
            "fork",
            False,
        ),
        "archived": normalized.get(
            "archived",
            False,
        ),
        "disabled": normalized.get(
            "disabled",
            False,
        ),

        "default_branch": normalized.get(
            "default_branch"
        ),
        "homepage": normalized.get(
            "homepage"
        ),

        "created_at": normalized.get(
            "created_at"
        ),
        "updated_at": normalized.get(
            "updated_at"
        ),
        "pushed_at": normalized.get(
            "pushed_at"
        ),

        "classification": classification,
        "relevance": relevance,
        "discovery": discovery,

        "readme_metadata": {
            "available": readme_available,
            "characters": len(readme),
            "truncated": readme_truncated,
            "error": readme_error,
            "content_hash": (
                stable_hash(readme)
                if readme
                else None
            ),
        },

        "repository_metadata": {
            "github_id": normalized.get(
                "github_id"
            ),
            "node_id": normalized.get(
                "node_id"
            ),
            "owner_type": normalized.get(
                "owner_type"
            ),
            "api_url": normalized.get(
                "api_url"
            ),
            "is_template": normalized.get(
                "is_template",
                False,
            ),
            "has_issues": normalized.get(
                "has_issues",
                False,
            ),
            "has_projects": normalized.get(
                "has_projects",
                False,
            ),
            "has_wiki": normalized.get(
                "has_wiki",
                False,
            ),
            "has_pages": normalized.get(
                "has_pages",
                False,
            ),
            "has_discussions": normalized.get(
                "has_discussions",
                False,
            ),
        },

        "source": {
            "name": "GitHub",
            "type": "public_repository",
            "retrieved_at": retrieved_at,
        },

        "search_text_hash": stable_hash(
            search_text
        ),
    }


# =============================================================================
# REPOSITORY PROCESSING
# =============================================================================

def process_repository(
    probe_repository: dict[str, Any],
    token: str | None,
    timeout: int,
    retries: int,
    readme_max_chars: int,
) -> dict[str, Any]:
    full_name = normalize_full_name(
        probe_repository.get(
            "full_name"
        )
    )

    if not full_name:
        raise ValueError(
            "Probe repository has no valid full_name"
        )

    LOG.info(
        "Fetching metadata: %s",
        full_name,
    )

    metadata, _ = github_get_json(
        url=repository_api_url(
            full_name
        ),
        token=token,
        timeout=timeout,
        retries=retries,
    )

    readme = ""
    readme_available = False
    readme_truncated = False
    readme_error: str | None = None

    try:
        LOG.info(
            "Fetching README: %s",
            full_name,
        )

        raw_readme, _, _ = github_get_raw_text(
            url=readme_api_url(
                full_name
            ),
            token=token,
            timeout=timeout,
            retries=retries,
        )

        readme, readme_truncated = (
            truncate_readme(
                text=raw_readme,
                maximum_chars=readme_max_chars,
            )
        )

        readme_available = bool(readme)

    except FileNotFoundError:
        readme_error = "README not found"

        LOG.info(
            "README not found: %s",
            full_name,
        )

    except Exception as exc:
        readme_error = clean_text(exc)

        LOG.warning(
            "README retrieval failed for %s: %s",
            full_name,
            exc,
        )

    return build_record(
        probe_repository=probe_repository,
        metadata=metadata,
        readme=readme,
        readme_available=readme_available,
        readme_truncated=readme_truncated,
        readme_error=readme_error,
        retrieved_at=utc_now(),
    )


# =============================================================================
# OUTPUT BUILDING
# =============================================================================

def output_statistics(
    records: list[dict[str, Any]],
    selected_count: int,
    errors: dict[str, Any],
) -> dict[str, Any]:
    relevance_levels = Counter(
        clean_text(
            (
                record.get("relevance")
                or {}
            ).get("level")
        ).lower()
        for record in records
    )

    languages = Counter(
        clean_text(
            record.get("language")
        )
        for record in records
        if clean_text(
            record.get("language")
        )
    )

    classifications = Counter()

    for record in records:
        classification = record.get(
            "classification"
        )

        if not isinstance(
            classification,
            dict,
        ):
            continue

        if classification.get(
            "automotive"
        ):
            classifications[
                "automotive"
            ] += 1

        if classification.get(
            "ev_charging"
        ):
            classifications[
                "ev_charging"
            ] += 1

        if (
            classification.get(
                "automotive"
            )
            and classification.get(
                "ev_charging"
            )
        ):
            classifications[
                "automotive_and_ev_charging"
            ] += 1

        if classification.get(
            "security_related"
        ):
            classifications[
                "security_related"
            ] += 1

    return {
        "selected_from_probe": selected_count,
        "records_written": len(records),
        "processing_errors": len(errors),

        "readmes_available": sum(
            1
            for record in records
            if (
                record.get(
                    "readme_metadata"
                )
                or {}
            ).get(
                "available"
            )
        ),

        "readmes_missing": sum(
            1
            for record in records
            if not (
                record.get(
                    "readme_metadata"
                )
                or {}
            ).get(
                "available"
            )
        ),

        "readmes_truncated": sum(
            1
            for record in records
            if (
                record.get(
                    "readme_metadata"
                )
                or {}
            ).get(
                "truncated"
            )
        ),

        "repositories_archived": sum(
            1
            for record in records
            if record.get(
                "archived"
            )
        ),

        "repositories_forks": sum(
            1
            for record in records
            if record.get(
                "fork"
            )
        ),

        "classification": {
            "automotive": classifications.get(
                "automotive",
                0,
            ),
            "ev_charging": classifications.get(
                "ev_charging",
                0,
            ),
            "automotive_and_ev_charging": (
                classifications.get(
                    "automotive_and_ev_charging",
                    0,
                )
            ),
            "security_related": (
                classifications.get(
                    "security_related",
                    0,
                )
            ),
        },

        "relevance_levels": {
            "direct": relevance_levels.get(
                "direct",
                0,
            ),
            "strong": relevance_levels.get(
                "strong",
                0,
            ),
            "contextual": relevance_levels.get(
                "contextual",
                0,
            ),
            "low": relevance_levels.get(
                "low",
                0,
            ),
        },

        "top_languages": [
            {
                "language": language,
                "count": count,
            }
            for language, count
            in languages.most_common(20)
        ],
    }


def build_output(
    input_path: Path,
    levels: list[str],
    selected_count: int,
    completed: dict[str, Any],
    errors: dict[str, Any],
    started_at: str,
) -> dict[str, Any]:
    records = [
        record
        for record in completed.values()
        if isinstance(record, dict)
    ]

    records.sort(
        key=lambda record: (
            safe_float(
                (
                    record.get("relevance")
                    or {}
                ).get("score")
            ),
            safe_int(
                record.get("stars")
            ),
            clean_text(
                record.get("full_name")
            ).lower(),
        ),
        reverse=True,
    )

    return {
        "schema_version": SCHEMA_VERSION,
        "script_version": SCRIPT_VERSION,
        "generated_at": utc_now(),
        "collection_started_at": started_at,

        "description": (
            "Normalized GitHub repository evidence for the "
            "NCRC Automotive Cybersecurity Threat Register."
        ),

        "source_input": str(input_path),

        "selection": {
            "levels": levels,
            "selected_count": selected_count,
        },

        "statistics": output_statistics(
            records=records,
            selected_count=selected_count,
            errors=errors,
        ),

        "errors": errors,

        "repositories": records,
    }


# =============================================================================
# VALIDATION
# =============================================================================

def validate_probe(
    repositories: list[dict[str, Any]],
) -> list[str]:
    problems: list[str] = []
    seen: set[str] = set()

    for index, repository in enumerate(
        repositories
    ):
        full_name = normalize_full_name(
            repository.get("full_name")
            or repository.get("id")
            or repository.get("url")
        )

        if not full_name:
            problems.append(
                f"Repository row {index} has no valid full_name"
            )
            continue

        key = full_name.lower()

        if key in seen:
            problems.append(
                f"Duplicate repository identity: {full_name}"
            )

        seen.add(key)

        relevance = repository.get(
            "relevance"
        )

        if not isinstance(
            relevance,
            dict,
        ):
            problems.append(
                f"{full_name}: missing relevance object"
            )
            continue

        level = clean_text(
            relevance.get("level")
        ).lower()

        if level not in {
            "direct",
            "strong",
            "contextual",
            "low",
        }:
            problems.append(
                f"{full_name}: invalid relevance level {level!r}"
            )

    return problems


# =============================================================================
# CLI
# =============================================================================

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description=(
            "Build github/github.json from direct and strong "
            "repositories discovered by github_probe.py."
        )
    )

    parser.add_argument(
        "--input",
        type=Path,
        default=DEFAULT_INPUT,
        help=(
            "Input probe JSON "
            "(default: github_probe.json)"
        ),
    )

    parser.add_argument(
        "--output",
        type=Path,
        default=DEFAULT_OUTPUT,
        help=(
            "Final output JSON "
            "(default: github/github.json)"
        ),
    )

    parser.add_argument(
        "--checkpoint",
        type=Path,
        default=DEFAULT_CHECKPOINT,
        help=(
            "Checkpoint JSON "
            "(default: github/github_checkpoint.json)"
        ),
    )

    parser.add_argument(
        "--levels",
        nargs="+",
        default=DEFAULT_LEVELS,
        choices=[
            "direct",
            "strong",
            "contextual",
            "low",
        ],
        help=(
            "Relevance levels to include "
            "(default: direct strong)"
        ),
    )

    parser.add_argument(
        "--limit",
        type=int,
        default=None,
        help=(
            "Optional maximum number of repositories "
            "for testing"
        ),
    )

    parser.add_argument(
        "--readme-max-chars",
        type=int,
        default=DEFAULT_README_MAX_CHARS,
        help=(
            "Maximum README characters stored per repository "
            f"(default: {DEFAULT_README_MAX_CHARS})"
        ),
    )

    parser.add_argument(
        "--checkpoint-interval",
        type=int,
        default=DEFAULT_CHECKPOINT_INTERVAL,
        help=(
            "Save checkpoint after this many newly processed "
            f"repositories (default: {DEFAULT_CHECKPOINT_INTERVAL})"
        ),
    )

    parser.add_argument(
        "--request-delay",
        type=float,
        default=DEFAULT_REQUEST_DELAY,
        help=(
            "Delay between repositories in seconds "
            f"(default: {DEFAULT_REQUEST_DELAY})"
        ),
    )

    parser.add_argument(
        "--timeout",
        type=int,
        default=DEFAULT_TIMEOUT,
        help=(
            "HTTP timeout in seconds "
            f"(default: {DEFAULT_TIMEOUT})"
        ),
    )

    parser.add_argument(
        "--retries",
        type=int,
        default=DEFAULT_RETRIES,
        help=(
            "Maximum retries for API or rate-limit responses "
            f"(default: {DEFAULT_RETRIES})"
        ),
    )

    parser.add_argument(
        "--token",
        default=None,
        help=(
            "GitHub token. Prefer the GITHUB_TOKEN "
            "environment variable."
        ),
    )

    parser.add_argument(
        "--resume",
        action="store_true",
        help=(
            "Resume from an existing checkpoint"
        ),
    )

    parser.add_argument(
        "--force",
        action="store_true",
        help=(
            "Ignore and overwrite existing checkpoint/output"
        ),
    )

    parser.add_argument(
        "--validate-only",
        action="store_true",
        help=(
            "Validate input and selection without API retrieval"
        ),
    )

    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        help=(
            "Increase logging verbosity. "
            "Use -vv for debug output."
        ),
    )

    return parser.parse_args()


# =============================================================================
# MAIN
# =============================================================================

def main() -> int:
    args = parse_args()

    configure_logging(
        args.verbose
    )

    if not args.input.exists():
        LOG.error(
            "Input file not found: %s",
            args.input,
        )
        return 2

    if (
        args.checkpoint_interval < 1
    ):
        raise ValueError(
            "--checkpoint-interval must be at least 1"
        )

    if args.retries < 0:
        raise ValueError(
            "--retries cannot be negative"
        )

    if args.timeout < 1:
        raise ValueError(
            "--timeout must be at least 1"
        )

    if args.readme_max_chars < 0:
        raise ValueError(
            "--readme-max-chars cannot be negative"
        )

    probe_payload = load_json(
        args.input
    )

    probe_repositories = (
        get_probe_repositories(
            probe_payload
        )
    )

    validation_problems = validate_probe(
        probe_repositories
    )

    if validation_problems:
        LOG.warning(
            "Probe validation found %d issue(s)",
            len(validation_problems),
        )

        for problem in validation_problems[:50]:
            LOG.warning(
                "Validation: %s",
                problem,
            )

    levels = [
        level.lower()
        for level in args.levels
    ]

    selected = select_repositories(
        repositories=probe_repositories,
        levels=set(levels),
        limit=args.limit,
    )

    LOG.info(
        "Probe repositories loaded: %d",
        len(probe_repositories),
    )

    LOG.info(
        "Selected relevance levels: %s",
        ", ".join(levels),
    )

    LOG.info(
        "Repositories selected: %d",
        len(selected),
    )

    if args.validate_only:
        print()
        print("GitHub intelligence validation")
        print("==============================")
        print()
        print(
            f"Probe repositories          : "
            f"{len(probe_repositories)}"
        )
        print(
            f"Selected repositories       : "
            f"{len(selected)}"
        )
        print(
            f"Selected levels             : "
            f"{', '.join(levels)}"
        )
        print(
            f"Validation issues           : "
            f"{len(validation_problems)}"
        )
        print()
        print(
            "Validation completed; no API requests "
            "or output writes performed."
        )

        return 0

    token = (
        args.token
        or os.environ.get(
            "GITHUB_TOKEN"
        )
        or None
    )

    if token:
        LOG.info(
            "Using authenticated GitHub API requests"
        )
    else:
        LOG.warning(
            "No GitHub token configured; "
            "using unauthenticated requests"
        )

    if args.force:
        if args.checkpoint.exists():
            args.checkpoint.unlink()

        if args.output.exists():
            args.output.unlink()

    if (
        args.output.exists()
        and not args.resume
        and not args.force
    ):
        LOG.error(
            "Output already exists: %s. "
            "Use --force to rebuild or --resume "
            "with a checkpoint.",
            args.output,
        )
        return 2

    started_at = utc_now()

    if args.resume:
        if not args.checkpoint.exists():
            LOG.error(
                "Cannot resume; checkpoint not found: %s",
                args.checkpoint,
            )
            return 2

        checkpoint = load_checkpoint(
            args.checkpoint
        )

        LOG.info(
            "Resuming checkpoint with %d completed records",
            len(
                checkpoint.get(
                    "completed",
                    {},
                )
            ),
        )

        started_at = checkpoint.get(
            "created_at",
            started_at,
        )

    else:
        checkpoint = checkpoint_template(
            input_file=args.input,
            selected_count=len(selected),
            levels=levels,
        )

    completed = checkpoint.setdefault(
        "completed",
        {},
    )

    errors = checkpoint.setdefault(
        "errors",
        {},
    )

    selected_keys = {
        normalize_full_name(
            repository.get(
                "full_name"
            )
        ).lower()
        for repository in selected
        if normalize_full_name(
            repository.get(
                "full_name"
            )
        )
    }

    stale_completed = [
        key
        for key in completed
        if key not in selected_keys
    ]

    for key in stale_completed:
        LOG.warning(
            "Removing stale checkpoint record: %s",
            key,
        )
        completed.pop(
            key,
            None,
        )

    pending = [
        repository
        for repository in selected
        if normalize_full_name(
            repository.get(
                "full_name"
            )
        ).lower()
        not in completed
    ]

    LOG.info(
        "Already completed: %d",
        len(completed),
    )

    LOG.info(
        "Pending repositories: %d",
        len(pending),
    )

    newly_processed = 0

    for index, repository in enumerate(
        pending,
        start=1,
    ):
        full_name = normalize_full_name(
            repository.get(
                "full_name"
            )
        )

        key = full_name.lower()

        LOG.info(
            "[%d/%d] Processing %s",
            index,
            len(pending),
            full_name,
        )

        try:
            record = process_repository(
                probe_repository=repository,
                token=token,
                timeout=args.timeout,
                retries=args.retries,
                readme_max_chars=(
                    args.readme_max_chars
                ),
            )

            completed[key] = record
            errors.pop(
                key,
                None,
            )

            newly_processed += 1

        except FileNotFoundError as exc:
            message = clean_text(exc)

            errors[key] = {
                "full_name": full_name,
                "error": message,
                "recorded_at": utc_now(),
                "fatal": True,
            }

            LOG.warning(
                "Repository unavailable: %s",
                full_name,
            )

        except Exception as exc:
            message = clean_text(exc)

            errors[key] = {
                "full_name": full_name,
                "error": message,
                "recorded_at": utc_now(),
                "fatal": False,
            }

            LOG.exception(
                "Repository processing failed for %s",
                full_name,
            )

        if (
            newly_processed > 0
            and newly_processed
            % args.checkpoint_interval
            == 0
        ):
            save_checkpoint(
                args.checkpoint,
                checkpoint,
            )

            LOG.info(
                "Checkpoint saved: %d completed, "
                "%d errors",
                len(completed),
                len(errors),
            )

        if args.request_delay > 0:
            time.sleep(
                args.request_delay
            )

    save_checkpoint(
        args.checkpoint,
        checkpoint,
    )

    output = build_output(
        input_path=args.input,
        levels=levels,
        selected_count=len(selected),
        completed=completed,
        errors=errors,
        started_at=started_at,
    )

    atomic_write_json(
        args.output,
        output,
    )

    statistics = output[
        "statistics"
    ]

    print()
    print("GitHub intelligence dataset")
    print("===========================")
    print()

    print(
        f"Selected from probe        : "
        f"{statistics['selected_from_probe']}"
    )

    print(
        f"Repositories written       : "
        f"{statistics['records_written']}"
    )

    print(
        f"Processing errors          : "
        f"{statistics['processing_errors']}"
    )

    print()

    print(
        f"README available           : "
        f"{statistics['readmes_available']}"
    )

    print(
        f"README missing             : "
        f"{statistics['readmes_missing']}"
    )

    print(
        f"README truncated           : "
        f"{statistics['readmes_truncated']}"
    )

    print()

    print(
        f"Direct repositories        : "
        f"{statistics['relevance_levels']['direct']}"
    )

    print(
        f"Strong repositories        : "
        f"{statistics['relevance_levels']['strong']}"
    )

    print()

    print(
        f"Automotive                 : "
        f"{statistics['classification']['automotive']}"
    )

    print(
        f"EV charging                : "
        f"{statistics['classification']['ev_charging']}"
    )

    print(
        f"Automotive + EV charging   : "
        f"{statistics['classification']['automotive_and_ev_charging']}"
    )

    print()

    print(
        f"Output written to          : "
        f"{args.output}"
    )

    print(
        f"Checkpoint written to      : "
        f"{args.checkpoint}"
    )

    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(
            main()
        )
    except KeyboardInterrupt:
        print(
            "\nInterrupted. Resume with --resume.",
            file=sys.stderr,
        )
        raise SystemExit(130)
    except Exception as exc:
        LOG.exception(
            "Fatal error: %s",
            exc,
        )
        raise SystemExit(1)
