#!/usr/bin/env python3
"""
github_probe.py

Discovery probe for public GitHub repositories relevant to:

1. Automotive cybersecurity
2. EV charging cybersecurity

Purpose
-------
This script is intentionally lightweight.

It does NOT:
- download repository contents
- download README files
- run AI/LLM analysis
- modify an existing GitHub dataset
- correlate repositories against the Threat Register

It DOES:
- run targeted GitHub repository searches
- collect repository metadata
- deduplicate repositories by full_name
- classify repositories into discovery groups
- optionally compare them with an existing repository dataset
- write a JSON report for inspection

Authentication
--------------
Optionally provide a GitHub token:

    export GITHUB_TOKEN="..."

No token is embedded or written to output.

Examples
--------

Discovery probe:

    python3 github_probe.py

Verbose:

    python3 github_probe.py -v

Compare against an existing dataset:

    python3 github_probe.py \
        --existing-json github/github.json

Custom output:

    python3 github_probe.py \
        --output github_probe.json

Limit pages during testing:

    python3 github_probe.py \
        --max-pages 2 \
        -v
"""

from __future__ import annotations

import argparse
import json
import logging
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

from collections import Counter
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


SCRIPT_VERSION = "1.0.0"

GITHUB_API = "https://api.github.com"
SEARCH_ENDPOINT = f"{GITHUB_API}/search/repositories"

DEFAULT_OUTPUT = Path("github_probe.json")

LOG = logging.getLogger("github_probe")


# ---------------------------------------------------------------------------
# Search profiles
# ---------------------------------------------------------------------------

AUTOMOTIVE_QUERIES = [
    "automotive cybersecurity",
    "vehicle cybersecurity",
    "automotive security",
    "car hacking",
    "CAN bus security",
    "CAN bus hacking",
    "automotive CAN",
    "UDS automotive security",
    "ISO-TP automotive",
    "DoIP automotive",
    "automotive ECU security",
    "vehicle ECU hacking",
    "automotive diagnostics security",
    "connected vehicle security",
    "connected car security",
    "telematics security automotive",
    "automotive OTA security",
    "vehicle OTA security",
    "keyless entry security automotive",
    "automotive immobilizer security",
    "V2X security automotive",
    "ADAS security automotive",
    "autonomous vehicle security",
]


EV_CHARGING_QUERIES = [
    "EV charging security",
    "EV charger security",
    "electric vehicle charging security",
    "charging station security EV",
    "OCPP security",
    "OCPP charger",
    "OCPP EVSE",
    "EVSE security",
    "CSMS OCPP",
    "ISO 15118 security",
    "ISO 15118 EV",
    "Plug and Charge security",
    "V2G security",
    "vehicle-to-grid security",
]


# ---------------------------------------------------------------------------
# Classification vocabulary
#
# This does NOT determine whether a repository is good evidence.
# It only provides lightweight discovery labels for inspection.
# ---------------------------------------------------------------------------

AUTOMOTIVE_TERMS = {
    "automotive",
    "vehicle",
    "vehicles",
    "car",
    "cars",
    "can bus",
    "can-bus",
    "canbus",
    "can fd",
    "can-fd",
    "uds",
    "iso-tp",
    "isotp",
    "doip",
    "ecu",
    "ecus",
    "telematics",
    "connected vehicle",
    "connected car",
    "vehicle security",
    "automotive security",
    "automotive cybersecurity",
    "car hacking",
    "vehicle hacking",
    "keyless",
    "immobilizer",
    "v2x",
    "adas",
    "autonomous vehicle",
    "automotive ethernet",
    "obd",
    "obd2",
    "obd-ii",
}


EV_CHARGING_TERMS = {
    "ev charging",
    "ev charger",
    "ev chargers",
    "electric vehicle charging",
    "charging station",
    "charging stations",
    "charge point",
    "chargepoint",
    "evse",
    "ocpp",
    "csms",
    "iso 15118",
    "iso15118",
    "plug and charge",
    "plug-and-charge",
    "v2g",
    "vehicle-to-grid",
    "vehicle to grid",
    "charging infrastructure",
}


SECURITY_TERMS = {
    "security",
    "cybersecurity",
    "cyber security",
    "hacking",
    "hack",
    "pentest",
    "penetration testing",
    "vulnerability",
    "vulnerabilities",
    "exploit",
    "exploitation",
    "attack",
    "attacks",
    "threat",
    "threats",
    "reverse engineering",
    "fuzzing",
    "fuzzer",
    "forensics",
    "intrusion detection",
    "ids",
}


# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------

@dataclass
class RepositoryRecord:
    id: int
    full_name: str
    name: str
    owner: str
    html_url: str

    description: str = ""
    topics: list[str] = field(default_factory=list)

    language: str | None = None

    stars: int = 0
    forks: int = 0
    open_issues: int = 0
    watchers: int = 0

    archived: bool = False
    disabled: bool = False
    fork: bool = False

    created_at: str | None = None
    updated_at: str | None = None
    pushed_at: str | None = None

    license: str | None = None

    search_profiles: set[str] = field(default_factory=set)
    matched_queries: set[str] = field(default_factory=set)

    automotive_term_hits: list[str] = field(default_factory=list)
    ev_charging_term_hits: list[str] = field(default_factory=list)
    security_term_hits: list[str] = field(default_factory=list)

    automotive: bool = False
    ev_charging: bool = False
    security_related: bool = False

    def searchable_text(self) -> str:
        return " ".join(
            [
                self.full_name,
                self.name,
                self.description,
                " ".join(self.topics),
            ]
        ).lower()

    def classify(self) -> None:
        text = self.searchable_text()

        self.automotive_term_hits = sorted(
            term
            for term in AUTOMOTIVE_TERMS
            if term in text
        )

        self.ev_charging_term_hits = sorted(
            term
            for term in EV_CHARGING_TERMS
            if term in text
        )

        self.security_term_hits = sorted(
            term
            for term in SECURITY_TERMS
            if term in text
        )

        # Search origin also carries useful discovery information.
        automotive_origin = "automotive" in self.search_profiles
        charging_origin = "ev_charging" in self.search_profiles

        self.automotive = bool(
            self.automotive_term_hits
            or automotive_origin
        )

        self.ev_charging = bool(
            self.ev_charging_term_hits
            or charging_origin
        )

        self.security_related = bool(
            self.security_term_hits
        )

    def to_dict(self) -> dict[str, Any]:
        relevance = calculate_relevance(self)
        return {
            "id": f"github:{self.full_name.lower()}",
            "github_id": self.id,
            "full_name": self.full_name,
            "name": self.name,
            "owner": self.owner,
            "url": self.html_url,
            "description": self.description,
            "topics": sorted(self.topics),
            "language": self.language,
            "stars": self.stars,
            "forks": self.forks,
            "open_issues": self.open_issues,
            "watchers": self.watchers,
            "archived": self.archived,
            "disabled": self.disabled,
            "fork": self.fork,
            "license": self.license,
            "created_at": self.created_at,
            "updated_at": self.updated_at,
            "pushed_at": self.pushed_at,

            "discovery": {
                "search_profiles": sorted(self.search_profiles),
                "matched_queries": sorted(self.matched_queries),
            },

            "classification": {
                "automotive": self.automotive,
                "ev_charging": self.ev_charging,
                "security_related": self.security_related,
                "automotive_term_hits": self.automotive_term_hits,
                "ev_charging_term_hits": self.ev_charging_term_hits,
                "security_term_hits": self.security_term_hits,
            },
            "relevance": relevance,
        }   


# ---------------------------------------------------------------------------
# General helpers
# ---------------------------------------------------------------------------

def utc_now() -> str:
    return (
        datetime.now(timezone.utc)
        .replace(microsecond=0)
        .isoformat()
        .replace("+00:00", "Z")
    )


def configure_logging(verbose: int) -> None:
    if verbose >= 2:
        level = logging.DEBUG
    elif verbose == 1:
        level = logging.INFO
    else:
        level = logging.WARNING

    logging.basicConfig(
        level=level,
        format="%(asctime)s %(levelname)-8s %(message)s",
    )


def clean_text(value: Any) -> str:
    if value is None:
        return ""

    return " ".join(
        str(value)
        .replace("\r", " ")
        .replace("\n", " ")
        .split()
    )


def safe_int(value: Any) -> int:
    try:
        return int(value or 0)
    except (TypeError, ValueError):
        return 0


# ---------------------------------------------------------------------------
# GitHub API
# ---------------------------------------------------------------------------

def build_headers(token: str | None) -> dict[str, str]:
    headers = {
        "Accept": "application/vnd.github+json",
        "User-Agent": f"NCRC-GitHub-Probe/{SCRIPT_VERSION}",
        "X-GitHub-Api-Version": "2022-11-28",
    }

    if token:
        headers["Authorization"] = f"Bearer {token}"

    return headers


def parse_rate_headers(
    headers: Any,
) -> dict[str, Any]:
    result: dict[str, Any] = {}

    mappings = {
        "limit": "X-RateLimit-Limit",
        "remaining": "X-RateLimit-Remaining",
        "used": "X-RateLimit-Used",
        "reset": "X-RateLimit-Reset",
        "resource": "X-RateLimit-Resource",
    }

    for key, header_name in mappings.items():
        value = headers.get(header_name)
        if value is None:
            continue

        if key in {
            "limit",
            "remaining",
            "used",
            "reset",
        }:
            try:
                result[key] = int(value)
            except ValueError:
                result[key] = value
        else:
            result[key] = value

    return result


def github_get_json(
    url: str,
    headers: dict[str, str],
    timeout: int,
    retries: int,
) -> tuple[dict[str, Any], dict[str, Any]]:

    last_error: Exception | None = None

    for attempt in range(retries + 1):
        request = urllib.request.Request(
            url,
            headers=headers,
            method="GET",
        )

        try:
            with urllib.request.urlopen(
                request,
                timeout=timeout,
            ) as response:

                body = response.read().decode(
                    "utf-8",
                    errors="replace",
                )

                payload = json.loads(body)
                rate = parse_rate_headers(
                    response.headers
                )

                return payload, rate

        except urllib.error.HTTPError as exc:
            last_error = exc

            try:
                body = exc.read().decode(
                    "utf-8",
                    errors="replace",
                )
            except Exception:
                body = ""

            rate = parse_rate_headers(exc.headers)

            LOG.warning(
                "GitHub HTTP %s: %s",
                exc.code,
                url,
            )

            if body:
                try:
                    error_payload = json.loads(body)
                    LOG.warning(
                        "GitHub response: %s",
                        clean_text(
                            error_payload.get(
                                "message",
                                body,
                            )
                        ),
                    )
                except Exception:
                    LOG.debug(
                        "GitHub response body: %s",
                        clean_text(body),
                    )

            # Primary rate limit.

            if (
                exc.code in {403, 429}
                and rate.get("remaining") == 0
                and rate.get("reset")
            ):
                now = int(time.time())
                wait_for = max(
                    1,
                    int(rate["reset"]) - now + 2,
                )

                if attempt >= retries:
                    raise RuntimeError(
                        "GitHub rate limit exhausted "
                        "and retry budget was reached"
                    ) from exc

                LOG.warning(
                    "GitHub rate limit exhausted. "
                    "Waiting %d seconds until reset...",
                    wait_for,
                )

                time.sleep(wait_for)

                LOG.info(
                    "GitHub rate-limit window reset; retrying request"
                )

                continue




            # Retry-After / secondary limits.
            retry_after = exc.headers.get(
                "Retry-After"
            )

            if (
                exc.code in {403, 429}
                and attempt < retries
            ):
                if retry_after:
                    try:
                        wait_for = max(
                            1,
                            int(retry_after),
                        )
                    except ValueError:
                        wait_for = 60
                else:
                    wait_for = min(
                        60 * (attempt + 1),
                        180,
                    )

                LOG.warning(
                    "Retrying after %d seconds",
                    wait_for,
                )

                time.sleep(wait_for)
                continue

            raise

        except (
            urllib.error.URLError,
            TimeoutError,
            json.JSONDecodeError,
        ) as exc:
            last_error = exc

            if attempt >= retries:
                raise

            wait_for = 2 ** attempt

            LOG.warning(
                "Request failed (%s); retrying in %ds",
                exc,
                wait_for,
            )

            time.sleep(wait_for)

    if last_error:
        raise last_error

    raise RuntimeError(
        "GitHub request failed without an exception"
    )


def search_repositories(
    query: str,
    profile: str,
    headers: dict[str, str],
    per_page: int,
    max_pages: int,
    timeout: int,
    retries: int,
    request_delay: float,
) -> tuple[
    list[dict[str, Any]],
    dict[str, Any],
]:

    results: list[dict[str, Any]] = []

    metadata: dict[str, Any] = {
        "query": query,
        "profile": profile,
        "total_count": None,
        "incomplete_results": None,
        "pages_requested": 0,
        "items_received": 0,
        "rate": {},
        "error": None,
    }

    for page in range(1, max_pages + 1):
        params = {
            "q": query,
            "per_page": per_page,
            "page": page,
            "sort": "stars",
            "order": "desc",
        }

        url = (
            SEARCH_ENDPOINT
            + "?"
            + urllib.parse.urlencode(params)
        )

        LOG.info(
            "Searching %-11s page %-2d: %s",
            profile,
            page,
            query,
        )

        try:
            payload, rate = github_get_json(
                url=url,
                headers=headers,
                timeout=timeout,
                retries=retries,
            )

        except Exception as exc:
            metadata["error"] = str(exc)

            LOG.error(
                "Search failed for %r: %s",
                query,
                exc,
            )

            break

        metadata["pages_requested"] += 1
        metadata["rate"] = rate

        if metadata["total_count"] is None:
            metadata["total_count"] = payload.get(
                "total_count"
            )
            metadata["incomplete_results"] = (
                payload.get("incomplete_results")
            )

        items = payload.get("items", [])

        if not isinstance(items, list):
            LOG.warning(
                "Unexpected GitHub result for %r",
                query,
            )
            break

        results.extend(items)
        metadata["items_received"] += len(items)

        LOG.debug(
            "Query %r page %d returned %d items; "
            "rate remaining=%s",
            query,
            page,
            len(items),
            rate.get("remaining", "?"),
        )

        if len(items) < per_page:
            break

        if request_delay > 0:
            time.sleep(request_delay)

    return results, metadata


# ---------------------------------------------------------------------------
# Repository normalization
# ---------------------------------------------------------------------------

def repository_from_api(
    item: dict[str, Any],
    profile: str,
    query: str,
) -> RepositoryRecord:

    owner_obj = item.get("owner") or {}
    license_obj = item.get("license") or {}

    topics = item.get("topics") or []

    if not isinstance(topics, list):
        topics = []

    repo = RepositoryRecord(
        id=safe_int(item.get("id")),
        full_name=clean_text(
            item.get("full_name")
        ),
        name=clean_text(
            item.get("name")
        ),
        owner=clean_text(
            owner_obj.get("login")
        ),
        html_url=clean_text(
            item.get("html_url")
        ),
        description=clean_text(
            item.get("description")
        ),
        topics=[
            clean_text(topic).lower()
            for topic in topics
            if clean_text(topic)
        ],
        language=clean_text(
            item.get("language")
        ) or None,
        stars=safe_int(
            item.get("stargazers_count")
        ),
        forks=safe_int(
            item.get("forks_count")
        ),
        open_issues=safe_int(
            item.get("open_issues_count")
        ),
        watchers=safe_int(
            item.get("watchers_count")
        ),
        archived=bool(
            item.get("archived")
        ),
        disabled=bool(
            item.get("disabled")
        ),
        fork=bool(
            item.get("fork")
        ),
        created_at=item.get("created_at"),
        updated_at=item.get("updated_at"),
        pushed_at=item.get("pushed_at"),
        license=clean_text(
            license_obj.get("spdx_id")
        ) or None,
    )

    repo.search_profiles.add(profile)
    repo.matched_queries.add(query)

    repo.classify()

    return repo


def merge_repository(
    existing: RepositoryRecord,
    incoming: RepositoryRecord,
) -> None:

    existing.search_profiles.update(
        incoming.search_profiles
    )

    existing.matched_queries.update(
        incoming.matched_queries
    )

    # Prefer whichever description is more informative.
    if (
        len(incoming.description)
        > len(existing.description)
    ):
        existing.description = (
            incoming.description
        )

    existing.topics = sorted(
        set(existing.topics)
        | set(incoming.topics)
    )

    existing.stars = max(
        existing.stars,
        incoming.stars,
    )

    existing.forks = max(
        existing.forks,
        incoming.forks,
    )

    existing.open_issues = max(
        existing.open_issues,
        incoming.open_issues,
    )

    existing.watchers = max(
        existing.watchers,
        incoming.watchers,
    )

    if not existing.language:
        existing.language = incoming.language

    if not existing.license:
        existing.license = incoming.license

    existing.classify()


# ---------------------------------------------------------------------------
# Existing dataset comparison
# ---------------------------------------------------------------------------

def extract_existing_repo_names(
    payload: Any,
) -> set[str]:
    """
    Best-effort extraction of owner/repository names from an existing
    repository dataset.

    Supports common structures without assuming the schema of the
    older AI-generated dataset.
    """

    result: set[str] = set()

    def inspect(value: Any) -> None:
        if isinstance(value, list):
            for child in value:
                inspect(child)
            return

        if not isinstance(value, dict):
            return

        for field_name in (
            "full_name",
            "repo",
            "repository",
            "repository_name",
            "name_with_owner",
        ):
            candidate = value.get(field_name)

            if not isinstance(candidate, str):
                continue

            candidate = candidate.strip()

            if (
                candidate
                and "/" in candidate
                and not candidate.startswith(
                    "http"
                )
            ):
                result.add(
                    candidate.lower()
                )

        for field_name in (
            "url",
            "html_url",
            "github_url",
            "repository_url",
        ):
            candidate = value.get(field_name)

            if not isinstance(candidate, str):
                continue

            parsed = parse_github_repo_url(
                candidate
            )

            if parsed:
                result.add(parsed.lower())

        for child in value.values():
            if isinstance(
                child,
                (dict, list),
            ):
                inspect(child)

    inspect(payload)
    return result


def parse_github_repo_url(
    url: str,
) -> str | None:

    try:
        parsed = e.urlparse(url)
    except ValueError:
        return None

    if parsed.netloc.lower() not in {
        "github.com",
        "www.github.com",
    }:
        return None

    pieces = [
        piece
        for piece in parsed.path.split("/")
        if piece
    ]

    if len(pieces) < 2:
        return None

    owner = pieces[0]
    repo = pieces[1]

    if repo.endswith(".git"):
        repo = repo[:-4]

    if not owner or not repo:
        return None

    return f"{owner}/{repo}"


def load_existing_repositories(
    path: Path,
) -> set[str]:
    with path.open(
        "r",
        encoding="utf-8",
    ) as handle:
        payload = json.load(handle)

    names = extract_existing_repo_names(
        payload
    )

    LOG.info(
        "Loaded %d repository identities from %s",
        len(names),
        path,
    )

    return names


def calculate_relevance(
    repo: RepositoryRecord,
) -> dict[str, Any]:
    """
    Calculate a transparent discovery relevance score.

    This score estimates whether a discovered repository is worth
    deeper inspection. It is NOT a threat-correlation score.

    Levels:
        direct      >= 0.75
        strong      >= 0.55
        contextual  >= 0.30
        low         <  0.30
    """

    score = 0.0
    reasons: list[str] = []

    automotive_hits = len(
        repo.automotive_term_hits
    )
    charging_hits = len(
        repo.ev_charging_term_hits
    )
    security_hits = len(
        repo.security_term_hits
    )

    # ---------------------------------------------------------
    # Domain relevance
    # ---------------------------------------------------------

    if automotive_hits:
        contribution = min(
            0.28,
            automotive_hits * 0.07,
        )
        score += contribution
        reasons.append(
            f"automotive terminology ({automotive_hits} hits)"
        )

    if charging_hits:
        contribution = min(
            0.30,
            charging_hits * 0.08,
        )
        score += contribution
        reasons.append(
            f"EV charging terminology ({charging_hits} hits)"
        )

    # ---------------------------------------------------------
    # Explicit security relevance
    # ---------------------------------------------------------

    if security_hits:
        contribution = min(
            0.32,
            security_hits * 0.08,
        )
        score += contribution
        reasons.append(
            f"explicit security terminology ({security_hits} hits)"
        )

    # ---------------------------------------------------------
    # Search provenance
    #
    # Being discovered by several targeted searches is useful
    # evidence that the repository is not merely a lexical accident.
    # ---------------------------------------------------------

    matched_query_count = len(
        repo.matched_queries
    )

    if matched_query_count >= 2:
        contribution = min(
            0.12,
            matched_query_count * 0.025,
        )
        score += contribution
        reasons.append(
            f"matched {matched_query_count} targeted discovery queries"
        )

    # ---------------------------------------------------------
    # Cross-profile relevance
    # ---------------------------------------------------------

    if (
        "automotive" in repo.search_profiles
        and "ev_charging" in repo.search_profiles
    ):
        score += 0.08
        reasons.append(
            "discovered by both automotive and EV charging profiles"
        )

    # ---------------------------------------------------------
    # GitHub topics
    # ---------------------------------------------------------

    relevant_topic_hits = sorted(
        {
            topic
            for topic in repo.topics
            if (
                any(
                    term in topic
                    for term in AUTOMOTIVE_TERMS
                )
                or any(
                    term in topic
                    for term in EV_CHARGING_TERMS
                )
                or any(
                    term in topic
                    for term in SECURITY_TERMS
                )
            )
        }
    )

    if relevant_topic_hits:
        contribution = min(
            0.10,
            len(relevant_topic_hits) * 0.025,
        )
        score += contribution
        reasons.append(
            "relevant GitHub topics: "
            + ", ".join(
                relevant_topic_hits[:6]
            )
        )

    # ---------------------------------------------------------
    # Small repository-quality signal
    #
    # Stars do NOT determine cybersecurity relevance.
    # This is deliberately a very small contribution.
    # ---------------------------------------------------------

    if repo.stars >= 100:
        score += 0.05
        reasons.append(
            "established repository (100+ stars)"
        )
    elif repo.stars >= 20:
        score += 0.03
        reasons.append(
            "established repository (20+ stars)"
        )
    elif repo.stars >= 5:
        score += 0.01
        reasons.append(
            "repository has community interest"
        )

    # ---------------------------------------------------------
    # Negative indicators
    # ---------------------------------------------------------

    if repo.fork:
        score -= 0.08
        reasons.append(
            "fork repository"
        )

    if repo.archived:
        score -= 0.05
        reasons.append(
            "archived repository"
        )

    if not repo.description:
        score -= 0.04
        reasons.append(
            "repository has no description"
        )

    # ---------------------------------------------------------
    # Important combination bonus
    #
    # Explicit security + explicit automotive/charging terminology
    # is stronger than either signal independently.
    # ---------------------------------------------------------

    if (
        security_hits > 0
        and (
            automotive_hits > 0
            or charging_hits > 0
        )
    ):
        score += 0.18
        reasons.append(
            "explicit security and vehicle/charging context"
        )

    score = max(
        0.0,
        min(1.0, score),
    )

    score = round(
        score,
        4,
    )

    if score >= 0.75:
        level = "direct"
    elif score >= 0.55:
        level = "strong"
    elif score >= 0.30:
        level = "contextual"
    else:
        level = "low"
    
    return {
        "score": score,
        "level": level,
        "reasons": reasons,
           }

# ---------------------------------------------------------------------------
# Statistics
# ---------------------------------------------------------------------------

def classify_statistics(
    repos: list[RepositoryRecord],
) -> dict[str, Any]:

    automotive = [
        repo
        for repo in repos
        if repo.automotive
    ]

    charging = [
        repo
        for repo in repos
        if repo.ev_charging
    ]

    security_related = [
        repo
        for repo in repos
        if repo.security_related
    ]

    both = [
        repo
        for repo in repos
        if (
            repo.automotive
            and repo.ev_charging
        )
    ]

    automotive_only = [
        repo
        for repo in repos
        if (
            repo.automotive
            and not repo.ev_charging
        )
    ]

    charging_only = [
        repo
        for repo in repos
        if (
            repo.ev_charging
            and not repo.automotive
        )
    ]

    languages = Counter(
        repo.language
        for repo in repos
        if repo.language
    )
    relevance_results = [
        calculate_relevance(repo)
        for repo in repos
    ]

    relevance_levels = Counter(
        result["level"]
        for result in relevance_results
    )

    relevance_scores = [
        result["score"]
        for result in relevance_results
    ]
    return {
        "unique_repositories": len(repos),

        "classification": {
            "automotive": len(automotive),
            "ev_charging": len(charging),
            "security_related": len(
                security_related
            ),

            "automotive_and_ev_charging": (
                len(both)
            ),

            "automotive_only": len(
                automotive_only
            ),

            "ev_charging_only": len(
                charging_only
            ),
        },

        "relevance": {
            "direct": relevance_levels.get(
                "direct",
                0,
            ),
            "strong": relevance_levels.get(
                "strong",
                0,
            ),
            "contextual": relevance_levels.get(
                "contextual",
                0,
            ),
            "low": relevance_levels.get(
                "low",
                0,
            ),
            "direct_or_strong": (
                relevance_levels.get("direct", 0)
                + relevance_levels.get("strong", 0)
            ),
            "average_score": round(
                (
                    sum(relevance_scores)
                    / len(relevance_scores)
                )
                if relevance_scores
                else 0.0,
                4,
            ),
        },

        "repository_state": {
            "forks": sum(
                1
                for repo in repos
                if repo.fork
            ),
            "archived": sum(
                1
                for repo in repos
                if repo.archived
            ),
            "disabled": sum(
                1
                for repo in repos
                if repo.disabled
            ),
        },

        "top_languages": [
            {
                "language": language,
                "count": count,
            }
            for language, count
            in languages.most_common(20)
        ],
    }


def query_statistics(
    query_metadata: list[dict[str, Any]],
) -> dict[str, Any]:

    by_profile: dict[str, dict[str, int]] = {}

    for entry in query_metadata:
        profile = str(
            entry.get("profile")
            or "unknown"
        )

        profile_stats = by_profile.setdefault(
            profile,
            {
                "queries": 0,
                "reported_total_count": 0,
                "items_received": 0,
                "queries_with_errors": 0,
            },
        )

        profile_stats["queries"] += 1

        total_count = entry.get(
            "total_count"
        )

        if isinstance(total_count, int):
            profile_stats[
                "reported_total_count"
            ] += total_count

        profile_stats[
            "items_received"
        ] += safe_int(
            entry.get("items_received")
        )

        if entry.get("error"):
            profile_stats[
                "queries_with_errors"
            ] += 1

    return by_profile


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description=(
            "Probe GitHub repository discovery for "
            "automotive cybersecurity and EV charging."
        )
    )

    parser.add_argument(
        "--output",
        type=Path,
        default=DEFAULT_OUTPUT,
        help=(
            "Output JSON file "
            "(default: github_probe.json)"
        ),
    )

    parser.add_argument(
        "--existing-json",
        type=Path,
        default=None,
        help=(
            "Optional existing repository dataset "
            "for overlap comparison"
        ),
    )

    parser.add_argument(
        "--token",
        default=None,
        help=(
            "GitHub token. Prefer GITHUB_TOKEN "
            "environment variable."
        ),
    )

    parser.add_argument(
        "--max-pages",
        type=int,
        default=10,
        help=(
            "Maximum result pages per search query "
            "(default: 10)"
        ),
    )

    parser.add_argument(
        "--per-page",
        type=int,
        default=100,
        help=(
            "Results per GitHub search page "
            "(default: 100)"
        ),
    )

    parser.add_argument(
        "--timeout",
        type=int,
        default=30,
        help="HTTP timeout in seconds",
    )


    parser.add_argument(
        "--retries",
        type=int,
        default=5,
        help="Retries for failed or rate-limited HTTP requests",
    )


    parser.add_argument(
        "--request-delay",
        type=float,
        default=0.25,
        help=(
            "Delay between GitHub search pages "
            "(default: 0.25 seconds)"
        ),
    )

    parser.add_argument(
        "--include-forks",
        action="store_true",
        help=(
            "Keep fork repositories in final "
            "probe results"
        ),
    )

    parser.add_argument(
        "--include-archived",
        action="store_true",
        help=(
            "Keep archived repositories in final "
            "probe results"
        ),
    )

    parser.add_argument(
        "--min-stars",
        type=int,
        default=0,
        help=(
            "Minimum repository stars for final "
            "probe result (default: 0)"
        ),
    )

    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        help=(
            "Increase logging verbosity. "
            "Use -vv for debug logging."
        ),
    )

    return parser.parse_args()


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main() -> int:
    args = parse_args()

    configure_logging(args.verbose)

    token = (
        args.token
        or os.environ.get("GITHUB_TOKEN")
        or None
    )

    if token:
        LOG.info(
            "Using authenticated GitHub API requests"
        )
    else:
        LOG.warning(
            "No GitHub token configured; "
            "using unauthenticated API requests"
        )

    if args.max_pages < 1:
        raise ValueError(
            "--max-pages must be at least 1"
        )

    if not 1 <= args.per_page <= 100:
        raise ValueError(
            "--per-page must be between 1 and 100"
        )

    headers = build_headers(token)

    query_metadata: list[dict[str, Any]] = []

    repositories: dict[
        str,
        RepositoryRecord,
    ] = {}

    profiles = [
        (
            "automotive",
            AUTOMOTIVE_QUERIES,
        ),
        (
            "ev_charging",
            EV_CHARGING_QUERIES,
        ),
    ]

    raw_hits = 0

    for profile, queries in profiles:
        LOG.info(
            "Starting %s discovery: %d queries",
            profile,
            len(queries),
        )

        for query in queries:
            items, metadata = search_repositories(
                query=query,
                profile=profile,
                headers=headers,
                per_page=args.per_page,
                max_pages=args.max_pages,
                timeout=args.timeout,
                retries=args.retries,
                request_delay=args.request_delay,
            )

            query_metadata.append(metadata)
            raw_hits += len(items)

            for item in items:
                repo = repository_from_api(
                    item=item,
                    profile=profile,
                    query=query,
                )

                if not repo.full_name:
                    continue

                key = repo.full_name.lower()

                if key in repositories:
                    merge_repository(
                        repositories[key],
                        repo,
                    )
                else:
                    repositories[key] = repo

    discovered = list(
        repositories.values()
    )

    discovered.sort(
        key=lambda repo: (
            repo.stars,
            repo.full_name.lower(),
        ),
        reverse=True,
    )

    pre_filter_count = len(discovered)

    # Conservative final filtering.
    filtered: list[RepositoryRecord] = []

    for repo in discovered:
        if (
            not args.include_forks
            and repo.fork
        ):
            continue

        if (
            not args.include_archived
            and repo.archived
        ):
            continue

        if repo.disabled:
            continue

        if repo.stars < args.min_stars:
            continue

        filtered.append(repo)

    existing_names: set[str] = set()

    if args.existing_json:
        if not args.existing_json.exists():
            LOG.error(
                "Existing JSON not found: %s",
                args.existing_json,
            )
            return 2

        existing_names = (
            load_existing_repositories(
                args.existing_json
            )
        )

    filtered_names = {
        repo.full_name.lower()
        for repo in filtered
    }

    overlap = (
        filtered_names
        & existing_names
    )

    new_candidates = (
        filtered_names
        - existing_names
    )

    existing_not_rediscovered = (
        existing_names
        - filtered_names
    )

    stats = classify_statistics(
        filtered
    )

    comparison: dict[str, Any] | None = None

    if args.existing_json:
        comparison = {
            "existing_json": str(
                args.existing_json
            ),
            "existing_repository_identities": (
                len(existing_names)
            ),
            "rediscovered_existing": len(
                overlap
            ),
            "new_candidates": len(
                new_candidates
            ),
            "existing_not_rediscovered": len(
                existing_not_rediscovered
            ),
        }

    output = {
        "schema_version": "1.0",
        "script_version": SCRIPT_VERSION,
        "generated_at": utc_now(),

        "description": (
            "GitHub repository discovery probe for "
            "automotive cybersecurity and EV "
            "charging security."
        ),

        "configuration": {
            "authenticated": bool(token),
            "max_pages_per_query": (
                args.max_pages
            ),
            "per_page": args.per_page,
            "include_forks": (
                args.include_forks
            ),
            "include_archived": (
                args.include_archived
            ),
            "minimum_stars": (
                args.min_stars
            ),
            "automotive_queries": (
                AUTOMOTIVE_QUERIES
            ),
            "ev_charging_queries": (
                EV_CHARGING_QUERIES
            ),
        },

        "statistics": {
            "raw_search_hits": raw_hits,
            "unique_before_filtering": (
                pre_filter_count
            ),
            **stats,
            "query_profiles": (
                query_statistics(
                    query_metadata
                )
            ),
        },

        "existing_dataset_comparison": (
            comparison
        ),

        "query_results": query_metadata,

        "repositories": [
            repo.to_dict()
            for repo in filtered
        ],
    }

    args.output.parent.mkdir(
        parents=True,
        exist_ok=True,
    )

    with args.output.open(
        "w",
        encoding="utf-8",
    ) as handle:
        json.dump(
            output,
            handle,
            indent=2,
            ensure_ascii=False,
        )

        handle.write("\n")

    classification = stats[
        "classification"
    ]
    relevance = stats[
        "relevance"
    ]
    print()
    print("GitHub discovery probe")
    print("======================")
    print()

    print(
        f"Queries automotive        : "
        f"{len(AUTOMOTIVE_QUERIES)}"
    )

    print(
        f"Queries EV charging       : "
        f"{len(EV_CHARGING_QUERIES)}"
    )

    print()

    print(
        f"Raw search hits           : "
        f"{raw_hits}"
    )

    print(
        f"Unique before filtering   : "
        f"{pre_filter_count}"
    )

    print(
        f"Unique repositories       : "
        f"{len(filtered)}"
    )

    print()

    print("Classification")
    print(
        f"  Automotive              : "
        f"{classification['automotive']}"
    )

    print(
        f"  EV charging             : "
        f"{classification['ev_charging']}"
    )

    print(
        f"  Both                    : "
        f"{classification['automotive_and_ev_charging']}"
    )

    print(
        f"  Automotive only         : "
        f"{classification['automotive_only']}"
    )

    print(
        f"  EV charging only        : "
        f"{classification['ev_charging_only']}"
    )

    print(
        f"  Security terms present  : "
        f"{classification['security_related']}"
    )

    if comparison:
        print()
        print("Existing dataset comparison")

        print(
            f"  Existing identities     : "
            f"{comparison['existing_repository_identities']}"
        )

        print(
            f"  Rediscovered            : "
            f"{comparison['rediscovered_existing']}"
        )

        print(
            f"  New candidates          : "
            f"{comparison['new_candidates']}"
        )

        print(
            f"  Not rediscovered        : "
            f"{comparison['existing_not_rediscovered']}"
        )

    print()
    relevance = stats[
        "relevance"
    ]
    print()
    print("Relevance")

    print(
        f"  Direct                  : "
        f"{relevance['direct']}"
    )

    print(
        f"  Strong                  : "
        f"{relevance['strong']}"
    )

    print(
        f"  Contextual              : "
        f"{relevance['contextual']}"
    )

    print(
        f"  Low                     : "
        f"{relevance['low']}"
    )

    print(
        f"  Direct + strong         : "
        f"{relevance['direct_or_strong']}"
    )

    print(
        f"  Average score           : "
        f"{relevance['average_score']:.4f}"
    )    
    print(
        f"Output written to         : "
        f"{args.output}"
    )

    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except KeyboardInterrupt:
        print(
            "\nInterrupted.",
            file=sys.stderr,
        )
        raise SystemExit(130)
    except Exception as exc:
        LOG.exception(
            "Fatal error: %s",
            exc,
        )
        raise SystemExit(1)
