a
    Vuðj²  ã                
   @  s®  d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z
ddlZ
ddlZ
ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZ d	Zd
Ze› d�ZedƒZe d¡Zg d¢Zg d¢Zh d£Z h d£Z!h d£Z"eG dd„ dƒƒZ#ddœdd„Z$dddœdd„Z%dddœd d!„Z&dddœd"d#„Z'd$d%d&œd'd(„Z(dd)d*œd+d,„Z)dd%ddd-d.œd/d0„Z*ddd%ddddd1d2d3œ	d4d5„Z+d)dddd6œd7d8„Z,dddd9œd:d;„Z-dd<d=œd>d?„Z.dd$d@œdAdB„Z/dCd<dDœdEdF„Z0dd)dGœdHdI„Z1dJd)dKœdLdM„Z2dNd)dOœdPdQ„Z3dRdœdSdT„Z4ddœdUdV„Z5e6dWk�rªze7e5ƒ ƒ‚W nb e8�yp   e9dXej:dY� e7dZƒ‚Y n: e;�y¨ Z< z e =d[e<¡ e7d\ƒ‚W Y dZ<[<n
dZ<[<0 0 dS )]a”  
github_probe.py

Discovery probe for public GitHub repositories relevant to:

1. Automotive cybersecurity
2. EV charging cybersecurity

Purpose
-------
This script is intentionally lightweight.

It does NOT:
- download repository contents
- download README files
- run AI/LLM analysis
- modify an existing GitHub dataset
- correlate repositories against the Threat Register

It DOES:
- run targeted GitHub repository searches
- collect repository metadata
- deduplicate repositories by full_name
- classify repositories into discovery groups
- optionally compare them with an existing repository dataset
- write a JSON report for inspection

Authentication
--------------
Optionally provide a GitHub token:

    export GITHUB_TOKEN="..."

No token is embedded or written to output.

Examples
--------

Discovery probe:

    python3 github_probe.py

Verbose:

    python3 github_probe.py -v

Compare against an existing dataset:

    python3 github_probe.py         --existing-json github/github.json

Custom output:

    python3 github_probe.py         --output github_probe.json

Limit pages during testing:

    python3 github_probe.py         --max-pages 2         -v
é    )ÚannotationsN)ÚCounter)Ú	dataclassÚfield)ÚdatetimeÚtimezone)ÚPath)ÚAnyz1.0.0zhttps://api.github.comz/search/repositorieszgithub_probe.jsonZgithub_probe)úautomotive cybersecurityzvehicle cybersecurityúautomotive securityúcar hackingzCAN bus securityzCAN bus hackingzautomotive CANzUDS automotive securityzISO-TP automotivezDoIP automotivezautomotive ECU securityzvehicle ECU hackingzautomotive diagnostics securityzconnected vehicle securityzconnected car securityztelematics security automotivezautomotive OTA securityzvehicle OTA securityz!keyless entry security automotivezautomotive immobilizer securityzV2X security automotivezADAS security automotivezautonomous vehicle security)zEV charging securityzEV charger securityz"electric vehicle charging securityzcharging station security EVzOCPP securityzOCPP chargerz	OCPP EVSEzEVSE securityz	CSMS OCPPzISO 15118 securityzISO 15118 EVzPlug and Charge securityzV2G securityzvehicle-to-grid security>!   Zobd2zcan fdzconnected vehicler   zobd-iizcan-fdzautomotive ethernetzautonomous vehicleZkeylessr
   zcan busr   ZobdZcarszcan-busZudsZ
telematicszconnected carziso-tpZisotpZcarZecusÚ
automotiveZecuzvehicle securityZv2xZcanbusZadasZvehicleszvehicle hackingZimmobilizerZdoipZvehicle>   zvehicle to gridzcharging stationsZcsmsz	iso 15118zcharging infrastructurezcharge pointZv2gzev chargersZocppzplug-and-chargezvehicle-to-gridzcharging stationzev chargingzelectric vehicle chargingz
ev chargerZchargepointzplug and chargeZiso15118Zevse>   ZsecurityZthreatZthreatsZidsZattackZfuzzerzcyber securityZexploitationZ	forensicszpenetration testingZvulnerabilitieszintrusion detectionZhackingZpentestZattacksZexploitZvulnerabilityZcybersecurityzreverse engineeringZfuzzingZhackc                   @  sŒ  e Zd ZU ded< ded< ded< ded< ded< dZded	< eed
�Zded< dZded< dZ	ded< dZ
ded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< eed
�Zded< eed
�Zded < eed
�Zded!< eed
�Zded"< eed
�Zded#< dZded$< dZded%< dZded&< dd'œd(d)„Zd*d'œd+d,„Zd-d'œd.d/„ZdS )0ÚRepositoryRecordÚintÚidÚstrÚ	full_nameÚnameÚownerÚhtml_urlÚ Údescription)Údefault_factoryz	list[str]ÚtopicsNú
str | NoneÚlanguager   ÚstarsÚforksÚopen_issuesÚwatchersFÚboolÚarchivedÚdisabledÚforkÚ
created_atÚ
updated_atÚ	pushed_atÚlicenseúset[str]Úsearch_profilesÚmatched_queriesÚautomotive_term_hitsÚev_charging_term_hitsÚsecurity_term_hitsr   Úev_chargingÚsecurity_related©Úreturnc                 C  s$   d  | j| j| jd  | j¡g¡ ¡ S )Nú )Újoinr   r   r   r   Úlower)Úself© r6   úgithub_probe.pyÚsearchable_text  s    
üÿz RepositoryRecord.searchable_textÚNonec                   s”   |   ¡ ‰ t‡ fdd„tD ƒƒ| _t‡ fdd„tD ƒƒ| _t‡ fdd„tD ƒƒ| _d| jv }d| jv }t	| jpn|ƒ| _
t	| jp~|ƒ| _t	| jƒ| _d S )Nc                 3  s   | ]}|ˆ v r|V  qd S ©Nr6   ©Ú.0Zterm©Útextr6   r7   Ú	<genexpr>#  s   þz,RepositoryRecord.classify.<locals>.<genexpr>c                 3  s   | ]}|ˆ v r|V  qd S r:   r6   r;   r=   r6   r7   r?   )  s   þc                 3  s   | ]}|ˆ v r|V  qd S r:   r6   r;   r=   r6   r7   r?   /  s   þr   r.   )r8   ÚsortedÚAUTOMOTIVE_TERMSr+   ÚEV_CHARGING_TERMSr,   ÚSECURITY_TERMSr-   r)   r    r   r.   r/   )r5   Zautomotive_originZcharging_originr6   r=   r7   Úclassify   s.    þ
þ
þ


þþÿzRepositoryRecord.classifyúdict[str, Any]c                 C  sž   t | ƒ}d| j ¡ › �| j| j| j| j| j| jt| j	ƒ| j
| j| j| j| j| j| j| j| j| j| j| jt| jƒt| jƒdœ| j| j| j| j| j| jdœ|dœS )Nzgithub:)r)   r*   )r   r.   r/   r+   r,   r-   )r   Z	github_idr   r   r   Úurlr   r   r   r   r   r   r   r!   r"   r#   r'   r$   r%   r&   Z	discoveryÚclassificationÚ	relevance)Úcalculate_relevancer   r4   r   r   r   r   r   r@   r   r   r   r   r   r   r!   r"   r#   r'   r$   r%   r&   r)   r*   r   r.   r/   r+   r,   r-   )r5   rH   r6   r6   r7   Úto_dictG  sB    þúÝzRepositoryRecord.to_dict) Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r   r   Úlistr   r   r   r   r   r   r!   r"   r#   r$   r%   r&   r'   Úsetr)   r*   r+   r,   r-   r   r.   r/   r8   rD   rJ   r6   r6   r6   r7   r   ï   s<   

'r   r   r0   c                   C  s    t  tj¡jdd� ¡  dd¡S )Nr   )Zmicrosecondz+00:00ÚZ)r   Únowr   ZutcÚreplaceZ	isoformatr6   r6   r6   r7   Úutc_nowt  s    ÿ
ýÿrT   r   r9   )Úverboser1   c                 C  s8   | dkrt j}n| dkr t j}nt j}t j|dd� d S )Né   é   z'%(asctime)s %(levelname)-8s %(message)s)ÚlevelÚformat)ÚloggingÚDEBUGÚINFOZWARNINGZbasicConfig)rU   rX   r6   r6   r7   Úconfigure_logging}  s    þr]   r	   ©Úvaluer1   c                 C  s.   | d u rdS d  t| ƒ dd¡ dd¡ ¡ ¡S )Nr   r2   úÚ
)r3   r   rS   Úsplit©r_   r6   r6   r7   Ú
clean_text‹  s    ÿþÿrd   c              	   C  s,   zt | p
dƒW S  ttfy&   Y dS 0 d S )Nr   )r   Ú	TypeErrorÚ
ValueErrorrc   r6   r6   r7   Úsafe_int—  s    rg   r   zdict[str, str])Útokenr1   c                 C  s(   ddt › �ddœ}| r$d| › �|d< |S )Nzapplication/vnd.github+jsonzNCRC-GitHub-Probe/z
2022-11-28)ZAcceptz
User-AgentzX-GitHub-Api-VersionzBearer ZAuthorization)ÚSCRIPT_VERSION)rh   Úheadersr6   r6   r7   Úbuild_headers¢  s    ýrk   rE   )rj   r1   c              	   C  s|   i }ddddddœ}|  ¡ D ]Z\}}|  |¡}|d u r8q|dv rnzt|ƒ||< W qv tyj   |||< Y qv0 q|||< q|S )NzX-RateLimit-LimitzX-RateLimit-RemainingzX-RateLimit-UsedzX-RateLimit-ResetzX-RateLimit-Resource)ÚlimitÚ	remainingÚusedÚresetÚresource>   rm   rn   rl   ro   )ÚitemsÚgetr   rf   )rj   ÚresultZmappingsÚkeyZheader_namer_   r6   r6   r7   Úparse_rate_headers¯  s$    û

ru   z%tuple[dict[str, Any], dict[str, Any]])rF   rj   ÚtimeoutÚretriesr1   c                 C  sî  d }t |d ƒD �]Ä}tjj| |dd�}zhtjj||d��D}| ¡ jddd�}t |¡}	t	|j
ƒ}
|	|
fW  d   ƒ W   S 1 s„0    Y  W q tjj�yt } �zÄ|}z| ¡ jddd�}W n tyØ   d}Y n0 t	|j
ƒ}
t d	|j| ¡ |�rFz&t |¡}t d
t| d|¡ƒ¡ W n$ t�yD   t dt|ƒ¡ Y n0 |jdv �rÔ|
 d¡dk�rÔ|
 d¡�rÔtt ¡ ƒ}tdt|
d ƒ| d ƒ}||k�r¨tdƒ|‚t d|¡ t |¡ t d¡ W Y d }~q|j
 d¡}|jdv �r^||k �r^|�r*ztdt|ƒƒ}W n t�y&   d}Y n0 ntd|d  dƒ}t d|¡ t |¡ W Y d }~q‚ W Y d }~qd }~0  tjjttjf�yÔ } z<|}||k�r ‚ d| }t d||¡ t |¡ W Y d }~qd }~0 0 q|�râ|‚tdƒ‚d S )NrW   ZGET)rj   Úmethod)rv   úutf-8rS   )Úerrorsr   zGitHub HTTP %s: %szGitHub response: %sÚmessagezGitHub response body: %s>   i“  i­  rm   r   ro   rV   z8GitHub rate limit exhausted and retry budget was reachedz>GitHub rate limit exhausted. Waiting %d seconds until reset...z0GitHub rate-limit window reset; retrying requestzRetry-Afteré<   é´   zRetrying after %d secondsz$Request failed (%s); retrying in %dsz*GitHub request failed without an exception)ÚrangeÚurllibÚrequestZRequestZurlopenÚreadÚdecodeÚjsonÚloadsru   rj   ÚerrorZ	HTTPErrorÚ	ExceptionÚLOGÚwarningÚcoderd   rr   Údebugr   ÚtimeÚmaxÚRuntimeErrorÚsleepÚinforf   ÚminZURLErrorÚTimeoutErrorZJSONDecodeError)rF   rj   rv   rw   Z
last_errorZattemptr€   ZresponseÚbodyÚpayloadÚrateÚexcZerror_payloadrR   Zwait_forZretry_afterr6   r6   r7   Úgithub_get_jsonÑ  sü    ýþþ
ÿ0þ


ý
þÿþ	þ
ÿþýþ
ÿýý
ÿÿÿþþ
þþ
ý
ý"ÿr–   Úfloatz+tuple[list[dict[str, Any]], dict[str, Any]])	ÚqueryÚprofilerj   Úper_pageÚ	max_pagesrv   rw   Úrequest_delayr1   c                 C  s¢  g }| |d d ddi d dœ}	t d|d ƒD �]n}
| ||
dddœ}td tj |¡ }t d||
| ¡ zt||||d	�\}}W nN tyÈ } z6t	|ƒ|	d
< t 
d| |¡ W Y d }~ �qšW Y d }~n
d }~0 0 |	d  d7  < ||	d< |	d d u �r| d¡|	d< | d¡|	d< | dg ¡}t|tƒ�s6t d| ¡  �qš| |¡ |	d  t|ƒ7  < t d| |
t|ƒ| dd¡¡ t|ƒ|k �r† �qš|dkr(t |¡ q(||	fS )Nr   )r˜   r™   Útotal_countÚincomplete_resultsÚpages_requestedÚitems_receivedr”   r…   rW   r   Zdesc)Úqrš   ÚpageÚsortÚorderú?zSearching %-11s page %-2d: %s)rF   rj   rv   rw   r…   zSearch failed for %r: %srŸ   r”   r�   rž   rq   zUnexpected GitHub result for %rr    z5Query %r page %d returned %d items; rate remaining=%srm   )r~   ÚSEARCH_ENDPOINTr   ÚparseZ	urlencoder‡   r�   r–   r†   r   r…   rr   Ú
isinstancerO   rˆ   ÚextendÚlenrŠ   r‹   rŽ   )r˜   r™   rj   rš   r›   rv   rw   rœ   ZresultsÚmetadatar¢   ÚparamsrF   r“   r”   r•   rq   r6   r6   r7   Úsearch_repositoriesy  s’    øû	ÿ
þÿüüý&ÿÿþ

ú	r­   )Úitemr™   r˜   r1   c                 C  sF  |   d¡pi }|   d¡pi }|   d¡p(g }t|tƒs8g }tt|   d¡ƒt|   d¡ƒt|   d¡ƒt|  d¡ƒt|   d¡ƒt|   d	¡ƒd
d„ |D ƒt|   d¡ƒpžd t|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒ|   d¡|   d¡|   d¡t|  d¡ƒ�pd d�}|j |¡ |j	 |¡ | 
¡  |S )Nr   r'   r   r   r   r   Zloginr   r   c                 S  s    g | ]}t |ƒrt |ƒ ¡ ‘qS r6   )rd   r4   )r<   Útopicr6   r6   r7   Ú
<listcomp>	  s   þz'repository_from_api.<locals>.<listcomp>r   Zstargazers_countZforks_countZopen_issues_countZwatchers_countr!   r"   r#   r$   r%   r&   Zspdx_id)r   r   r   r   r   r   r   r   r   r   r   r   r!   r"   r#   r$   r%   r&   r'   )rr   r¨   rO   r   rg   rd   r    r)   Úaddr*   rD   )r®   r™   r˜   Z	owner_objZlicense_objr   Úrepor6   r6   r7   Úrepository_from_apiê  s|    
ÿÿÿÿÿþÿÿÿÿÿÿÿÿÿÍ6r³   )ÚexistingÚincomingr1   c                 C  sº   | j  |j ¡ | j |j¡ t|jƒt| jƒkr8|j| _tt| jƒt|jƒB ƒ| _t| j	|j	ƒ| _	t| j
|j
ƒ| _
t| j|jƒ| _t| j|jƒ| _| js |j| _| js®|j| _|  ¡  d S r:   )r)   Úupdater*   rª   r   r@   rP   r   rŒ   r   r   r   r   r   r'   rD   )r´   rµ   r6   r6   r7   Úmerge_repository6  sL    ÿÿÿÿÿÿÿþþþþr·   r(   )r“   r1   c                   s(   t ƒ ‰dddœ‡ ‡fdd„‰ ˆ | ƒ ˆS )zÆ
    Best-effort extraction of owner/repository names from an existing
    repository dataset.

    Supports common structures without assuming the schema of the
    older AI-generated dataset.
    r	   r9   r^   c                   sÜ   t | tƒr | D ]}ˆ |ƒ qd S t | tƒs.d S dD ]F}|  |¡}t |tƒsLq2| ¡ }|r2d|v r2| d¡s2ˆ | ¡ ¡ q2dD ]4}|  |¡}t |tƒs˜q~t	|ƒ}|r~ˆ | ¡ ¡ q~|  
¡ D ]}t |ttfƒr¼ˆ |ƒ q¼d S )N)r   r²   Z
repositoryZrepository_nameZname_with_ownerú/Zhttp)rF   r   Z
github_urlZrepository_url)r¨   rO   Údictrr   r   ÚstripÚ
startswithr±   r4   Úparse_github_repo_urlÚvalues)r_   ÚchildÚ
field_nameÚ	candidateÚparsed©Úinspectrs   r6   r7   rÃ     sJ    




ÿþÿýÿ

ÿþz,extract_existing_repo_names.<locals>.inspect)rP   )r“   r6   rÂ   r7   Úextract_existing_repo_namesr  s    ;rÄ   )rF   r1   c                 C  sœ   zt  | ¡}W n ty"   Y d S 0 |j ¡ dvr6d S dd„ |j d¡D ƒ}t|ƒdk r\d S |d }|d }| d¡r‚|d d	… }|rŠ|sŽd S |› d|› �S )
N>   z
github.comzwww.github.comc                 S  s   g | ]}|r|‘qS r6   r6   )r<   Zpiecer6   r6   r7   r°   Í  s   þz)parse_github_repo_url.<locals>.<listcomp>r¸   rV   r   rW   z.gitéüÿÿÿ)	ÚeZurlparserf   Znetlocr4   Úpathrb   rª   Úendswith)rF   rÁ   Úpiecesr   r²   r6   r6   r7   r¼   ¾  s$    
þ
r¼   r   )rÇ   r1   c                 C  sV   | j ddd��}t |¡}W d   ƒ n1 s.0    Y  t|ƒ}t dt|ƒ| ¡ |S )NÚrry   ©Úencodingz'Loaded %d repository identities from %s)Úopenrƒ   ÚloadrÄ   r‡   r�   rª   )rÇ   Úhandler“   Únamesr6   r6   r7   Úload_existing_repositoriesâ  s    þ(ÿýrÑ   )r²   r1   c           
      C  s”  d}g }t | jƒ}t | jƒ}t | jƒ}|rRtd|d ƒ}||7 }| d|› d�¡ |r~td|d ƒ}||7 }| d|› d�¡ |rªtd	|d ƒ}||7 }| d
|› d�¡ t | jƒ}|dkrätd|d ƒ}||7 }| d|› d�¡ d| jv �rd| jv �r|d7 }| d¡ tdd„ | j	D ƒƒ}|�r^tdt |ƒd ƒ}||7 }| dd 
|dd… ¡ ¡ | jdk�r~|d7 }| d¡ n>| jdk�rž|d7 }| d¡ n| jd k�r¼|d!7 }| d"¡ | j�rÖ|d8 }| d#¡ | j�rð|d8 }| d$¡ | j�s
|d%8 }| d&¡ |d'k�r:|d'k�s(|d'k�r:|d(7 }| d)¡ tdtd*|ƒƒ}t|d+ƒ}|d,k�rdd-}	n$|d.k�rtd/}	n|dk�r„d0}	nd1}	||	|d2œS )3a9  
    Calculate a transparent discovery relevance score.

    This score estimates whether a discovered repository is worth
    deeper inspection. It is NOT a threat-correlation score.

    Levels:
        direct      >= 0.75
        strong      >= 0.55
        contextual  >= 0.30
        low         <  0.30
    ç        gìQ¸…ëÑ?gìQ¸…ë±?zautomotive terminology (z hits)g333333Ó?g{®Gáz´?zEV charging terminology (g{®GázÔ?zexplicit security terminology (rV   g¸…ëQ¸¾?gš™™™™™™?zmatched z targeted discovery queriesr   r.   z6discovered by both automotive and EV charging profilesc                   sR   h | ]J‰ t ‡ fd d„tD ƒƒsJt ‡ fdd„tD ƒƒsJt ‡ fdd„tD ƒƒrˆ ’qS )c                 3  s   | ]}|ˆ v V  qd S r:   r6   r;   ©r¯   r6   r7   r?   f  s   ÿz0calculate_relevance.<locals>.<setcomp>.<genexpr>c                 3  s   | ]}|ˆ v V  qd S r:   r6   r;   rÓ   r6   r7   r?   j  s   ÿc                 3  s   | ]}|ˆ v V  qd S r:   r6   r;   rÓ   r6   r7   r?   n  s   ÿ)ÚanyrA   rB   rC   )r<   r6   rÓ   r7   Ú	<setcomp>b  s   þþþõz&calculate_relevance.<locals>.<setcomp>gš™™™™™¹?zrelevant GitHub topics: z, Né   éd   gš™™™™™©?z#established repository (100+ stars)é   g¸…ëQ¸ž?z"established repository (20+ stars)é   g{®Gáz„?z!repository has community interestzfork repositoryzarchived repositoryg{®Gáz¤?zrepository has no descriptionr   g
×£p=
Ç?z.explicit security and vehicle/charging contextg      ð?é   g      è?Údirectgš™™™™™á?ÚstrongÚ
contextualÚlow)ÚscorerX   Úreasons)rª   r+   r,   r-   r�   Úappendr*   r)   r@   r   r3   r   r#   r!   r   rŒ   Úround)
r²   rß   rà   Zautomotive_hitsZcharging_hitsZsecurity_hitsZcontributionZmatched_query_countZrelevant_topic_hitsrX   r6   r6   r7   rI   ø  s   ÿÿÿþ
ÿþ
ÿþ
ÿÿþ
ÿ	ÿþÿþÿ
þ
ÿÿÿÿÿÿÿÿÿÿýüÿþþ


ýrI   zlist[RepositoryRecord])Úreposr1   c                 C  sn  dd„ | D ƒ}dd„ | D ƒ}dd„ | D ƒ}dd„ | D ƒ}dd„ | D ƒ}dd„ | D ƒ}t dd	„ | D ƒƒ}d
d„ | D ƒ}t dd	„ |D ƒƒ}	dd„ |D ƒ}
t| ƒt|ƒt|ƒt|ƒt|ƒt|ƒt|ƒdœ|	 dd¡|	 dd¡|	 dd¡|	 dd¡|	 dd¡|	 dd¡ t|
�rt|
ƒt|
ƒ nddƒdœtdd	„ | D ƒƒtdd	„ | D ƒƒtdd	„ | D ƒƒdœdd„ | d¡D ƒdœS )Nc                 S  s   g | ]}|j r|‘qS r6   )r   ©r<   r²   r6   r6   r7   r°   ä  s   þz'classify_statistics.<locals>.<listcomp>c                 S  s   g | ]}|j r|‘qS r6   )r.   rä   r6   r6   r7   r°   ê  s   þc                 S  s   g | ]}|j r|‘qS r6   )r/   rä   r6   r6   r7   r°   ð  s   þc                 S  s   g | ]}|j r|jr|‘qS r6   ©r   r.   rä   r6   r6   r7   r°   ö  s   üc                 S  s   g | ]}|j r|js|‘qS r6   rå   rä   r6   r6   r7   r°   ÿ  s   üc                 S  s   g | ]}|j r|js|‘qS r6   )r.   r   rä   r6   r6   r7   r°     s   üc                 s  s   | ]}|j r|j V  qd S r:   )r   rä   r6   r6   r7   r?     s   þz&classify_statistics.<locals>.<genexpr>c                 S  s   g | ]}t |ƒ‘qS r6   )rI   rä   r6   r6   r7   r°     s   ÿc                 s  s   | ]}|d  V  qdS )rX   Nr6   ©r<   rs   r6   r6   r7   r?     s   ÿc                 S  s   g | ]}|d  ‘qS )rß   r6   ræ   r6   r6   r7   r°      s   ÿ)r   r.   r/   Úautomotive_and_ev_chargingÚautomotive_onlyÚev_charging_onlyrÛ   r   rÜ   rÝ   rÞ   rÒ   rÚ   )rÛ   rÜ   rÝ   rÞ   Údirect_or_strongÚaverage_scorec                 s  s   | ]}|j rd V  qdS ©rW   N)r#   rä   r6   r6   r7   r?   \  s   þc                 s  s   | ]}|j rd V  qdS rì   )r!   rä   r6   r6   r7   r?   a  s   þc                 s  s   | ]}|j rd V  qdS rì   )r"   rä   r6   r6   r7   r?   f  s   þ)r   r!   r"   c                 S  s   g | ]\}}||d œ‘qS ))r   Úcountr6   )r<   r   rí   r6   r6   r7   r°   m  s   ýþrØ   )Zunique_repositoriesrG   rH   Zrepository_stateZtop_languages)r   rª   rr   râ   ÚsumÚmost_common)rã   r   Zchargingr/   Zbothrè   Zcharging_onlyZ	languagesZrelevance_resultsZrelevance_levelsZrelevance_scoresr6   r6   r7   Úclassify_statisticsà  s°    þþþþ	þ	þ	þþþþÿÿÿñþþþþ

ÿüÿÿùë!þþþõú·rð   zlist[dict[str, Any]])Úquery_metadatar1   c              	   C  s¢   i }| D ]”}t | d¡pdƒ}| |dddddœ¡}|d  d7  < | d¡}t|tƒrh|d  |7  < |d	  t| d	¡ƒ7  < | d
¡r|d  d7  < q|S )Nr™   Úunknownr   )ÚqueriesÚreported_total_countr    Úqueries_with_errorsró   rW   r�   rô   r    r…   rõ   )r   rr   Ú
setdefaultr¨   r   rg   )rñ   Z
by_profileÚentryr™   Zprofile_statsr�   r6   r6   r7   Úquery_statisticsx  sL    
þüþ
ÿ
ÿþÿÿþ
ÿþrø   zargparse.Namespacec                  C  sè   t jdd�} | jdttdd� | jdtd dd� | jdd d	d
� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jdddd� | jdddd� | jd td!d"d� | jd#d$d%d!d&d'� |  ¡ S )(NzOProbe GitHub repository discovery for automotive cybersecurity and EV charging.)r   z--outputz-Output JSON file (default: github_probe.json))ÚtypeÚdefaultÚhelpz--existing-jsonz;Optional existing repository dataset for overlap comparisonz--tokenz7GitHub token. Prefer GITHUB_TOKEN environment variable.)rú   rû   z--max-pagesé
   z3Maximum result pages per search query (default: 10)z
--per-pager×   z-Results per GitHub search page (default: 100)z	--timeouté   zHTTP timeout in secondsz	--retriesrÙ   z0Retries for failed or rate-limited HTTP requestsz--request-delayg      Ð?z9Delay between GitHub search pages (default: 0.25 seconds)z--include-forksÚ
store_truez-Keep fork repositories in final probe results)Úactionrû   z--include-archivedz1Keep archived repositories in final probe resultsz--min-starsr   z<Minimum repository stars for final probe result (default: 0)z-vz	--verboserí   z6Increase logging verbosity. Use -vv for debug logging.)rÿ   rú   rû   )ÚargparseÚArgumentParserÚadd_argumentr   ÚDEFAULT_OUTPUTr   r—   Ú
parse_args)Úparserr6   r6   r7   r  «  s”    þû
û
ü	û
û
üüû
ü	ü	û
úr  c                  C  s  t ƒ } t| jƒ | jp$tj d¡p$d }|r6t d¡ n
t 	d¡ | j
dk rRtdƒ‚d| j  krhdksrn tdƒ‚t|ƒ}g }i }dtfd	tfg}d
}|D ]®\}}t d|t|ƒ¡ |D ]Ž}	t|	||| j| j
| j| j| jd�\}
}| |¡ |t|
ƒ7 }|
D ]H}t|||	d�}|j�sqü|j ¡ }||v �r<t|| |ƒ qü|||< qüq¸qšt| ¡ ƒ}|jdd„ dd� t|ƒ}g }|D ]V}| j�s�|j�r��qx| j�s¤|j �r¤�qx|j!�r°�qx|j"| j#k �rÂ�qx| |¡ �qxt$ƒ }| j%�r| j% &¡ �süt 'd| j%¡ dS t(| j%ƒ}dd„ |D ƒ}||@ }|| }|| }t)|ƒ}d }| j%�rft*| j%ƒt|ƒt|ƒt|ƒt|ƒdœ}dt+t,ƒ dt-|ƒ| j
| j| j| j| j#ttdœ||dœ|¥dt.|ƒi¥||dd„ |D ƒdœ	}| j/j0j1ddd� | j/j2d d!d"��,}t3j4||dd#d$� | 5d%¡ W d   ƒ n1 �s0    Y  |d& }|d' }t6ƒ  t6d(ƒ t6d)ƒ t6ƒ  t6d*ttƒ› �ƒ t6d+ttƒ› �ƒ t6ƒ  t6d,|› �ƒ t6d-|› �ƒ t6d.t|ƒ› �ƒ t6ƒ  t6d/ƒ t6d0|d › �ƒ t6d1|d	 › �ƒ t6d2|d3 › �ƒ t6d4|d5 › �ƒ t6d6|d7 › �ƒ t6d8|d9 › �ƒ |�rzt6ƒ  t6d:ƒ t6d;|d< › �ƒ t6d=|d> › �ƒ t6d?|d@ › �ƒ t6dA|dB › �ƒ t6ƒ  |d' }t6ƒ  t6dCƒ t6dD|dE › �ƒ t6dF|dG › �ƒ t6dH|dI › �ƒ t6dJ|dK › �ƒ t6dL|dM › �ƒ t6dN|dO dP›�ƒ t6dQ| j/› �ƒ d
S )RNZGITHUB_TOKENz'Using authenticated GitHub API requestsz>No GitHub token configured; using unauthenticated API requestsrW   z--max-pages must be at least 1r×   z$--per-page must be between 1 and 100r   r.   r   z!Starting %s discovery: %d queries)r˜   r™   rj   rš   r›   rv   rw   rœ   )r®   r™   r˜   c                 S  s   | j | j ¡ fS r:   )r   r   r4   )r²   r6   r6   r7   Ú<lambda>�  s    þzmain.<locals>.<lambda>T)rt   ÚreversezExisting JSON not found: %srV   c                 S  s   h | ]}|j  ¡ ’qS r6   )r   r4   rä   r6   r6   r7   rÕ   Á  s   ÿzmain.<locals>.<setcomp>)Úexisting_jsonÚexisting_repository_identitiesÚrediscovered_existingÚnew_candidatesÚexisting_not_rediscoveredz1.0zXGitHub repository discovery probe for automotive cybersecurity and EV charging security.)ZauthenticatedZmax_pages_per_queryrš   Úinclude_forksÚinclude_archivedZminimum_starsZautomotive_queriesZev_charging_queries)Zraw_search_hitsZunique_before_filteringZquery_profilesc                 S  s   g | ]}|  ¡ ‘qS r6   )rJ   rä   r6   r6   r7   r°   #  s   ÿzmain.<locals>.<listcomp>)	Zschema_versionZscript_versionZgenerated_atr   ZconfigurationZ
statisticsZexisting_dataset_comparisonZquery_resultsÚrepositories)ÚparentsÚexist_okÚwry   rË   F)ÚindentZensure_asciira   rG   rH   zGitHub discovery probez======================zQueries automotive        : zQueries EV charging       : zRaw search hits           : zUnique before filtering   : zUnique repositories       : ZClassificationz  Automotive              : z  EV charging             : z  Both                    : rç   z  Automotive only         : rè   z  EV charging only        : ré   z  Security terms present  : r/   zExisting dataset comparisonz  Existing identities     : r	  z  Rediscovered            : r
  z  New candidates          : r  z  Not rediscovered        : r  Z	Relevancez  Direct                  : rÛ   z  Strong                  : rÜ   z  Contextual              : rÝ   z  Low                     : rÞ   z  Direct + strong         : rê   z  Average score           : rë   z.4fzOutput written to         : )7r  r]   rU   rh   ÚosÚenvironrr   r‡   r�   rˆ   r›   rf   rš   rk   ÚAUTOMOTIVE_QUERIESÚEV_CHARGING_QUERIESrª   r­   rv   rw   rœ   rá   r³   r   r4   r·   rO   r½   r£   r  r#   r  r!   r"   r   Z	min_starsrP   r  Úexistsr…   rÑ   rð   r   ri   rT   r    rø   ÚoutputÚparentÚmkdirrÍ   rƒ   ÚdumpÚwriteÚprint)Úargsrh   rj   rñ   r  ZprofilesZraw_hitsr™   ró   r˜   rq   r«   r®   r²   rt   Z
discoveredZpre_filter_countZfilteredZexisting_namesZfiltered_namesZoverlapr  r  ZstatsZ
comparisonr  rÏ   rG   rH   r6   r6   r7   Úmain,  sz   

ÿýÿÿ
ÿÿýþþûýø

ý

þÿûÿþÿþþÿÿþÿÿÿÿÿÿÿÿÿÿÿóíýûÿùþË;þþü*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿr   Ú__main__z
Interrupted.)Úfileé‚   zFatal error: %srW   )>Ú__doc__Z
__future__r   r   rƒ   rZ   r  Úsysr‹   Zurllib.errorr   Zurllib.parseZurllib.requestÚcollectionsr   Zdataclassesr   r   r   r   Úpathlibr   Útypingr	   ri   Z
GITHUB_APIr¦   r  Z	getLoggerr‡   r  r  rA   rB   rC   r   rT   r]   rd   rg   rk   ru   r–   r­   r³   r·   rÄ   r¼   rÑ   rI   rð   rø   r  r   rK   Ú
SystemExitÚKeyboardInterruptr  Ústderrr†   r•   Z	exceptionr6   r6   r6   r7   Ú<module>   sˆ   ?

% 	" )qL<L$ i 3    
þþ