a
    R|ðj Ê  ã                
   @  s"  d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZ ddlmZmZ ddlmZ ddlmZ dZd	Zd
ZedƒZedƒZedƒZddgZdZdZdZ dZ!dZ"e #d¡Z$ddœdd„Z%dddœdd„Z&ddd œd!d"„Z'ddd œd#d$„Z(ddd œd%d&„Z)dd'd œd(d)„Z*ddd œd*d+„Z+dšdddd-œd.d/„Z,d0ddd1œd2d3„Z-d0dd4œd5d6„Z.ddd œd7d8„Z/dd9d:œd;d<„Z0ddd:œd=d>„Z1ddd:œd?d@„Z2d›dBddCdDœdEdF„Z3ddGdHœdIdJ„Z4dGdBdddKœdLdM„Z5ddBddddNdOœdPdQ„Z6ddBdddRdSœdTdU„Z7ddBdddVdSœdWdX„Z8ddYdZœd[d\„Z9dGdd]œd^d_„Z:dGd'd]œd`da„Z;dYdbdcdYddœdedf„Z<d0ddgdGdhœdidj„Z=d0dGd4œdkdl„Z>d0dGddmœdndo„Z?ddgd œdpdq„Z@dGdGdrœdsdt„ZAdddudvœdwdx„ZBddBdyœdzd{„ZCdGddd|œd}d~„ZDdGdGddddBddGd€œd�d‚„ZEdGdBddddGdƒœd„d…„ZFdYddGdGd†œd‡dˆ„ZGd0dgddGdGddGd‰œdŠd‹„ZHdYdgdŒœd�dŽ„ZId�dœd�d‘„ZJddœd’d“„ZKeLd”k�rzeMeKƒ ƒ‚W nb eN�yä   eOd•e	jPd–� eMd—ƒ‚Y n: eQ�y ZR z e$ Sd˜eR¡ eMd™ƒ‚W Y dZR[Rn
dZR[R0 0 dS )œaá  
github_intel.py

Build a normalized GitHub repository evidence dataset for the NCRC
Automotive Cybersecurity Threat Register.

Input
-----
github_probe.json

The input is produced by github_probe.py and contains discovered GitHub
repositories with deterministic relevance classifications.

Selection
---------
Only repositories with relevance levels:

    direct
    strong

are selected by default.

Processing
----------
For each selected repository:

1. Refresh repository metadata through the GitHub REST API.
2. Retrieve the repository README as raw text.
3. Preserve discovery classification and relevance information.
4. Normalize the result into a paper-like evidence structure.
5. Write checkpoint data during processing.
6. Write the final dataset to github/github.json.

The script does NOT use AI or LLM analysis.

Authentication
--------------
Set a GitHub personal access token:

    export GITHUB_TOKEN="github_pat_..."

The token is never written to output.

Examples
--------

Standard run:

    python3 github_intel.py -v

Resume an interrupted run:

    python3 github_intel.py --resume -v

Force a clean rebuild:

    python3 github_intel.py --force -v

Test only 10 repositories:

    python3 github_intel.py --limit 10 -v

Select contextual repositories too:

    python3 github_intel.py         --levels direct strong contextual         -v

Validate input without API retrieval:

    python3 github_intel.py --validate-only

Show detailed logging:

    python3 github_intel.py -vv
é    )ÚannotationsN)ÚCounter)ÚdatetimeÚtimezone)ÚPath)ÚAnyz1.0.0z1.0zhttps://api.github.comzgithub_probe.jsonzgithub/github.jsonzgithub/github_checkpoint.jsonÚdirectÚstrongi€8 é
   gš™™™™™É?é   é   Zgithub_intelÚstr)Úreturnc                   C  s    t  tj¡jdd� ¡  dd¡S )Nr   )Zmicrosecondz+00:00ÚZ)r   Únowr   ZutcÚreplaceZ	isoformat© r   r   úgithub_intel.pyÚutc_now„   s    ÿ
ýÿr   ÚintÚNone)Úverboser   c                 C  s8   | dkrt j}n| dkr t j}nt j}t j|dd� d S )Né   é   z'%(asctime)s %(levelname)-8s %(message)s)ÚlevelÚformat)ÚloggingÚDEBUGÚINFOZWARNINGZbasicConfig)r   r   r   r   r   Úconfigure_logging�   s    þr   r   )Úvaluer   c                 C  s.   | d u rdS d  t| ƒ dd¡ dd¡ ¡ ¡S )NÚ ú úÚ
)Újoinr   r   Úsplit©r    r   r   r   Ú
clean_text›   s    ÿþÿr(   c                 C  s‚   | d u rdS t | ƒ dd¡ dd¡}| dd¡}dd„ | ¡ D ƒ}|r^|d  ¡ s^| d¡ qB|rx|d	  ¡ sx| ¡  q^d |¡S )
Nr!   z
r$   r#   ú c                 S  s   g | ]}|  ¡ ‘qS r   )Úrstrip)Ú.0Úliner   r   r   Ú
<listcomp>¯   s   ÿz,normalize_multiline_text.<locals>.<listcomp>r   éÿÿÿÿ)r   r   Ú
splitlinesÚstripÚpopr%   )r    ÚtextÚlinesr   r   r   Únormalize_multiline_text§   s    þ
r4   c              	   C  s,   zt | p
dƒW S  ttfy&   Y dS 0 d S )Nr   )r   Ú	TypeErrorÚ
ValueErrorr'   r   r   r   Úsafe_int½   s    r7   Úfloatc              	   C  s,   zt | p
dƒW S  ttfy&   Y dS 0 d S )Nr   ç        )r8   r5   r6   r'   r   r   r   Ú
safe_floatÄ   s    r:   c                 C  s   t  | jddd�¡ ¡ S )Núutf-8r   ©Úerrors)ÚhashlibZsha256ÚencodeZ	hexdigestr'   r   r   r   Ústable_hashË   s    þÿr@   é   )r    Úlengthr   c                 C  s   t | ƒd |… S ©N)r@   )r    rB   r   r   r   Úcompact_hashÔ   s    rD   r   )ÚpathÚpayloadr   c                 C  sx   | j jddd� |  | jd ¡}|jddd��,}tj||ddd	� | d
¡ W d   ƒ n1 s`0    Y  | | ¡ d S )NT)ÚparentsÚexist_okz.tmpÚwr;   ©Úencodingr   F)ÚindentZensure_asciir$   )	ÚparentÚmkdirZwith_suffixÚsuffixÚopenÚjsonÚdumpÚwriter   )rE   rF   Z	temporaryÚhandler   r   r   Úatomic_write_jsonØ   s(    þÿþü(rU   )rE   r   c                 C  s<   | j ddd��}t |¡W  d   ƒ S 1 s.0    Y  d S )NÚrr;   rJ   )rP   rQ   Úload)rE   rT   r   r   r   Ú	load_jsonô   s    þrX   c                 C  sÀ   t | ƒ d¡}| ¡  d¡r(|dd … }| d¡rB|tdƒd … }| d¡r\|tdƒd … }dd„ | d¡D ƒ}t|ƒdk r€d	S |d
 }|d }| d¡r¦|d d… }|r®|s²d	S |› d|› �S )Nú/úgithub:é   úhttps://github.com/zhttp://github.com/c                 S  s   g | ]}|r|‘qS r   r   )r+   Zpiecer   r   r   r-     s   þz'normalize_full_name.<locals>.<listcomp>r   r!   r   r   z.gitéüÿÿÿ)r(   r0   ÚlowerÚ
startswithÚlenr&   Úendswith)r    r2   ÚpiecesÚownerÚ
repositoryr   r   r   Únormalize_full_name   s&    

þ
re   ztuple[str, str])Ú	full_namer   c                 C  s2   t | ƒ}|std| ›�ƒ‚| dd¡\}}||fS )Nz$Invalid GitHub repository identity: rY   r   )re   r6   r&   )rf   Ú
normalizedrc   rd   r   r   r   Úsplit_full_name!  s    ÿÿþrh   c                 C  s8   t | ƒ\}}t› dtjj|dd�› dtjj|dd�› �S )Nz/repos/r!   )ZsaferY   )rh   Ú
GITHUB_APIÚurllibÚparseZquote)rf   rc   rd   r   r   r   Úrepository_api_url5  s    ÿÿþÿrl   c                 C  s   t | ƒd S )Nz/readme)rl   ©rf   r   r   r   Úreadme_api_urlC  s    rn   úapplication/vnd.github+jsonz
str | Nonezdict[str, str])ÚtokenÚacceptr   c                 C  s(   |dt › �ddœ}| r$d| › �|d< |S )NzNCRC-GitHub-Intel/z
2022-11-28)ZAcceptz
User-AgentzX-GitHub-Api-VersionzBearer ZAuthorization)ÚSCRIPT_VERSION)rp   rq   Úheadersr   r   r   Úbuild_headersM  s    ýrt   zdict[str, Any])rs   r   c              
   C  s€   i }ddddddœ}|  ¡ D ]^\}}|  |¡}|d u r8q|dv rrzt|ƒ||< W qz ttfyn   |||< Y qz0 q|||< q|S )NzX-RateLimit-LimitzX-RateLimit-RemainingzX-RateLimit-UsedzX-RateLimit-ResetzX-RateLimit-Resource)ÚlimitÚ	remainingÚusedÚresetÚresource>   rw   rv   rx   ru   )ÚitemsÚgetr   r5   r6   )rs   ÚresultZmappingsÚkeyZheader_namer    r   r   r   Úparse_rate_headers]  s$    û

r~   )ÚrateÚretry_afterÚattemptr   c              	   C  sˆ   |r,zt dt|ƒƒW S  ttfy*   Y n0 |  d¡}|rvz$tt ¡ ƒ}t dt|ƒ| d ƒW S  ttfyt   Y n0 td|d  dƒS )Nr   rx   r   é<   i,  )Úmaxr   r5   r6   r{   ÚtimeÚmin)r   r€   r�   rx   r   r   r   r   Úcalculate_rate_wait€  s,    þ
þ
þr†   z,tuple[bytes, dict[str, str], dict[str, Any]])Úurlrp   rq   ÚtimeoutÚretriesr   c                 C  s  d }t |d ƒD �]Ú}t||d�}tjj| |dd�}zŠtjj||d��f}	|	 ¡ }
dd„ |	j ¡ D ƒ}t	|	jƒ}t
 d| | d	d
¡| dd
¡¡ |
||fW  d   ƒ W   S 1 s²0    Y  W q tjj�yˆ } �zª|}t	|jƒ}z| ¡ jddd�}W n t�y   d}Y n0 d}|�rZzt |¡}t| d|¡ƒ}W n t�yX   t|ƒ}Y n0 |jdk�rvtd| › �ƒ|‚|jdv �rø||k�r–tdƒ|‚t||j d¡|d�}t
 d|j| |�pÀd¡ t
 d||d |¡ t |¡ t
 d¡ W Y d }~qd|j  k�rdk �rVn n@||k�r"‚ td| dƒ}t
 d|j|¡ t |¡ W Y d }~qtd |j› d!|�pj| › �ƒ|‚W Y d }~qd }~0  tjjtf�yê } zB|}||k�r°‚ td| dƒ}t
 d"||¡ t |¡ W Y d }~qd }~0 0 q|�rø|‚td#ƒ‚d S )$Nr   )rp   rq   ZGET)r‡   rs   Úmethod)rˆ   c                 S  s   i | ]\}}||“qS r   r   )r+   r}   r    r   r   r   Ú
<dictcomp>Á  s   ÿz"github_request.<locals>.<dictcomp>z7GitHub request succeeded: %s (remaining=%s resource=%s)rv   ú?ry   r;   r   r<   r!   Úmessagei”  zGitHub resource not found: >   i“  i­  z+GitHub rate limit or retry budget exhaustedzRetry-After)r   r€   r�   zGitHub HTTP %d for %s: %szrate limit or secondary limitz/Waiting %d seconds before retry (attempt %d/%d)z"Retrying GitHub request after waitiô  iX  r   r   z3GitHub server error HTTP %d; retrying in %d secondszGitHub HTTP z: z1GitHub request failed: %s; retrying in %d secondsz*GitHub request failed without an exception)Úrangert   rj   ÚrequestZRequestZurlopenÚreadrs   rz   r~   ÚLOGÚdebugr{   ÚerrorZ	HTTPErrorÚdecodeÚ	ExceptionrQ   Úloadsr(   ÚcodeÚFileNotFoundErrorÚRuntimeErrorr†   Úwarningr„   ÚsleepÚinfor…   ZURLErrorÚTimeoutError)r‡   rp   rq   rˆ   r‰   Z
last_errorr�   rs   r�   ZresponseÚbodyZresponse_headersr   ÚexcZ
error_bodyZerror_messageZerror_payloadZwait_forr   r   r   Úgithub_request¡  s   þýþýÿ

û	ý,ÿþ

ÿþÿÿ
ÿþ
ÿþÿûûû
ÿ
þü

ÿÿýþ
þü"ÿr    z%tuple[dict[str, Any], dict[str, Any]])r‡   rp   rˆ   r‰   r   c           	   
   C  sˆ   t | |d||d�\}}}zt |jddd�¡}W n6 tjyf } ztd| › �ƒ|‚W Y d }~n
d }~0 0 t|tƒs€td| › �ƒ‚||fS )Nro   ©r‡   rp   rq   rˆ   r‰   r;   r   r<   z$Invalid JSON returned by GitHub for z$Unexpected GitHub JSON response for )r    rQ   r–   r”   ZJSONDecodeErrorr™   Ú
isinstanceÚdict)	r‡   rp   rˆ   r‰   rž   Ú_r   rF   rŸ   r   r   r   Úgithub_get_json]  s2    	ûþÿÿþ
ÿr¥   z*tuple[str, dict[str, str], dict[str, Any]]c                 C  sŠ   t | |d||d�\}}}| dd¡}d}tjd|tjd�}	|	rH|	 d¡}z|j|d	d
�}
W n  tyz   |jdd	d
�}
Y n0 t|
ƒ||fS )Nzapplication/vnd.github.raw+jsonr¡   zContent-Typer!   r;   zcharset=([A-Za-z0-9._-]+))Úflagsr   r   r<   )	r    r{   ÚreÚsearchÚ
IGNORECASEÚgroupr”   ÚLookupErrorr4   )r‡   rp   rˆ   r‰   rž   rs   r   Zcontent_typeÚcharsetÚmatchr2   r   r   r   Úgithub_get_raw_text‚  sB    
ûþý
þ
þýr®   zlist[dict[str, Any]])rF   r   c                 C  s<   t | tƒstdƒ‚|  d¡}t |tƒs.tdƒ‚dd„ |D ƒS )Nz!Probe input must be a JSON objectÚrepositoriesz.Probe input must contain a 'repositories' listc                 S  s   g | ]}t |tƒr|‘qS r   ©r¢   r£   ©r+   rd   r   r   r   r-   Ë  s   
þz*get_probe_repositories.<locals>.<listcomp>)r¢   r£   r6   r{   Úlist)rF   r¯   r   r   r   Úget_probe_repositoriesº  s    
ÿÿ
ÿþr³   )rd   r   c                 C  s*   |   d¡}t|tƒsdS t|  d¡ƒ ¡ S )NÚ	relevancer!   r   )r{   r¢   r£   r(   r^   ©rd   r´   r   r   r   Úrepository_relevance_levelÒ  s    ÿ
ÿr¶   c                 C  s&   |   d¡}t|tƒsdS t|  d¡ƒS )Nr´   r9   Úscore)r{   r¢   r£   r:   rµ   r   r   r   Úrepository_relevance_scoreá  s    ÿ
ÿr¸   zset[str]z
int | None)r¯   Úlevelsru   r   c           	      C  sÎ   g }t ƒ }| D ]”}t|ƒ}||vr$qt| d¡pB| d¡pB| d¡ƒ}|slt d| d¡pf| d¡pfd¡ q| ¡ }||v r~qt|ƒ}||d< | |¡ | 	|¡ q|j
dd„ d	d
� |d urÊ|d |… }|S )Nrf   Úidr‡   z/Skipping repository without valid full_name: %sÚnameÚunknownc                 S  s$   t | ƒt|  d¡ƒ|  dd¡ ¡ fS )NÚstarsrf   r!   )r¸   r7   r{   r^   )rd   r   r   r   Ú<lambda>  s    ÿÿþùz%select_repositories.<locals>.<lambda>T©r}   Úreverse)Úsetr¶   re   r{   r‘   rš   r^   r£   ÚappendÚaddÚsort)	r¯   r¹   ru   ÚselectedÚseenrd   r   rf   r}   r   r   r   Úselect_repositoriesð  sJ    ÿ
ÿý
ÿü
órÇ   z	list[str])Ú
input_fileÚselected_countr¹   r   c              
   C  s    t ttƒ tƒ t| ƒ||i i dœ	S )N)	Úschema_versionÚscript_versionÚ
created_atÚ
updated_atrÈ   rÉ   Zselected_levelsÚ	completedr=   )ÚSCHEMA_VERSIONrr   r   r   ©rÈ   rÉ   r¹   r   r   r   Úcheckpoint_template5  s    ÷rÑ   c                 C  s\   t | ƒ}t|tƒs td| › �ƒ‚| d¡}t|tƒs<i |d< | d¡}t|tƒsXi |d< |S )Nz"Checkpoint must be a JSON object: rÎ   r=   )rX   r¢   r£   r6   r{   )rE   rF   rÎ   r=   r   r   r   Úload_checkpointG  s     
ÿÿ
ÿ
rÒ   )rE   Ú
checkpointr   c                 C  s   t ƒ |d< t| |ƒ d S )NrÍ   )r   rU   )rE   rÓ   r   r   r   Úsave_checkpointb  s
    
þrÔ   c                 C  s$   t | tƒsg S dd„ | D ƒ}t|ƒS )Nc                 S  s    h | ]}t |ƒrt |ƒ ¡ ’qS r   )r(   r^   )r+   Ztopicr   r   r   Ú	<setcomp>x  s   þz#normalize_topics.<locals>.<setcomp>)r¢   r²   Úsorted)r    Útopicsr   r   r   Únormalize_topicsr  s    
þrØ   )Úmetadatar   c              !   C  sØ  |   d¡}t|tƒsi }|   d¡}t|tƒs0i }t|   d¡ƒ}t|   d¡ƒt|   d¡ƒpZd |t|   d¡ƒt|  d¡ƒt|  d¡ƒp„d t|   d	¡ƒt|   d
¡ƒt|   d¡ƒt|   d¡ƒp¸d t|   d¡ƒt|   d¡ƒpÔd t|   d¡ƒpäd t|   d¡ƒt|   d¡ƒt|   d¡�p|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|   d¡ƒt|  d¡ƒ�pºt|  d¡ƒ�pºd |   d ¡|   d!¡|   d"¡d#œS )$Nrc   Úlicenserf   rº   Únode_idr»   ZloginÚtypeZhtml_urlr‡   ÚdescriptionÚhomepager×   ÚlanguageÚdefault_branchZstargazers_countZforks_countZsubscribers_countZwatchers_countZopen_issues_countÚsizeÚforkÚarchivedÚdisabledÚis_templateÚ
has_issuesÚhas_projectsÚhas_wikiÚ	has_pagesÚhas_discussionsZspdx_idrÌ   rÍ   Ú	pushed_at)Ú	github_idrÛ   rf   r»   rc   Ú
owner_typer‡   Úapi_urlrÝ   rÞ   r×   rß   rà   r½   ÚforksÚwatchersÚopen_issuesÚsize_kbrâ   rã   rä   rå   ræ   rç   rè   ré   rê   rÚ   rÌ   rÍ   rë   )r{   r¢   r£   re   r7   r(   rØ   Úbool)rÙ   rc   Zlicense_datarf   r   r   r   Únormalize_repository_metadata€  sà    



ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿþÿÿÿÿÿÿÿÿÿÿÿÿÿýÿÿÿ rô   ztuple[str, bool])r2   Úmaximum_charsr   c                 C  sR   |dkrdt | ƒfS t| ƒ|kr(| dfS d}td|t|ƒ ƒ}| d |… | dfS )Nr   r!   Fz'

[README truncated by github_intel.py]T)ró   r`   rƒ   )r2   rõ   rO   Zallowedr   r   r   Útruncate_readme÷  s    ÿ
þrö   )Úreadmer   c                 C  sB   |   ¡ D ]4}| ¡ }| d¡r| d¡ ¡ }|rt|ƒ  S qd S )Nú#)r/   r0   r_   Úlstripr(   )r÷   r,   ÚstrippedÚtitler   r   r   Úreadme_title  s    
ÿrü   )rÙ   r÷   r   c                 C  sD   |   d¡|   d¡|   d¡d |   d¡p(g ¡|g}d dd„ |D ƒ¡S )	Nrf   r»   rÝ   r"   r×   r$   c                 s  s   | ]}|rt |ƒV  qd S rC   )r   )r+   r    r   r   r   Ú	<genexpr>-  s   þz&combine_search_text.<locals>.<genexpr>)r{   r%   )rÙ   r÷   Úvaluesr   r   r   Úcombine_search_text  s    ÿù

þrÿ   ró   )Úprobe_repositoryrÙ   r÷   Úreadme_availableÚreadme_truncatedÚreadme_errorÚretrieved_atr   c              +   C  sZ  t |ƒ}| d¡pt|  d¡ƒ}|s,tdƒ‚| d¡pBt|  d¡ƒ}	tt| d¡pTg ƒtt|  d¡ƒƒB ƒ}
|
|d< |	|d< |  d¡}t|t	ƒsœddg dœ}|  d	¡}t|t	ƒs´i }|  d
¡}t|t	ƒsÌi }t
||d�}| d¡pð| dd¡d }t|ƒpü|}d| ¡ › �dd|||| d¡| d¡�p0d|› �|	||
| d¡| d¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| d ¡| d!¡| d"¡| d#¡| d$¡||||t|ƒ|||�rÞt|ƒnd d%œ| d&¡| d'¡| d(¡| d)¡| d*d¡| d+d¡| d,d¡| d-d¡| d.d¡| d/d¡d0œ
d1d2|d3œt|ƒd4œ!S )5Nrf   z3Unable to determine normalized repository full_namerÝ   r×   r´   r9   r¼   )r·   r   ZreasonsÚclassificationÚ	discovery)rÙ   r÷   r»   rY   r   rZ   rd   rc   r‡   r\   rß   rÚ   r½   r   rï   rð   rñ   rò   râ   Frã   rä   rà   rÞ   rÌ   rÍ   rë   )Ú	availableZ
charactersÚ	truncatedr“   Zcontent_hashrì   rÛ   rí   rî   rå   ræ   rç   rè   ré   rê   )
rì   rÛ   rí   rî   rå   ræ   rç   rè   ré   rê   ZGitHubZpublic_repository)r»   rÜ   r  )!rº   Zevidence_typeZrecord_typerû   rf   r»   rc   r‡   rÝ   r÷   r×   rß   rÚ   r½   rï   rð   rñ   rò   râ   rã   rä   rà   rÞ   rÌ   rÍ   rë   r  r´   r  Úreadme_metadataZrepository_metadataÚsourceZsearch_text_hash)rô   r{   re   r6   r(   rÖ   rÁ   rØ   r¢   r£   rÿ   r&   rü   r^   r`   r@   )r   rÙ   r÷   r  r  r  r  rg   rf   rÝ   r×   r´   r  r  Zsearch_textZrepository_nameZdisplay_titler   r   r   Úbuild_record4  s~   	ÿ
ÿÿþ	ÿ
ÿÿþ	ÿÿÿÿýÿÿ
ýÿ
ÿ
þ
þýþ	þÿÿþþþþþþþþÿÿÿÿÿÿøÿÿÿÿþþþþþþß(ýÿ€ ör  )r   rp   rˆ   r‰   Úreadme_max_charsr   c              
   C  s  t |  d¡ƒ}|stdƒ‚t d|¡ tt|ƒ|||d�\}}d}d}	d}
d }zBt d|¡ tt|ƒ|||d�\}}}t	||d�\}}
t
|ƒ}	W nZ ty²   d	}t d
|¡ Y n: tyê } z"t|ƒ}t d||¡ W Y d }~n
d }~0 0 t| |||	|
|tƒ d�S )Nrf   z'Probe repository has no valid full_namezFetching metadata: %s)r‡   rp   rˆ   r‰   r!   FzFetching README: %s)r2   rõ   zREADME not foundzREADME not found: %sz"README retrieval failed for %s: %s)r   rÙ   r÷   r  r  r  r  )re   r{   r6   r‘   rœ   r¥   rl   r®   rn   rö   ró   r˜   r•   r(   rš   r  r   )r   rp   rˆ   r‰   r  rf   rÙ   r¤   r÷   r  r  r  Z
raw_readmerŸ   r   r   r   Úprocess_repository'  s„    ÿÿÿþÿú
	þÿú
þÿþýùr  )ÚrecordsrÉ   r=   r   c                 C  sŠ  t dd„ | D ƒƒ}t dd„ | D ƒƒ}t ƒ }| D ]Œ}| d¡}t|tƒsHq.| d¡rb|d  d7  < | d¡r||d  d7  < | d¡r | d¡r |d  d7  < | d	¡r.|d	  d7  < q.|t| ƒt|ƒtd
d„ | D ƒƒtdd„ | D ƒƒtdd„ | D ƒƒtdd„ | D ƒƒtdd„ | D ƒƒ| dd¡| dd¡| dd¡| d	d¡dœ| dd¡| dd¡| dd¡| dd¡dœdd„ | d¡D ƒdœS )Nc                 s  s*   | ]"}t | d ¡pi  d¡ƒ ¡ V  qdS )r´   r   N)r(   r{   r^   ©r+   Úrecordr   r   r   rý   ‰  s   ú
þýÿz$output_statistics.<locals>.<genexpr>c                 s  s*   | ]"}t | d ¡ƒrt | d ¡ƒV  qdS )rß   N)r(   r{   r  r   r   r   rý   “  s   ÿüÿr  Ú
automotiver   Úev_chargingÚautomotive_and_ev_chargingÚsecurity_relatedc                 s  s&   | ]}|  d ¡pi   d¡rdV  qdS ©r	  r  r   N©r{   r  r   r   r   rý   Ð  s   ÿüúþc                 s  s&   | ]}|  d ¡pi   d¡sdV  qdS r  r  r  r   r   r   rý   Ý  s   ÿüúþc                 s  s&   | ]}|  d ¡pi   d¡rdV  qdS )r	  r  r   Nr  r  r   r   r   rý   ê  s   ÿüúþc                 s  s   | ]}|  d ¡rdV  qdS )rã   r   Nr  r  r   r   r   rý   ÷  s
   ÿþc                 s  s   | ]}|  d ¡rdV  qdS )râ   r   Nr  r  r   r   r   rý   ÿ  s
   ÿþr   )r  r  r  r  r   r	   Ú
contextualÚlow©r   r	   r  r  c                 S  s   g | ]\}}||d œ‘qS ))rß   Úcountr   )r+   rß   r  r   r   r   r-   1  s   ýþz%output_statistics.<locals>.<listcomp>é   )Úselected_from_probeÚrecords_writtenÚprocessing_errorsÚreadmes_availableÚreadmes_missingÚreadmes_truncatedZrepositories_archivedZrepositories_forksr  Úrelevance_levelsZtop_languages)r   r{   r¢   r£   r`   ÚsumÚmost_common)r  rÉ   r=   r"  Z	languagesZclassificationsr  r  r   r   r   Úoutput_statistics„  sÚ    ù
ü
ÿþÿÿþÿÿþÿÿÿüÿþÿÿþþþþþþ	þþþþðþþþþóúšr%  )Ú
input_pathr¹   rÉ   rÎ   r=   Ú
started_atr   c                 C  sT   dd„ |  ¡ D ƒ}|jdd„ dd� tttƒ |dt| ƒ||dœt|||d	�||d
œ
S )Nc                 S  s   g | ]}t |tƒr|‘qS r   r°   r  r   r   r   r-   D  s   
þz build_output.<locals>.<listcomp>c                 S  s6   t |  d¡pi  d¡ƒt|  d¡ƒt|  d¡ƒ ¡ fS )Nr´   r·   r½   rf   )r:   r{   r7   r(   r^   )r  r   r   r   r¾   K  s    
þýÿÿÿözbuild_output.<locals>.<lambda>Tr¿   z\Normalized GitHub repository evidence for the NCRC Automotive Cybersecurity Threat Register.)r¹   rÉ   )r  rÉ   r=   )
rÊ   rË   Zgenerated_atZcollection_started_atrÝ   Zsource_inputZ	selectionÚ
statisticsr=   r¯   )rþ   rÄ   rÏ   rr   r   r   r%  )r&  r¹   rÉ   rÎ   r=   r'  r  r   r   r   Úbuild_output<  s0    þñþýær)  )r¯   r   c           	      C  sØ   g }t ƒ }t| ƒD ]À\}}t| d¡p8| d¡p8| d¡ƒ}|sT| d|› d�¡ q| ¡ }||v rt| d|› �¡ | |¡ | d¡}t|tƒs¤| |› d�¡ qt	| d	¡ƒ ¡ }|d
vr| |› d|›�¡ q|S )Nrf   rº   r‡   zRepository row z has no valid full_namezDuplicate repository identity: r´   z: missing relevance objectr   >   r  r   r	   r  z: invalid relevance level )
rÁ   Ú	enumeratere   r{   rÂ   r^   rÃ   r¢   r£   r(   )	r¯   ZproblemsrÆ   Úindexrd   rf   r}   r´   r   r   r   r   Úvalidate_probe~  sR    ÿ
ÿý
ÿÿ
ÿþÿÿÿr,  zargparse.Namespacec                  C  sJ  t jdd�} | jdttdd� | jdttdd� | jdttd	d� | jd
dtg d¢dd� | jdtd dd� | jdtt	dt	› d�d� | jdtt
dt
› d�d� | jdttdt› d�d� | jdttdt› d�d� | jdttdt› d�d� | jdd dd� | jdd d!d"� | jd#d d$d"� | jd%d d&d"� | jd'd(d)d*d+d,� |  ¡ S )-Nz[Build github/github.json from direct and strong repositories discovered by github_probe.py.)rÝ   z--inputz-Input probe JSON (default: github_probe.json))rÜ   ÚdefaultÚhelpz--outputz/Final output JSON (default: github/github.json)z--checkpointz8Checkpoint JSON (default: github/github_checkpoint.json)z--levelsú+r  z4Relevance levels to include (default: direct strong))Únargsr-  Úchoicesr.  z--limitz3Optional maximum number of repositories for testingz--readme-max-charsz:Maximum README characters stored per repository (default: ú)z--checkpoint-intervalzGSave checkpoint after this many newly processed repositories (default: z--request-delayz0Delay between repositories in seconds (default: z	--timeoutz"HTTP timeout in seconds (default: z	--retriesz:Maximum retries for API or rate-limit responses (default: z--tokenz;GitHub token. Prefer the GITHUB_TOKEN environment variable.)r-  r.  z--resumeÚ
store_truez"Resume from an existing checkpoint)Úactionr.  z--forcez/Ignore and overwrite existing checkpoint/outputz--validate-onlyz2Validate input and selection without API retrievalz-vz	--verboser  r   z5Increase logging verbosity. Use -vv for debug output.)r4  r-  r.  )ÚargparseÚArgumentParserÚadd_argumentr   ÚDEFAULT_INPUTÚDEFAULT_OUTPUTÚDEFAULT_CHECKPOINTÚDEFAULT_LEVELSr   ÚDEFAULT_README_MAX_CHARSÚDEFAULT_CHECKPOINT_INTERVALr8   ÚDEFAULT_REQUEST_DELAYÚDEFAULT_TIMEOUTÚDEFAULT_RETRIESÚ
parse_args)Úparserr   r   r   rA  ¾  sÌ    þû
û
û
õû
ÿû
ÿû
ÿû
ÿû
ÿû
ü	üüüúrA  c                    s  t ƒ } t| jƒ | j ¡ s,t d| j¡ dS | jdk r>tdƒ‚| j	dk rPtdƒ‚| j
dk rbtdƒ‚| jdk rttdƒ‚t| jƒ}t|ƒ}t|ƒ}|rÀt d	t|ƒ¡ |d d
… D ]}t d|¡ q®dd„ | jD ƒ}t|t|ƒ| jd�}t dt|ƒ¡ t dd |¡¡ t dt|ƒ¡ | j�r–tƒ  tdƒ tdƒ tƒ  tdt|ƒ› �ƒ tdt|ƒ› �ƒ tdd |¡› �ƒ tdt|ƒ› �ƒ tƒ  tdƒ dS | j�p®tj d¡�p®d }|�rÂt d¡ n
t d¡ | j�r | j ¡ �rê| j ¡  | j  ¡ �r | j  ¡  | j  ¡ �r.| j!�s.| j�s.t d| j ¡ dS t"ƒ }| j!�rŠ| j ¡ �sZt d| j¡ dS t#| jƒ}	t dt|	 d i ¡ƒ¡ |	 d!|¡}nt$| jt|ƒ|d"�}	|	 %d i ¡‰ |	 %d#i ¡}
d$d%„ |D ƒ‰‡fd&d„ˆ D ƒ}|D ]}t d'|¡ ˆ  &|d ¡ �qÚ‡ fd(d„|D ƒ}t d)tˆ ƒ¡ t d*t|ƒ¡ d}t'|dd+�D �]\\}}t(| d,¡ƒ}| )¡ }t d-|t|ƒ|¡ z8t*||| j
| j	| jd.�}|ˆ |< |
 &|d ¡ |d7 }W nš t+�yö } z4t,|ƒ}||t"ƒ d/d0œ|
|< t d1|¡ W Y d }~nVd }~0  t-�yB } z4t,|ƒ}||t"ƒ d2d0œ|
|< t .d3|¡ W Y d }~n
d }~0 0 |dk�r€|| j dk�r€t/| j|	ƒ t d4tˆ ƒt|
ƒ¡ | j0dk�r<t1 2| j0¡ �q<t/| j|	ƒ t3| j|t|ƒˆ |
|d5�}t4| j |ƒ |d6 }tƒ  td7ƒ td8ƒ tƒ  td9|d: › �ƒ td;|d< › �ƒ td=|d> › �ƒ tƒ  td?|d@ › �ƒ tdA|dB › �ƒ tdC|dD › �ƒ tƒ  tdE|dF dG › �ƒ tdH|dF dI › �ƒ tƒ  tdJ|dK dL › �ƒ tdM|dK dN › �ƒ tdO|dK dP › �ƒ tƒ  tdQ| j › �ƒ tdR| j› �ƒ dS )SNzInput file not found: %sr   r   z(--checkpoint-interval must be at least 1r   z--retries cannot be negativez--timeout must be at least 1z%--readme-max-chars cannot be negativez"Probe validation found %d issue(s)é2   zValidation: %sc                 S  s   g | ]}|  ¡ ‘qS r   )r^   )r+   r   r   r   r   r-   ¡  s   ÿzmain.<locals>.<listcomp>)r¯   r¹   ru   zProbe repositories loaded: %dzSelected relevance levels: %sz, zRepositories selected: %dzGitHub intelligence validationz==============================zProbe repositories          : zSelected repositories       : zSelected levels             : zValidation issues           : zAValidation completed; no API requests or output writes performed.ZGITHUB_TOKENz'Using authenticated GitHub API requestsz:No GitHub token configured; using unauthenticated requestszPOutput already exists: %s. Use --force to rebuild or --resume with a checkpoint.z'Cannot resume; checkpoint not found: %sz-Resuming checkpoint with %d completed recordsrÎ   rÌ   rÐ   r=   c                 S  s,   h | ]$}t | d ¡ƒrt | d ¡ƒ ¡ ’qS rm   ©re   r{   r^   r±   r   r   r   rÕ   ,  s   ÿÿúÿÿzmain.<locals>.<setcomp>c                   s   g | ]}|ˆ vr|‘qS r   r   )r+   r}   )Úselected_keysr   r   r-   :  s   þz$Removing stale checkpoint record: %sc                   s&   g | ]}t | d ¡ƒ ¡ ˆ vr|‘qS rm   rD  r±   )rÎ   r   r   r-   J  s   ÿÿûþzAlready completed: %dzPending repositories: %d)Ústartrf   z[%d/%d] Processing %s)r   rp   rˆ   r‰   r  T)rf   r“   Zrecorded_atZfatalzRepository unavailable: %sFz#Repository processing failed for %sz)Checkpoint saved: %d completed, %d errors)r&  r¹   rÉ   rÎ   r=   r'  r(  zGitHub intelligence datasetz===========================zSelected from probe        : r  zRepositories written       : r  zProcessing errors          : r  zREADME available           : r  zREADME missing             : r   zREADME truncated           : r!  zDirect repositories        : r"  r   zStrong repositories        : r	   zAutomotive                 : r  r  zEV charging                : r  zAutomotive + EV charging   : r  zOutput written to          : zCheckpoint written to      : )5rA  r   r   ÚinputÚexistsr‘   r“   Zcheckpoint_intervalr6   r‰   rˆ   r  rX   r³   r,  rš   r`   r¹   rÇ   rÁ   ru   rœ   r%   Zvalidate_onlyÚprintrp   ÚosÚenvironr{   ZforcerÓ   ÚunlinkÚoutputZresumer   rÒ   rÑ   Ú
setdefaultr1   r*  re   r^   r  r˜   r(   r•   Ú	exceptionrÔ   Zrequest_delayr„   r›   r)  rU   )ÚargsZprobe_payloadZprobe_repositoriesZvalidation_problemsZproblemr¹   rÅ   rp   r'  rÓ   r=   Zstale_completedr}   ÚpendingZnewly_processedr+  rd   rf   r  rŸ   r�   rM  r(  r   )rÎ   rE  r   Úmainc  sÊ   ÿ
þÿÿ
ÿ
ÿ
ÿÿÿÿÿþþþýþþþÿÿÿÿÿÿÿÿÿÿÿûÿÿ

ÿþýüþÿþÿþ
þýþþú
þþþ
þþþþÿÿüú
þü
þü
þÿÿþþþüÿþú	þÿÿÿÿÿÿÿÿÿÿÿÿÿ
ÿÿ
ÿÿ
ÿÿ
ÿÿ
ÿÿÿÿÿÿrR  Ú__main__z#
Interrupted. Resume with --resume.)Úfileé‚   zFatal error: %sr   )rA   )ro   )TÚ__doc__Z
__future__r   r5  r>   rQ   r   rJ  r§   Úsysr„   Zurllib.errorrj   Zurllib.parseZurllib.requestÚcollectionsr   r   r   Úpathlibr   Útypingr   rr   rÏ   ri   r8  r9  r:  r;  r<  r=  r>  r?  r@  Z	getLoggerr‘   r   r   r(   r4   r7   r:   r@   rD   rU   rX   re   rh   rl   rn   rt   r~   r†   r    r¥   r®   r³   r¶   r¸   rÇ   rÑ   rÒ   rÔ   rØ   rô   rö   rü   rÿ   r  r  r%  r)  r,  rA  rR  Ú__name__Ú
SystemExitÚKeyboardInterruptrI  Ústderrr•   rŸ   rO  r   r   r   r   Ú<module>   s¾   Mþ
		! þ#! =%8Ew t] 9B@ &   D
ÿþþ