a
    ÔžójsÍ  ã                   @  s¨  d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZmZ dd
lmZmZ dZdZedƒZedƒZ edƒZ!dZ"dZ#dZ$dZ%dZ&dZ'e (dej)¡Z*e (dej)¡Z+e (dej)¡Z,e (dej)¡Z-e (dej)¡Z.e (dej)¡Z/e (dej)¡Z0e (dej)¡Z1e (dej)¡Z2e (dej)¡Z3e (dej)¡Z4G d d!„ d!e5ƒZ6G d"d#„ d#e6ƒZ7G d$d%„ d%ƒZ8G d&d'„ d'ƒZ9d(d)œd*d+„Z:d,d(d-œd.d/„Z;d­d,d1d(d2œd3d4„Z<d5d,d6d7œd8d9„Z=d5d:d;œd<d=„Z>d,d:d-œd>d?„Z?d,d@d-œdAdB„Z@d,d(d(dCœdDdE„ZAd,d(d(d(dFœdGdH„ZBd,d(d(dIœdJdK„ZCd:d(dLœdMdN„ZDd:dOdPœdQdR„ZEdSdTdUœdVdW„ZFd,dXd-œdYdZ„ZGd(d(d[d\œd]d^„ZHd(d[d_œd`da„ZId:d(dbœdcdd„ZJd(dedfœdgdh„ZKd:didbœdjdk„ZLdldOdmœdndo„ZMe (dpej)¡ZNe (dqej)¡ZOd,d[drœdsdt„ZPd,d(d[duœdvdw„ZQd:dTdxœdydz„ZRd:d(dxœd{d|„ZSd:d(d}d~œdd€„ZTdOd�d‚œdƒd„„ZUd(dOdOdOdOd…d†œd‡dˆ„ZVdedOdOdOdOdTd‰dŠœd‹dŒ„ZWd'd(d�dŽœd�d�„ZXd'd(d1d�d‘œd’d“„ZYd'd(d1d�d‘œd”d•„ZZdOdOd–œd—d˜„Z[dOdOdOdOdTd‰d™œdšd›„Z\dldTdmœdœd�„Z]d'd:d1d1d:džœdŸd „Z^d¡d)œd¢d£„Z_d¡d6d¤œd¥d¦„Z`dOdTd1dOd§œd¨d©„Zad1d)œdªd«„Zbecd¬k�r¤e	 debƒ ¡ dS )®a  
github_cve_fixes.py - enrich automotive CVEs with public GitHub remediation evidence.

Design goals
------------
* Keep the existing CVE pipeline untouched.
* Read zero_day_threats/zero_day.json.
* Write a separate zero_day_threats/cve_fixes.json.
* Read GITHUB_TOKEN from the environment; never write or print the token.
* Use only the Python standard library.
* Cache GitHub API responses and write output atomically.
* Distinguish mere CVE mentions from possible and verified fixes.

Evidence sources
----------------
1. GitHub Global Security Advisories, queried by CVE ID.
2. GitHub commit search, queried by CVE ID.
3. GitHub issue/pull-request search, queried by CVE ID.
4. URLs referenced by advisories, including commits, pull requests and releases.

Classification is deliberately conservative:
* verified_fix: explicit patched version in a GitHub advisory, an explicit fixing
  commit, or a merged pull request with explicit fix language.
* fix_candidate: fix/patch/remediation language is present but verification is
  incomplete.
* mention_only: the CVE is mentioned, but there is no reliable fix evidence.
* no_public_fix_found: no usable GitHub evidence was found.
* lookup_error: one or more required lookups failed and no stronger result exists.

Usage
-----
    python3 github_cve_fixes.py
    python3 github_cve_fixes.py --limit 10 --verbose
    python3 github_cve_fixes.py --refresh
    python3 github_cve_fixes.py --input zero_day_threats/zero_day.json         --output zero_day_threats/cve_fixes.json

Environment
-----------
    GITHUB_TOKEN=github_pat_...

The token needs access only to the GitHub resources you want to query. Public
advisory lookup itself can work without authentication, but this script requires
GITHUB_TOKEN so all searches use an authenticated request budget.
é    )ÚannotationsN)ÚCounter)ÚdatetimeÚtimezone)ÚPath)ÚAnyÚIterable)Ú	HTTPErrorÚURLError)Ú	urlencodeÚurlparse)ÚRequestÚurlopenzhttps://api.github.comz
2022-11-28zzero_day_threats/zero_day.jsonzzero_day_threats/cve_fixes.jsonz&zero_day_threats/github_cve_cache.jsoné¨   é   é   gš™™™™™ñ?zNCRC-github-cve-fixes/1.0z^CVE-\d{4}-\d{4,}$zI^https://github\.com/([^/]+)/([^/]+)/commit/([0-9a-f]{7,64})(?:[/?#].*)?$zF^https://github\.com/([^/]+)/([^/]+)/(?:pull|pulls)/(\d+)(?:[/?#].*)?$z>^https://github\.com/([^/]+)/([^/]+)/issues/(\d+)(?:[/?#].*)?$zM^https://github\.com/([^/]+)/([^/]+)/releases(?:/tag/([^/?#]+))?(?:[/?#].*)?$zC^https://github\.com/([^/]+)/([^/]+)/compare/([^/?#]+)(?:[/?#].*)?$zì\b(?:fix(?:e[ds])?|fixed|fixing|patch(?:e[ds])?|patched|patching|remediat(?:e[ds]?|ion)|resolv(?:e[ds]?|ing)|mitigat(?:e[ds]?|ion)|security\s+fix|backport(?:ed|ing)?|hotfix|correct(?:ed|ion)|address(?:e[ds]|ing)?|prevent(?:ed|s|ing)?)\bz�\b(?:not\s+fixed|unfixed|no\s+fix|without\s+(?:a\s+)?fix|workaround\s+only|false\s+positive|revert(?:ed|ing)?\s+(?:the\s+)?fix)\bz¬\b(?:security|vulnerab(?:ility|le)|exploit|cve|ghsa|advisory|overflow|use-after-free|injection|authentication|authorization|privilege|denial.of.service|memory.corruption)\bzt\b(?:regression\s+test|security\s+test|test\s+case|unit\s+test|integration\s+test|reproducer|proof.of.concept|poc)\bzd\b(?:fixed|patched|resolved|available)\s+(?:in|since|from)?\s*v?\d+(?:\.\d+){1,3}(?:[-+._a-z0-9]*)\bc                   @  s   e Zd ZdZdS )ÚGitHubAPIErrorzA GitHub API request failed.N©Ú__name__Ú
__module__Ú__qualname__Ú__doc__© r   r   ú-/mnt/bigstor/www/usecases/github_cve_fixes.pyr   „   s   r   c                   @  s   e Zd ZdZdS )ÚRateLimitErrorz:GitHub refused a request because a rate limit was reached.Nr   r   r   r   r   r   ˆ   s   r   c                   @  st   e Zd Zddddddœdd„Zdd	œd
d„Zedddœdd„ƒZdddœdd„Zddddœdd„Zdd	œdd„Z	dS )Ú	JsonCacheFr   ÚintÚboolÚNone)ÚpathÚ	ttl_hoursÚrefreshÚreturnc                 C  s4   || _ td|ƒd | _|| _i | _d| _|  ¡  d S )Nr   i  F)r   ÚmaxÚttl_secondsr!   ÚentriesÚdirtyÚload)Úselfr   r    r!   r   r   r   Ú__init__�   s    zJsonCache.__init__©r"   c              	   C  s”   | j  ¡ sd S zb| j jddd��}t |¡}W d   ƒ n1 s@0    Y  t|tƒrnt| d¡tƒrn|d | _W n t	tj
fyŽ   i | _Y n0 d S )NÚrúutf-8©Úencodingr%   )r   ÚexistsÚopenÚjsonr'   Ú
isinstanceÚdictÚgetr%   ÚOSErrorÚJSONDecodeError)r(   ÚhandleÚrawr   r   r   r'   •   s    
(zJsonCache.loadÚstr©Úurlr"   c                 C  s   t  |  d¡¡ ¡ S )Nr,   )ÚhashlibZsha256ÚencodeZ	hexdigest)r;   r   r   r   Úkey    s    zJsonCache.keyz
Any | Nonec                 C  sl   | j r
d S | j |  |¡¡}t|tƒs*d S | d¡}t|ttfƒsFd S | jrbt	 	¡ | | jkrbd S | d¡S )NÚ	stored_atÚdata)
r!   r%   r4   r>   r2   r3   r   Úfloatr$   Útime)r(   r;   Úentryr?   r   r   r   r4   ¤   s    

zJsonCache.getr   )r;   r@   r"   c                 C  s&   |t   ¡ |dœ| j|  |¡< d| _d S )N)r;   r?   r@   T)rB   r%   r>   r&   )r(   r;   r@   r   r   r   Úset±   s
    ýzJsonCache.setc                 C  sB   | j s
d S | jjjddd� dtƒ | jdœ}t| j|ƒ d| _ d S )NT©ÚparentsÚexist_oké   )Úschema_versionÚgenerated_isor%   F)r&   r   ÚparentÚmkdirÚutc_nowr%   Úatomic_write_json)r(   Úpayloadr   r   r   Úsave¹   s    ýzJsonCache.saveN)F)
r   r   r   r)   r'   Ústaticmethodr>   r4   rD   rP   r   r   r   r   r   Œ   s   r   c                	   @  sÄ   e Zd Zd*ddddddddœd	d
„Zdddœdd„Zddœdd„Zddœdd„Zd+ddddœdd„Zdddœdd„Zdddœdd„Z	e
d dd!œd"d#„ƒZe
dddd$œd%d&„ƒZd ddd'œd(d)„ZdS ),ÚGitHubClientFr9   r   rA   r   r   r   )ÚtokenÚcacheÚrequest_delayÚtimeoutÚretriesÚverboser"   c                 C  sX   || _ || _td|ƒ| _td|ƒ| _td|ƒ| _|| _d| _d| _d| _	d | _
d | _d S )Nç        é   r   )rS   rT   r#   rU   rV   rW   rX   Úlast_request_atÚrequestsÚ
cache_hitsÚrate_limit_remainingÚrate_limit_reset)r(   rS   rT   rU   rV   rW   rX   r   r   r   r)   Ç   s    	zGitHubClient.__init__)Úmessager"   c                 C  s   | j rtd|› �ƒ d S )Nz	[github] )rX   Úprint)r(   r`   r   r   r   Ú_logÜ   s    zGitHubClient._logr*   c                 C  s.   t  ¡ | j }| j| }|dkr*t  |¡ d S )Nr   )rB   Ú	monotonicr[   rU   Úsleep)r(   ÚelapsedÚwaitr   r   r   Ú_paceà   s    
zGitHubClient._pacezdict[str, str]c                 C  s   dd| j › �ttdœS )Nzapplication/vnd.github+jsonzBearer )ZAcceptZAuthorizationzX-GitHub-Api-Versionz
User-Agent)rS   ÚAPI_VERSIONÚ
USER_AGENT)r(   r   r   r   Ú_headersæ   s
    
üzGitHubClient._headersNúdict[str, Any] | Noner   )r   Úparamsr"   c                 C  sÞ  t dd„ |pi  ¡ D ƒdd�}t› |› �}|r:|d| 7 }| j |¡}|d urp|  jd7  _|  d|› �¡ |S d }t| jd ƒD �]@}|  	¡  t
||  ¡ dd	�}z¢|  d
|› �¡ t|| jd��n}	t ¡ | _|  jd7  _|  |	j¡ |	 ¡  d¡}
|
�rt |
¡nd }| j ||¡ |W  d   ƒ W   S 1 �s80    Y  W q‚ t�yP } zòt ¡ | _|  jd7  _|  |j¡ |  |¡}|  ||j¡}|jdv �rì|  ||¡�rì| j}|�rÌt j!|t"j#d� $¡ nd}t%d|› d|› d|› �ƒ|‚|jdk�r|| jk �r|}|  &|¡ W Y d }~q‚t'd|j› d|› d|› �ƒ|‚W Y d }~q‚d }~0  t(t)tj*f�yÂ } zRt ¡ | _|}|| jk �r˜|  &|¡ W Y d }~q‚t'd|› d|› �ƒ|‚W Y d }~q‚d }~0 0 q‚t'd|› d|› �ƒ‚d S )Nc                 S  s    g | ]\}}|d ur||f‘qS ©Nr   )Ú.0r>   Úvaluer   r   r   Ú
<listcomp>ð   ó    z$GitHubClient.get.<locals>.<listcomp>T)Zdoseqú?rH   zcache hit: ZGET)ÚheadersÚmethodzGET )rV   r,   )i“  i­  )ZtzÚunknownzGitHub rate limit reached for z; reset=z
; message=éô  zGitHub API returned HTTP z for ú: zGitHub request failed for )+r   ÚitemsÚAPI_ROOTrT   r4   r]   rb   ÚrangerW   rg   r   rj   r   rV   rB   rc   r[   r\   Ú_capture_rate_limitrs   ÚreadÚdecoder1   ÚloadsrD   r	   Ú_error_bodyÚ_error_messageÚreasonÚcodeÚ_is_rate_limitedr_   r   Zfromtimestampr   ÚutcÚ	isoformatr   Ú_retry_sleepr   r
   ÚTimeoutErrorr6   )r(   r   rl   Zqueryr;   ÚcachedZ
last_errorÚattemptZrequestÚresponser8   r@   ÚexcÚbodyr`   ÚresetZ
reset_textr   r   r   r4   î   s|    þ
.

ÿýÿÿý
ÿþ

.zGitHubClient.get)r‰   r"   c                 C  s8   t dd| t ¡  ƒ}|  d|d›d�¡ t |¡ d S )Ng      >@é   ztemporary failure; retry in z.1fÚs)ÚminÚrandomrb   rB   rd   )r(   r‰   Zdelayr   r   r   r†   0  s    zGitHubClient._retry_sleep)rs   r"   c              	   C  sl   |d u rd S zD|  d¡}|  d¡}|d ur2t|ƒnd | _|d urHt|ƒnd | _W n ttfyf   Y n0 d S )NúX-RateLimit-RemainingzX-RateLimit-Reset)r4   r   r^   r_   Ú	TypeErrorÚ
ValueError)r(   rs   Ú	remainingr�   r   r   r   r{   5  s    

z GitHubClient._capture_rate_limitr	   )r‹   r"   c              	   C  sH   z&|   ¡ jddd�}|r"t |¡ni W S  ttjfyB   i  Y S 0 d S )Nr,   Úreplace)Úerrors)r|   r}   r1   r~   r5   r6   )r‹   r8   r   r   r   r   @  s
    zGitHubClient._error_body)rŒ   Úfallbackr"   c                 C  s,   t | tƒr |  d¡r t| d ƒS t|p(dƒS )Nr`   zunknown error)r2   r3   r4   r9   )rŒ   r˜   r   r   r   r€   H  s    zGitHubClient._error_message)r‹   r`   r"   c                 C  sL   d }z|j  d¡}W n ty&   Y n0 t|ƒdkpJd| ¡ v pJd| ¡ v S )Nr’   Ú0z
rate limitzsecondary rate limit)rs   r4   ÚAttributeErrorr9   Úcasefold)r(   r‹   r`   r•   r   r   r   rƒ   N  s    
ÿ
ýzGitHubClient._is_rate_limited)F)N)r   r   r   r)   rb   rg   rj   r4   r†   r{   rQ   r   r€   rƒ   r   r   r   r   rR   Æ   s    ùBrR   r9   r*   c                   C  s   t  tj¡ ¡ S rm   )r   Znowr   r„   r…   r   r   r   r   rM   `  s    rM   r   )ro   r"   c                 C  s   t  ddt| pdƒ¡ ¡ S )Nz\s+ú Ú )ÚreÚsubr9   Ústrip©ro   r   r   r   Úcleand  s    r¢   rv   r   )ro   Úlimitr"   c                 C  s<   t | ƒ}t|ƒ|kr|S |d |d …  dd¡d  ¡ d S )Né   rœ   rH   r   z...)r¢   ÚlenÚrsplitÚrstrip)ro   r£   Útextr   r   r   Ú
short_texth  s    r©   r   r   )r   rO   r"   c                 C  s’   | j jddd� |  | jd ¡}|jddd��D}tj||dddd	� | d
¡ | ¡  t	 
| ¡ ¡ W d   ƒ n1 sx0    Y  t	 || ¡ d S )NTrE   z.tmpÚwr,   r-   rŽ   F)ÚindentZensure_asciiZ	sort_keysÚ
)rK   rL   Z	with_nameÚnamer0   r1   ÚdumpÚwriteÚflushÚosÚfsyncÚfilenor–   )r   rO   Z	temporaryr7   r   r   r   rN   o  s    
,rN   zdict[str, Any])r   r"   c                 C  sj   |   ¡ std| › �ƒ‚| jddd��}t |¡}W d   ƒ n1 sD0    Y  t|tƒsftd| › �ƒ‚|S )NzInput file not found: r+   r,   r-   zExpected a JSON object in )r/   ÚFileNotFoundErrorr0   r1   r'   r2   r3   r”   )r   r7   r@   r   r   r   Ú	load_jsonz  s    (
rµ   c                 C  s   t | tƒr| S i S )zBReturn value as a dict, or an empty dict for any other JSON shape.©r2   r3   r¡   r   r   r   Úas_dict„  s    r·   z	list[Any]c                 C  s2   | du rg S t | tƒr| S t | tƒr,t| ƒS | gS )z@Return value as a list; wrap scalar/dict values and ignore null.N)r2   ÚlistÚtupler¡   r   r   r   Úas_list‰  s    

rº   )ro   Úpreferred_keysr"   c                 G  s²   | du rdS t | tƒrt| ƒS t | tttfƒr6t| ƒS t | tƒr‚|pFd}|D ]0}|| v rL| | durLt| | ƒ}|rL|  S qLdS t | tt	t
fƒrªd tdd„ | D ƒƒ¡S t| ƒS )zCSafely normalize strings, numbers, booleans and common API objects.Nr�   )Ú
identifierr­   ro   r;   Úhtml_urlZloginÚ	full_nameú, c                 s  s   | ]}t |ƒV  qd S rm   )Úscalar_text)rn   Úxr   r   r   Ú	<genexpr>§  rq   zscalar_text.<locals>.<genexpr>)r2   r9   r¢   r   rA   r   r3   rÀ   r¸   r¹   rD   ÚjoinÚunique_strings)ro   r»   Úkeysr>   r¨   r   r   r   rÀ   ”  s"    


rÀ   )ro   r>   Úfallback_keysr"   c                 G  s@   t | ƒ}|g|¢R D ]&}||v rt| |¡ƒ}|r|  S qdS ©Nr�   )r·   rÀ   r4   )ro   r>   rÆ   ÚobjÚ	candidater¨   r   r   r   Ú	dict_text«  s    
rÊ   )ro   r   r"   c                 G  s0   | }|D ]}t |tƒs dS | |¡}qt|ƒS rÇ   )r2   r3   r4   rÀ   )ro   r   Zcurrentr>   r   r   r   Únested_textµ  s    
rË   )Úrecordr"   c                 C  sP   t | ƒ} t|  d¡p0|  d¡p0|  d¡p0|  d¡dddƒ ¡ }t |¡rL|S dS )Nr­   Úcve_idÚidÚcver�   )r·   rÀ   r4   ÚupperÚCVE_RXÚ	fullmatch)rÌ   ro   r   r   r   Úcve_id_from_record¾  s    
ÿþù	rÓ   zlist[dict[str, Any]])r@   r"   c                 C  s<   dD ]*}|   |¡}t|tƒrdd„ |D ƒ  S qtdƒ‚d S )N)ÚrecordsZcvesrx   Úvulnerabilitiesc                 S  s   g | ]}t |tƒr|‘qS r   r¶   ©rn   Úitemr   r   r   rp   Ð  rq   z!input_records.<locals>.<listcomp>zANo CVE records found; expected records/cves/items/vulnerabilities)r4   r2   r¸   r”   )r@   r>   ro   r   r   r   Úinput_recordsÌ  s
    

rØ   zIterable[Any]z	list[str])Úvaluesr"   c                 C  sH   g }t ƒ }| D ]4}t|ƒ}| ¡ }|r||vr| |¡ | |¡ q|S rm   )rD   r¢   r›   ÚappendÚadd)rÙ   ÚoutputÚseenro   r¨   r>   r   r   r   rÄ   Ô  s    
rÄ   rA   c              	   C  s(   z
t | ƒW S  ttfy"   Y dS 0 d S )NrY   )rA   r“   r”   r¡   r   r   r   Ú
safe_floatà  s    
rÞ   r   )r¨   rÍ   r"   c                 C  s   |  ¡ |   ¡ v S rm   )r›   )r¨   rÍ   r   r   r   Úhas_explicit_cveç  s    rß   )r¨   r"   c                 C  s   t t | ¡ƒot t | ¡ƒ S rm   )r   ÚFIX_RXÚsearchÚNEGATIVE_FIX_RX©r¨   r   r   r   Úfix_languageë  s    rä   )r×   r"   c                 C  s€   t | ƒ} |  d¡}t|tƒr8t| d¡ddƒ}|rR|S nt|tƒrRd|v rRt|ƒS t|  d¡dƒ}t d|¡}|r|| 	d¡S d	S )
NÚ
repositoryr¾   r­   ú/Zrepository_urlr;   z/repos/([^/]+/[^/]+)$rH   r�   )
r·   r4   r2   r3   rÀ   r9   r¢   rž   rá   Úgroup)r×   rå   r¾   Zrepo_urlÚmatchr   r   r   Úget_repo_from_api_itemï  s    

ré   rk   r:   c                 C  s8  t | ƒ} | sd S t | ¡}|rD| ¡ \}}}d|› d|› �|| dœS t | ¡}|r|| ¡ \}}}d|› d|› �t|ƒ| dœS t | ¡}|r´| ¡ \}}}d|› d|› �t|ƒ| dœS t | ¡}|rì| ¡ \}}}d|› d|› �|päd| d	œS t | ¡}|�r"| ¡ \}}}d
|› d|› �|| dœS dt	| ƒj
 ¡ | dœS )NÚcommit_referenceræ   )Úkindrå   Úshar;   Úpull_request_reference)rë   rå   Únumberr;   Zissue_referenceZrelease_referencer�   )rë   rå   Útagr;   Zcompare_reference)rë   rå   Ú
comparisonr;   Zexternal_reference)rë   Zhostr;   )r¢   ÚCOMMIT_URL_RXrè   ÚgroupsÚPULL_URL_RXr   ÚISSUE_URL_RXÚRELEASE_URL_RXÚCOMPARE_URL_RXr   Znetlocr›   )r;   rè   ÚownerÚreporì   rî   rï   rð   r   r   r   Úclassify_reference_urlþ  s^    
ü
ü
ü
ü
üýrù   ztuple[Any, ...]c                 C  s4   |   d¡|   d¡|   d¡|   d¡|   d¡|   d¡fS )Nrë   r;   rå   rì   rî   Úghsa_id©r4   ©r×   r   r   r   Úevidence_key<  s    úrý   zIterable[dict[str, Any]])rx   r"   c                 C  s>   g }t ƒ }| D ]*}t|ƒ}||v r$q| |¡ | |¡ q|S rm   )rD   rý   rÛ   rÚ   )rx   rÜ   rÝ   r×   r>   r   r   r   Údeduplicate_evidenceG  s    
rþ   a  (?:^|/)(?:cvelistv5|cvelistv5-cveservicesadaptertest|cve[_-]?reports?|poc-cve-collection|cve-poc|vulnerability-poc|threatvault|linux-cve-announce(?:\\.0)?|linux_security_vulns|cve-research|cve-updates|cybervault-cve-intel|mitre_mapping_site|web-csirt)(?:$|[-_/])z„\\b(?:sync|bulk|database|dataset|feed|catalog|cvelist|threat[ -]?vault|layer updates?|update layers?|base\\.lock|metadata|mirror)\\b)rå   r"   c                 C  s   t t t| ƒ¡ƒS rm   )r   ÚNON_PRODUCT_REPO_RXrá   r¢   )rå   r   r   r   Úrepository_is_non_producth  s    r   )r`   rÍ   r"   c                   sP   t | ƒ‰ t |¡}d}d|› d|› d�d|› d|› d�f}t‡ fdd„|D ƒƒS )z?Require fix/patch wording to be directly connected to this CVE.z=(?:fix(?:e[ds])?|patch(?:e[ds])?|patched|backport(?:ed|ing)?)z\\bz\\b[^\\n.]{0,80}\\bc                 3  s   | ]}t  |ˆ t j¡V  qd S rm   )rž   rá   ÚI)rn   Úpatternrã   r   r   rÂ   u  rq   z*direct_cve_fix_language.<locals>.<genexpr>)r¢   rž   ÚescapeÚany)r`   rÍ   ZescapedZfixZpatternsr   rã   r   Údirect_cve_fix_languagel  s    
þr  )Úcommitr"   c                 C  s,   t |  d¡ƒ|  d¡g }tt|ƒtjd�S )NÚrepositoriesrå   ©r>   )rº   r4   ÚsortedrÄ   r9   r›   )r  rÙ   r   r   r   Úcommit_repositoriesx  s    r
  c                 C  s>   t | ƒ}dd„ |D ƒ}|r.t|dd„ d�d S |r:|d S dS )Nc                 S  s   g | ]}t |ƒs|‘qS r   )r   )rn   rø   r   r   r   rp     rq   z0preferred_product_repository.<locals>.<listcomp>c                 S  s   t | ƒ|  ¡ fS rm   )r¥   r›   r¡   r   r   r   Ú<lambda>ƒ  rq   z.preferred_product_repository.<locals>.<lambda>r  r   r�   )r
  r	  )r  r  Úproductr   r   r   Úpreferred_product_repository}  s
    r  ztuple[int, list[str]])r  rÍ   r"   c                 C  sF  t |  d¡ƒ}t| ƒ}g }d}|  d¡r8| ¡ | ¡ vrBddgfS |d7 }| d¡ t||ƒrr|d7 }| d¡ n|  d	¡rŽ|d
7 }| d¡ |  d¡rª|d7 }| d¡ |  d¡rÆ|d7 }| d¡ t|ƒrâ|d8 }| d¡ n|d
7 }| d¡ t |¡�r|d8 }| d¡ t	|ƒdk�r2|d7 }| d¡ t
dtd|ƒƒ|fS )z8Score a SHA-deduplicated commit as remediation evidence.r`   r   Úexplicit_cve_referenceZ!no_explicit_cve_in_commit_messager   Zexplicit_cve_in_commit_messageé-   Z#fix_language_directly_linked_to_cverä   é
   Zgeneric_fix_languageÚsecurity_contextrZ   Úregression_test_languageé<   Znon_product_repositoryZ(product_or_upstream_repository_candidateé   Zgeneric_or_bulk_update_messageé,  Zfocused_commit_messageéd   )r¢   r4   r  r›   rÚ   r  r   ÚGENERIC_UPDATE_RXrá   r¥   r#   r�   )r  rÍ   r`   rå   ÚreasonsÚscorer   r   r   Úcommit_fix_score‡  s@    











r  zset[str])Ú
referencesr"   c                 C  s   dd„ | D ƒS )Nc                 S  sD   h | ]<}t |tƒr| d ¡dkrt| d¡ƒrt| d¡ƒ ¡ ’qS )rë   rê   rì   )r2   r3   r4   r¢   r›   rÖ   r   r   r   Ú	<setcomp>µ  s
   
üz'advisory_commit_shas.<locals>.<setcomp>r   )r  r   r   r   Úadvisory_commit_shas´  s    þr  z2tuple[dict[str, Any] | None, list[dict[str, Any]]])rÍ   Ú
advisoriesÚcommitsÚissuesr  r"   c                 C  sò  g }|D ]”}|  d¡dkrqt|  d¡ƒD ]p}t|tƒr*t|  d¡ƒr*| ddd| t|  d¡ƒt|  d	¡ƒt|  d
¡ƒt|  d¡ƒt|  d¡ƒdgdœ
¡ q*qt|ƒ}g }	|D ]ò}
t|
tƒs¾q®t|
| ƒ\}}t|
  d¡ƒ}t|oê| 	¡ |v ƒ}|�rt
|dƒ}| d¡ t|
ƒ}|�r,t|
  d¡| ƒ�r,dn"|dk�rLt|
  d¡| ƒ�rLdnd}|	 d||| |t|
ƒ|t|
  d	¡ƒt|
  d¡ƒt|
  dd¡�p�dƒt|ƒdœ¡ q®|	jdd„ dd� |�rÖ|jdd„ d� |d |	fS td d!„ |	D ƒd"ƒ}||	fS )#zDReturn one primary remediation object plus ranked commit candidates.Úadvisory_typeÚreviewedÚfixed_versionsÚversionÚpatched_versionÚverifiedr  rú   r;   Ú	ecosystemÚpackageÚ2reviewed_github_advisory_documents_patched_version)
ÚtypeÚverification_levelr  rÍ   rú   Zadvisory_urlr'  r(  r$  r  rì   é_   Z$github_advisory_references_exact_shar`   r  rÉ   Zmentionr  Úduplicate_countrH   )r*  r+  r  rÍ   rå   r  rì   r;   r`   r-  r  c                 S  s6   |   d¡dk|   d¡dkt|   dd¡ƒt|   d¡ƒfS )Nr+  r&  rÉ   r  r   rì   )r4   r   r¢   rü   r   r   r   r  ü  s
    üz$select_primary_fix.<locals>.<lambda>T)r>   Úreversec                 S  s$   t |  d¡ƒ ¡ t |  d¡ƒ ¡ fS )Nr(  r$  )r¢   r4   r›   rü   r   r   r   r    s    þr  r   c                 s  s    | ]}|  d ¡dv r|V  qdS )r+  >   r&  rÉ   Nrû   rÖ   r   r   r   rÂ     rq   z%select_primary_fix.<locals>.<genexpr>N)r4   rº   r2   r3   r¢   rÚ   r  r  r   r›   r#   r  r  r
  r   rÄ   ÚsortÚnext)rÍ   r  r  r   r  Zreviewed_versionsÚadvisoryr$  Zreferenced_shasZrankedr  r  r  rì   Zadvisory_referencerå   r+  Zprimary_commitr   r   r   Úselect_primary_fix¾  s†    ö


ÿÿÿÿûõ
ù
ÿþr2  ztuple[str, str, list[str]])Úprimary_fixr  r  r   r  r—   r"   c                 C  sˆ   | r(|   d¡dkr(ddt|   dg ¡ƒfS | rP|   d¡dkrPddt|   dg ¡ƒfS |s`|s`|s`|rld	d
dgfS |r|dddgfS dddgfS )Nr+  r&  Úverified_fixÚhighr  rÉ   Úfix_candidateÚmediumÚmention_onlyÚlowZ3github_mentions_found_without_strictly_verified_fixÚlookup_errorru   Úlookup_incomplete_due_to_errorsÚno_public_fix_foundÚnoneÚno_github_evidence_found)r4   rÄ   )r3  r  r  r   r  r—   r   r   r   Ústrict_remediation_status  s    r?  z&tuple[list[dict[str, Any]], list[str]])ÚclientrÍ   r"   c                 C  s„  g }g }dD �]l}z|   d||ddœ¡}W nF typ } z.| d|› d|› �¡ W Y d }~qW Y d }~n
d }~0 0 t|ƒD �]ü}t|ƒ}|sŽqzg }	g }
t|  d¡ƒD ]´}t|ƒ}|s¶q¤|  d¡}t|ƒ}t|rØ|  d	¡nd
d	ƒ}t|rð|  d¡n|ddƒ}|  d¡}t|ddddƒ}t|  d¡dddƒ}||||dœ}|	 |¡ |r¤|
 |||ddœ¡ q¤g }t|  d¡ƒD ]B}t|tƒ�rŒt|dddƒ}nt|ƒ}t|ƒ}|�rl| |¡ �qlt|  d¡ddƒ}| d|t|  d¡ddƒ�pà||t|  d¡ddƒt	t|  d¡ƒdƒt	t|  d¡ƒd ƒt|  d!¡ƒt|  d"¡ƒt|  d#¡ƒt|  d¡�pP|  d¡ddƒ|	|
|d$t
|
ƒ|
�rnd%nd&d'œ¡ qzq||fS )(N)r"  Z
unreviewedz/advisoriesr  )rÍ   r*  Úper_pagez	advisory:rw   rÕ   r(  r'  r�   r­   Úfirst_patched_versionr¼   r$  ro   Úvulnerable_version_rangerz   )r'  r(  rC  rB  Zgithub_advisory©r'  r(  r$  Úsourcer  r;   r½   Zhrefrú   rÎ   rÍ   ÚseverityÚsummaryi�  Údescriptioni„  Úpublished_atÚ
updated_atÚwithdrawn_atTZpatched_version_documentedZ advisory_without_patched_version)rë   rú   rÍ   r!  rF  rG  rH  rI  rJ  rK  r;   rÕ   r#  r  r  rä   Úverification)r4   r   rÚ   rº   r·   rÀ   r2   r3   rù   r©   r   )r@  rÍ   Úevidencer—   r!  rŠ   r‹   Zraw_advisoryr1  rÕ   r#  Zraw_vulnerabilityZvulnerabilityZraw_packager(  r'  Zpackage_nameZraw_patchedr%  Zvulnerable_ranger×   r  Zraw_referenceZref_urlZ
classifiedrú   r   r   r   Úcollect_advisories-  s¸    

þ"
þý
ûüü
üÿ	ÿìÿrN  )r@  rÍ   Úmax_resultsr"   c                 C  s‚  g }|dkr|g fS z(|   dd|› d�ddt|dƒdœ¡}W n4 typ } zg d|› �gfW  Y d }~S d }~0 0 t|ƒ}t|  d	¡ƒD ]è}t|ƒ}|sšqˆt|  d
¡ƒ}	t|	  d¡ƒ}
t|	  d¡ƒ}t|	  d¡ƒ}t||ƒ}|oæt|ƒ}| dt	|ƒt|  d¡ddƒt|  d¡�p|  d¡ddƒt
|dƒt|
  d¡ƒt|  d¡ƒ||tt |¡ƒtt |¡ƒ|�rfdnddœ¡ qˆ|d |… g fS )Nr   z/search/commitsú"zcommitter-dateÚdescr  ©Úqr/  ÚorderrA  z	commits: rx   r  ÚauthorÚ	committerr`   Zcommit_search_resultrì   rÎ   r½   r;   é¼  ÚdateZexplicit_cve_fix_commitZcve_mention_in_commit)rë   rå   rì   r;   r`   Zauthor_dateZcommitter_dater  rä   r  r  rL  )r4   r�   r   r·   rº   rÀ   rß   rä   rÚ   ré   r©   r   ÚSECURITY_RXrá   ÚTEST_RX)r@  rÍ   rO  rM  rŠ   r‹   Úresponse_objÚraw_itemr×   r  rU  rV  r`   ÚexplicitÚfixingr   r   r   Úcollect_commitsž  sR    
üþ	&
ôÿr_  c                 C  s,  g }|dkr|g fS z(|   dd|› d�ddt|dƒdœ¡}W n4 typ } zg d	|› �gfW  Y d }~S d }~0 0 t|ƒ}t|  d
¡ƒD �]�}t|ƒ}|sœqˆt|  d¡ƒ}	t|  d¡ƒ}
|	› d|
› � ¡ }t|  d¡pÞ|  d¡ddƒ}t|ƒpði }|  d¡}t|ƒ}t|ƒ}t	||ƒ}|�o"t
|ƒ}t|  d¡ƒ}| |�r@dndt|  d¡ƒt|  d¡tƒ�rj|  d¡nt|  d¡ƒ|t|	dƒt|
dƒt|  d¡ddƒt|  d¡ƒt|  d¡ƒt|  d¡ƒ|||tt |¡ƒtt |¡ƒtt |¡ƒ|�rü|�rü|�rüdn|�rdn|�rd nd!d"œ¡ qˆ|d |… g fS )#Nr   z/search/issuesrP  z" in:title,bodyÚupdatedrQ  r  rR  zissues: rx   ÚtitlerŒ   rœ   r½   r;   Zpull_requestÚ	merged_atÚpull_request_search_resultZissue_search_resultrå   rî   r  rW  Ústater­   ro   Ú
created_atrJ  Ú	closed_atZ$merged_explicit_cve_fix_pull_requestZexplicit_cve_fix_candidateZcve_mention_in_pull_requestZcve_mention_in_issue)rë   rå   rî   r;   ra  Zbody_excerptrd  re  rJ  rf  rb  r  rä   r  r  Zversion_languagerL  )r4   r�   r   r·   rº   rÀ   r    rù   r   rß   rä   rÚ   r2   r   r©   rY  rá   rZ  Ú
VERSION_RX)r@  rÍ   rO  rM  rŠ   r‹   r[  r\  r×   ra  rŒ   r¨   r½   ZparsedZraw_pullZpullZis_pull_requestr]  r^  rb  r   r   r   Úcollect_issues_and_pullsÔ  sz    
üþ	&

(ÿÿÿÿÿèÿrh  )r  r"   c                 C  sz   g }| D ]l}|  d¡}|  d¡}|  dg ¡p.g D ]B}t|tƒs@q0t|ƒ}d|d< ||d< ||d< d|d< | |¡ q0q|S )Nrú   r!  r  Zgithub_advisory_referencerE  Tr  )r4   r2   r3   rÚ   )r  rÜ   r1  rú   r!  Z	referencer×   r   r   r   Úflatten_advisory_references"  s    


ri  )r  r  r   r  r—   r"   c                 C  sv  g }t dd„ | D ƒƒ}t dd„ | D ƒƒ}dd„ |D ƒ}dd„ |D ƒ}	dd„ |D ƒ}
dd„ |D ƒ}d	d„ || D ƒ}| | | | }|r�| d
¡ |rž| d¡ |	r¬| d¡ |
rº| d¡ |rÈ| d¡ |sÔ|sÔ|	râddt|ƒfS |rþ| d¡ ddt|ƒfS |�s|
�s|�r.|�r | d¡ ddt|ƒfS |�rL| d¡ ddt|ƒfS |�rj| d¡ ddt|ƒfS dddgfS )Nc                 s  s&   | ]}|  d ¡dko|  d¡V  qdS )r!  r"  r#  Nrû   rÖ   r   r   r   rÂ   <  s   ÿz%remediation_status.<locals>.<genexpr>c                 s  s   | ]}|  d ¡V  qdS )r#  Nrû   rÖ   r   r   r   rÂ   @  rq   c                 S  s   g | ]}|  d ¡r|‘qS )rä   rû   rÖ   r   r   r   rp   A  rq   z&remediation_status.<locals>.<listcomp>c                 S  s2   g | ]*}|  d ¡dkr|  d¡r|  d¡r|‘qS )rë   rc  rb  rä   rû   rÖ   r   r   r   rp   B  s
   

üc                 S  s   g | ]}|  d ¡dkr|‘qS )rë   rê   rû   rÖ   r   r   r   rp   I  s   c                 S  s   g | ]}|  d ¡dkr|‘qS )rë   rí   rû   rÖ   r   r   r   rp   L  s   c                 S  s$   g | ]}|  d ¡r|  d¡r|‘qS )r  rä   rû   rÖ   r   r   r   rp   O  s   þr)  Zexplicit_cve_fix_commit_foundZ*merged_explicit_cve_fix_pull_request_foundZ!github_advisory_references_commitZ'github_advisory_references_pull_requestr4  r5  Z-unreviewed_advisory_documents_patched_versionr6  r7  Z2explicit_fix_language_requires_manual_verificationZ3github_mentions_found_without_verified_fix_evidencer8  r9  r;  r:  ru   r<  r=  r>  )r  rÚ   rÄ   )r  r  r   r  r—   r  Zreviewed_patchedZany_patchedZexplicit_fix_commitsZmerged_fix_prsZadvisory_commit_refsZadvisory_pull_refsZ
candidatesZmentionsr   r   r   Úremediation_status3  sZ    þþÿÿþ








rj  c                 C  s   t tdd„ | D ƒƒtjd�S )Nc                 s  s"   | ]}t |tƒr| d ¡V  qdS )rå   N)r2   r3   r4   rÖ   r   r   r   rÂ   z  rq   z-repositories_from_evidence.<locals>.<genexpr>r  )r	  rÄ   r9   r›   )rx   r   r   r   Úrepositories_from_evidencex  s    þrk  )r@  Úsource_recordÚmax_commitsÚ
max_issuesr"   c                 C  s0  t |ƒ}|sddddgg dœS t| |ƒ\}}t| ||ƒ\}}t| ||ƒ\}	}
t|ƒ}|| |
 }t|ƒ}t|ƒ}t|	ƒ}	t|ƒ}t||||	|ƒ\}}t||||	||ƒ\}}}g }tƒ }|D ]¬}| 	dg ¡pÌg D ]–}t
|tƒsÞqÎt| 	d¡ƒt| 	d¡ƒt| 	d	¡ƒt| 	d
¡ƒdœ}|d  ¡ |d  ¡ |d	  ¡ |d
  ¡ f}|d	 rÎ||vrÎ| |¡ | |¡ qÎqº|| |	 | }|tt|ƒ 	d¡ƒtt|ƒ 	d¡ddƒt|ƒ 	d¡ttt|ƒ 	d¡ƒƒtt|ƒ 	d¡ddƒdœ||||dd„ |D ƒd d… t|ƒ|t|ƒt|ƒt|	ƒt|ƒt|ƒdœ|||	|dœ|dœS )Nr�   Zinvalid_cve_idr=  Z!source_record_has_no_valid_cve_id)rÍ   rj  Ú
confidencer  r—   r#  r'  r(  r$  rE  rD  Ú	publishedr  r­   ro   ÚcvssÚsurfacesr;   r½   )rp  rF  rq  rr  Znvd_urlc                 S  s   g | ]}|  d ¡dv r|‘qS )r+  >   r&  rÉ   rû   rÖ   r   r   r   rp   Å  s   ÿzenrich_cve.<locals>.<listcomp>r  ©r  r  Úissues_and_pull_requestsÚadvisory_referencesr#  ©r  r  rt  ru  )rÍ   rE  rj  ro  r  r3  Úfix_candidatesr  r#  ÚcountsÚgithubr—   )rÓ   rN  r_  rh  ri  rþ   r2  r?  rD   r4   r2   r3   r¢   r›   rÚ   rÛ   rÀ   r·   rÄ   rº   rk  r¥   )r@  rl  rm  rn  rÍ   r  Zadvisory_errorsr  Zcommit_errorsr   Zissue_errorsr  r—   r3  Zranked_fix_candidatesÚstatusro  r  r#  Zseen_fixed_versionsr1  Zraw_versionr×   r>   Zall_evidencer   r   r   Ú
enrich_cve  s”    û
ÿÿ

ü



ü
ûÿýûüàr{  zargparse.Namespacec                  C  sæ   t jdd�} | jdttd� | jdttd� | jdttd� | jdttd� | jdd	d
d� | jdtddd� | jddg dd� | jdtt	d� | jdtt
d� | jdttd� | jdtdd� | jdtdd� | jdd	d� |  ¡ S )Nz<Find public GitHub remediation evidence for automotive CVEs.)rH  z--input)r*  Údefaultz--outputz--cachez--cache-ttl-hoursz	--refreshÚ
store_truez4Ignore cached GitHub responses and fetch them again.)ÚactionÚhelpz--limitr   z0Process only the first N CVEs; 0 means all CVEs.)r*  r|  r  z--cverÚ   z*Process only this CVE ID; may be repeated.)r~  r|  r  z--max-commitsz--max-issuesz--request-delayz	--timeoutr   z	--retriesrŽ   z	--verbose)r~  )ÚargparseÚArgumentParserÚadd_argumentr   ÚDEFAULT_INPUTÚDEFAULT_OUTPUTÚDEFAULT_CACHEr   ÚDEFAULT_CACHE_TTL_HOURSÚDEFAULT_MAX_COMMITSÚDEFAULT_MAX_ISSUESrA   ÚDEFAULT_REQUEST_DELAYÚ
parse_args)Úparserr   r   r   rŠ  á  sF    ÿÿýüüÿrŠ  )Úargsr"   c                 C  s^   | j dk rtdƒ‚| jdk r$tdƒ‚| jdk r6tdƒ‚| jdk rHtdƒ‚| jdk rZtdƒ‚d S )Nr   z--limit must be 0 or greaterz"--max-commits must be 0 or greaterz!--max-issues must be 0 or greaterz&--cache-ttl-hours must be 0 or greaterz$--request-delay must be 0 or greater)r£   r”   rm  rn  Úcache_ttl_hoursrU   )rŒ  r   r   r   Úvalidate_args  s    




rŽ  )rÔ   Úrequested_cvesr£   r"   c           	      C  sž   dd„ |D ƒ}g }t ƒ }| D ]P}t|ƒ}|r||v r6q|rD||vrDq| |¡ | |¡ |rt|ƒ|kr qnq|ršt|| ƒ}|rštdd |¡ tj	d� |S )Nc                 S  s    h | ]}t |ƒrt |ƒ ¡ ’qS r   )r¢   rÐ   )rn   ro   r   r   r   r    rq   z!select_records.<locals>.<setcomp>z,[WARN] Requested CVEs not present in input: r¿   ©Úfile)
rD   rÓ   rÚ   rÛ   r¥   r	  ra   rÃ   ÚsysÚstderr)	rÔ   r�  r£   Z	requestedÚselectedrÝ   rÌ   rÍ   Zmissingr   r   r   Úselect_records  s*    

þr•  c                  C  sø  t ƒ } �z‚t| ƒ ttj d¡ƒ}|s.tdƒ‚t| jƒ}t	|ƒ}t
|| j| jƒ}|s\tdƒ‚t| j| j| jd�}t||| j| j| j| jd�}tdt|ƒ› dt|ƒ› d�ƒ g }d	}d
}	t|dd�D �]†\}
}t|ƒ}td|
› dt|ƒ› d|› �ƒ �zJzFt||| j| jd�}| |¡ td|› d|d › d|d › d�ƒ W nö t�yœ } zFd}t|ƒ}	td|› �t j!d� W Y d }~W | "¡   �qLW Y d }~n d }~0  t#�y2 } z~| |dddgd g g g ddddddœg g g g dœt$|ƒj%› d|› �gdœ¡ td|› d t$|ƒj%› d|› �t j!d� W Y d }~n
d }~0 0 W | "¡  qÂ| "¡  0 qÂt&d!d"„ |D ƒƒ}t&d#d"„ |D ƒƒ}dd$t'ƒ t| jƒt| d%¡�p–| d&¡ƒ| �o®t|ƒt|ƒk||	| j| j| j| jt(| jƒd'œt|ƒt|ƒt|ƒt)t*| +¡ ƒƒt)t*| +¡ ƒƒ|j,|j-|j.|j/d(œ	|d)œ}t0| j1|ƒ | "¡  td*t|ƒ› d+| j1› �ƒ td,t)|ƒ› �ƒ td-|j,› d.|j-› d/|j.› �ƒ |d0 �r†dnd1W S  t2tt3t4j5f�yÐ } z"td2|› �t j!d� W Y d }~d3S d }~0  t6�yò   td4t j!d� Y d5S 0 d S )6NZGITHUB_TOKENz*GITHUB_TOKEN is not set in the environmentzNo valid CVE records selected)r   r    r!   )rS   rT   rU   rV   rW   rX   z[GITHUB-CVE] Processing z of z input recordsFr�   rH   )Ústartz[GITHUB-CVE] ræ   rœ   )r@  rl  rm  rn  rw   rj  z (ro  ú)Tz[WARN] r�  Zparse_errorru   Z'unexpected_record_shape_or_parser_errorr   rs  rv  )rÍ   rj  ro  r  r3  rw  r  r#  rx  ry  r—   z: parse_error: c                 s  s    | ]}t | d ¡ƒpdV  qdS )rj  ru   N©r¢   r4   ©rn   rÌ   r   r   r   rÂ   Œ  s   ÿzmain.<locals>.<genexpr>c                 s  s    | ]}t | d ¡ƒpdV  qdS )ro  ru   Nr˜  r™  r   r   r   rÂ   �  s   ÿZgithub_cve_fixesrJ   Z	generated)Zmax_commits_per_cveZmax_issues_per_cver�  Zrequest_delay_secondsr!   )	Úsource_recordsZselected_recordsZprocessed_recordsrz  ro  Zgithub_api_requestsr]   r^   r_   )rI   ÚmodulerJ   Zsource_fileZsource_generated_isoÚcompleteÚstopped_earlyÚstop_reasonZconfigurationZ
statisticsrÔ   z[GITHUB-CVE] Wrote z records to z[GITHUB-CVE] Status summary: z[GITHUB-CVE] API requests=z, cache hits=z, remaining=rœ  r¤   zERROR: rŽ   zERROR: interrupted by useré‚   )7rŠ  rŽ  r¢   r±   Úenvironr4   r”   rµ   ÚinputrØ   r•  rÏ   r£   r   rT   r�  r!   rR   rU   rV   rW   rX   ra   r¥   Ú	enumeraterÓ   r{  rm  rn  rÚ   r   r9   r’  r“  rP   Ú	Exceptionr*  r   r   rM   r   r3   r	  rx   r\   r]   r^   r_   rN   rÜ   r´   r5   r1   r6   ÚKeyboardInterrupt)rŒ  rS   Zsource_datarš  r”  rT   r@  Zoutput_recordsr�  rž  Úindexrl  rÍ   Úresultr‹   rG  Zconfidence_summaryrO   r   r   r   Úmain2  s   
ýú	ÿÿü
ÿþÿáûüêþþþÿû÷ã 
ÿþÿr§  Ú__main__)rv   )er   Z
__future__r   r€  r<   r1   r±   r‘   rž   r’  rB   Úcollectionsr   r   r   Úpathlibr   Útypingr   r   Zurllib.errorr	   r
   Zurllib.parser   r   Zurllib.requestr   r   ry   rh   rƒ  r„  r…  r†  ZDEFAULT_PER_PAGEr‡  rˆ  r‰  ri   Úcompiler  rÑ   rñ   ró   rô   rõ   rö   rà   râ   rY  rZ  rg  ÚRuntimeErrorr   r   r   rR   rM   r¢   r©   rN   rµ   r·   rº   rÀ   rÊ   rË   rÓ   rØ   rÄ   rÞ   rß   rä   ré   rù   rý   rþ   rÿ   r  r   r  r
  r  r  r  r2  r?  rN  r_  rh  ri  rj  rk  r{  rŠ  rŽ  r•  r§  r   Úexitr   r   r   r   Ú<module>   sö   .þþþþþûýüýý: 

	>øý
-
Wq6NEb& 
