a
    Gšíjg   ã                   @   s&  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	Z	dZ
dZe d¡Zer^dnd	Zd
ZdZedƒZejdd� edƒZejdd� edƒZejdd� ed Zed ZddiZerÊeed< ejejde e¡e ¡ gd� e d¡Zddgddgddgdd gd!gd"gd#d$gg d%¢g d&¢d'd(gd)gd*d+gg d,¢d-œZg d.¢Zeed/< d0d1„ Z d2d3„ Z!d4d5„ Z"dNd7d8„Z#d9d:„ Z$d;d<„ Z%d=d>„ Z&d?d@„ Z'dAdB„ Z(e)dCk�r"e(ƒ Z*i Z+e*D ]*Z,e+ -e, -dDdE¡d¡dF e+e, -dDdE¡< �q´e ¡ e.e*ƒe+e*dGœZ/edH Z0e0j1ej2e/dIdJ�dKdL� e 3dMe0¡ dS )OaX  
feeds_automotive.py - Automotive Threat Intelligence Collector (NVD)

Changes vs. previous version:
  * Pagination (totalResults / startIndex) - nothing is silently cut off at page 1
  * keywordExactMatch so "can bus" is searched as a phrase, not can AND bus
  * Proper URL encoding
  * NVD rate limits respected (6.5 s without API key, 0.7 s with NVD_API_KEY env var)
  * Backoff on 403/429/503 (NVD answers 403 when rate limited)
  * Cache has a TTL (otherwise new CVEs never show up) and partial results are never cached
  * Extra queries on OEM / Tier-1 vendor names, not only protocol keywords
é    N)ÚPath)Úquoteé   z0https://services.nvd.nist.gov/rest/json/cves/2.0ÚNVD_API_KEYgffffffæ?g      @iÐ  é   ÚcacheT)Úexist_okÚoutputZlogsznvd_cache.jsonz	feeds.logz
User-AgentzNCRC-Threat-Collector/1.1ZapiKeyz%%(asctime)s %(levelname)s %(message)s)ÚlevelÚformatÚhandlersÚfeedszcontroller area networkzcan buszunified diagnostic servicesÚudsziso-tpÚisotpzdiagnostics over ipÚdoipzsome/ipÚj1939zover-the-air updatez
ota update)ÚevseÚocppzcharging station)Ú
telematicsztelematics control unitzvehicle cloudÚinfotainmentzin-vehicle infotainmentzfleet managementzelectronic control unitzautomotive ecu)zkeyless entryzkey fobÚimmobilizerzrolling codeÚtpms)zCAN BuszUDS DiagnosticszISO-TPÚDOIPzSOME/IPÚJ1939ÚOTAzEV ChargingÚ
TelematicsÚInfotainmentÚFleetÚECUzKeyless / Hardware)ÚteslaÚbmwÚaudiÚ
volkswagenÚporscheÚskodazmercedes-benzÚtoyotaÚlexusÚhyundaiÚkiaÚhondaÚnissanz
volvo carsÚpolestarÚrenaultÚ
stellantisÚjeepÚrivianÚbydÚxpengÚsuzukiz
ford motorzgeneral motorsÚharmanÚdensoÚvisteonÚaptivÚautosarÚqnxzvector informatikzOEM / Supplierc               
   C   sZ   t  ¡ si S zt t jdd�¡W S  tyT }  zt d| ¡ i W  Y d } ~ S d } ~ 0 0 d S )Núutf-8©ÚencodingzCache load failed: %s)Ú
CACHE_FILEÚexistsÚjsonÚloadsZ	read_textÚ	ExceptionÚloggerÚwarning)Úe© rD   ú-/mnt/bigstor/www/usecases/feeds_automotive.pyÚ
load_cacheX   s    rF   c                 C   s,   t  d¡}|jt | ¡dd� | t ¡ d S )Nz.tmpr9   r:   )r<   Zwith_suffixÚ
write_textr>   ÚdumpsÚreplace)r   ÚtmprD   rD   rE   Ú
save_cacheb   s    
rK   c                 C   s2   t | tƒo0d| v o0d| v o0t ¡ | d  td k S )NÚtsÚentryi  )Ú
isinstanceÚdictÚtimeÚCACHE_TTL_HOURS)ÚitemrD   rD   rE   Úcache_freshh   s    ÿrS   é   c                 C   s¬   t d|d ƒD ]˜}|d }zNtj| ttd�}|jdkrD| ¡ W   S t d|j| ¡ |jdv rfd| }W n2 t	yš } zt d||¡ W Y d }~n
d }~0 0 t
 |¡ qd S )	Né   é   )ZtimeoutZheaderséÈ   zHTTP %s: %s)i“  i­  i÷  é
   zAttempt %s failed: %s)ÚrangeÚrequestsÚgetÚTIMEOUTÚHEADERSZstatus_coder>   rA   rB   r@   rP   Úsleep)ÚurlZretriesZattemptÚwaitÚrrC   rD   rD   rE   Úsafe_get_jsonp   s    

$rb   c                 C   s”   g dd  }}}t › dt| ƒ› dt› d|› �}t|ƒ}|du rBdS | | dg ¡¡ | dd¡}|| dt¡ppt7 }||kr~qŠt t¡ q||d	œS )
zHFetch ALL pages for one keyword phrase. Returns None if any page failed.r   z?keywordSearch=z"&keywordExactMatch&resultsPerPage=z&startIndex=NÚvulnerabilitiesÚtotalResultsZresultsPerPage)rd   rc   )	ÚNVD_URLr   Ú	PAGE_SIZErb   Úextendr[   rP   r^   ÚREQUEST_DELAY)ÚsearchÚvulnsÚstartZtotalr_   ÚdatarD   rD   rE   Ú	nvd_queryƒ   s     ÿÿrm   c               	   C   s  t ƒ } g }t dttƒ¡ t ¡ D ]Ê\}}|D ]¼}|› d|› �}t|  |¡ƒrnt d|¡ | 	| | d ¡ q.t d|¡ t
|ƒ}|d u r˜t d|¡ q.d|||dœ}| 	|¡ t ¡ |d	œ| |< t| ƒ t d
|t|d ƒ¡ t t¡ q.q"t dt|ƒ¡ |S )Nz%Starting NVD collection (api key: %s)ú:zCACHE HIT %srM   zNVD QUERY -> %sz#No/partial data for %s (not cached)ÚNVD)ÚsourceÚsurfaceÚkeywordrl   )rL   rM   z  %s -> %s CVEsrc   z!NVD collection complete: %s feeds)rF   rA   ÚinfoÚboolr   ÚSURFACE_QUERIESÚitemsrS   r[   Úappendrm   rB   rP   rK   Úlenr^   rh   )r   Úresultsrq   Zsearchesri   Úkeyrl   rM   rD   rD   rE   Úfeed_nvd•   s2    ÿ
r{   c                  C   s,   d} t | ƒ}|st d¡ g S dd|dœgS )NzBhttps://www.cisa.gov/sites/default/files/feeds/ics-advisories.jsonzCISA feed unavailableÚCISA_ICSZICS)rp   rq   rl   )rb   rA   rB   )r_   rl   rD   rD   rE   Úfeed_cisa_ics¸   s    
r}   c                 C   sV   |   dd¡}ddddœ}|dkr2|jdd	d
d� n|dkrJ|jdddd� || d< | S )Nrp   Ú ÚunknownZgenericZintel)ZseverityZdomainZcategoryro   ZmediumZvulnerabilityÚcver|   ZhighZicsZadvisoryÚmeta)r[   Úupdate)rM   rp   r�   rD   rD   rE   Úenrich_feed_entryÁ   s    rƒ   c                  C   s2   g } |   tƒ ¡ dd„ | D ƒ} t dt| ƒ¡ | S )Nc                 S   s   g | ]}t |ƒ‘qS rD   )rƒ   )Ú.0ÚxrD   rD   rE   Ú
<listcomp>Ð   ó    z%collect_all_feeds.<locals>.<listcomp>zTotal feeds collected: %s)rg   r{   rA   rs   rx   )r   rD   rD   rE   Úcollect_all_feedsÌ   s
    rˆ   Ú__main__rq   ÚUnknownrU   )Ú	generatedZ
feed_countZsurface_summaryr   zautomotive_feeds.jsonrV   )Úindentr9   r:   zWritten: %s)rT   )4Ú__doc__r>   ZloggingÚosrP   Úpathlibr   Zurllib.parser   rZ   r\   re   Úgetenvr   rh   rf   rQ   Z	CACHE_DIRÚmkdirZ
OUTPUT_DIRZLOG_DIRr<   ZLOG_FILEr]   ZbasicConfigÚINFOZFileHandlerZStreamHandlerZ	getLoggerrA   ru   ZVENDOR_QUERIESrF   rK   rS   rb   rm   r{   r}   rƒ   rˆ   Ú__name__r   ZcountsrR   r[   rx   ÚsummaryZoutfilerG   rH   rs   rD   rD   rD   rE   Ú<module>   s~   
ý
ó

#		
(ÿ