a
    R”ñju  ã                   @   s†  d Z ddlZddlZddlZddlZddlZddlZddlm  m	Z
 ddlmZ ddlmZmZ ddlmZ ddlZee dd¡ƒZdZdZdZd	d
„ ZedeƒZedeƒZe dd¡Ze dd¡dkZdZdZdZdZedƒZ e j!dd� edƒZ"e"j!dd� e d Z#de�rde› d�ndiZ$ej%ej&de 'e"d ¡e (¡ gd � e )d!¡Z*d"gZ+d#a,d$Z-d%d&„ Z.g d'¢Z/g d(¢Z0g d)¢Z1g d*¢Z2d‹d,d-„Z3e3ƒ Z4d.d/„ Z5d0d1„ Z6g d2¢Z7g d3¢e7 Z8g d4¢Z9g d5¢Z:e5e8ƒe5e9ƒe5e:ƒ  Z;Z<Z=e5e7ƒZ>g d6¢g d7¢g d8¢g d9¢g d:¢g d;¢g d<¢g d=¢d>d?gg d@¢g dA¢g dB¢g dC¢g dD¢g dE¢g dF¢dGœZ?dHdI„ e? @¡ D ƒZAe5g dJ¢ƒZBe CdKejD¡ZEdLdM„ ZFdNdO„ ZGdŒdQdR„ZHdSdT„ ZIdUdV„ ZJdWdX„ ZKdYdZ„ ZLd[d\„ ZMd]ZNd^ZOd_d`„ ZPd�dbdc„ZQddde„ ZRdfZSdgZTdhdi„ ZUdjZVdkdldmœZWdndo„ ZXdŽdqdr„ZYdsZZdtdu„ Z[dvdw„ Z\dxdy„ Z]dzd{„ Z^d|d}„ Z_d~d„ Z`d€d�„ Zad‚dƒ„ Zbd„d…„ Zcd†d‡„ Zddˆd‰„ ZeefdŠk�r‚eeƒ  dS )�a‰  
papers_intel.py - Automotive Cybersecurity Paper Collector (NCRC)

Sources : OpenAlex, Semantic Scholar, arXiv, DBLP   (metadata only - no PDF downloads)
Output  : papers_threats/papers.json  (read by threat_papers.html)

Pipeline: collect (cached) -> normalize -> merge duplicates (DOI / arXiv id / title)
          -> relevance filter (automotive AND security, word-boundary) -> tag topics,
          vendors, CVE ids, top venues -> write JSON

Environment variables (all optional except where noted):
  OPENALEX_API_KEY   free key, https://openalex.org/settings/api  (OpenAlex is skipped without it)
  S2_API_KEY         Semantic Scholar key (works without, but is rate limited hard)
  CONTACT_EMAIL      put in the User-Agent (polite use of the APIs)
  PAPERS_MIN_YEAR    default 2015
  PAPERS_SNOWBALL    1 (default) / 0 - also fetch papers citing the SEED_DOIS below

Run:  python3 papers_intel.py
é    N)ÚCounter)ÚdatetimeÚtimezone)ÚPathZPAPERS_MIN_YEARZ2015Ú c                 C   s$   t  | ¡p|pd ¡  d¡}|p"d S )Nr   z"')ÚosÚgetenvÚstrip)Zenv_nameZinlineÚval© r   úpapers_intel.pyÚ_key-   s    r   ZOPENALEX_API_KEYZ
S2_API_KEYZCONTACT_EMAILZPAPERS_SNOWBALLÚ1Ú0é   é   zpapers_threats/papers.jsonzzero_day_threats/zero_day.jsonÚcacheT©Úexist_okZlogszpapers_cache.jsonz
User-Agentz!NCRC-Paper-Collector/1.0 (mailto:ú)zNCRC-Paper-Collector/1.0z%%(asctime)s %(levelname)s %(message)sz
papers.log)ÚlevelÚformatÚhandlersÚpapersz10.1109/sp.2010.34g        g       @c                  C   s.   t   ¡ t } | tk r"t  t|  ¡ t   ¡ ad S ©N)ÚtimeÚ_LAST_S2_CALLÚS2_MIN_INTERVALÚsleep)Úelapsedr   r   r   Ús2_waitX   s    r    )z@"CAN bus" AND (attack OR intrusion OR security OR vulnerability)zMautomotive AND (cybersecurity OR "cyber security" OR attack OR vulnerability)z:"in-vehicle network" AND (security OR attack OR intrusion)z]("ISO 21434" OR "ISO/SAE 21434" OR "UNECE R155") AND (vehicle OR automotive OR cybersecurity)zR("connected vehicle" OR "connected car") AND (security OR attack OR vulnerability)zK(V2X OR "vehicle-to-everything") AND (security OR attack OR authentication)zh("electric vehicle charging" OR "EV charging" OR OCPP OR EVSE) AND (security OR attack OR vulnerability)zL(keyless OR "key fob" OR immobilizer) AND (relay OR attack OR vulnerability)zL("over-the-air" OR OTA) AND (vehicle OR automotive) AND (security OR attack)zg(ECU OR "electronic control unit") AND (attack OR fuzzing OR vulnerability) AND (vehicle OR automotive)zV("autonomous vehicle" OR "autonomous driving") AND (adversarial OR attack OR security)z@telematics AND (vulnerability OR attack OR security) AND vehiclez5infotainment AND (vulnerability OR attack OR exploit)zk(vehicle OR automotive) AND (penetration OR fuzzing OR "threat analysis" OR "risk assessment") AND security)
úautomotive cybersecurityzCAN bus intrusion detectionzvehicle ECU attackz)ISO 21434 threat analysis risk assessmentzV2X security attackzEV charging security OCPPzkeyless entry relay attackz$over-the-air update vehicle securityzconnected car vulnerabilityúin-vehicle network security)zicat:cs.CR AND (abs:automotive OR abs:"in-vehicle" OR abs:"connected vehicle" OR abs:"autonomous vehicle")z?abs:"CAN bus" AND (abs:attack OR abs:intrusion OR abs:security)z<(abs:"ISO 21434" OR abs:"UNECE R155" OR abs:"ISO/SAE 21434")z(abs:V2X AND (abs:security OR abs:attack)zc(abs:"EV charging" OR abs:"electric vehicle charging" OR abs:OCPP) AND (abs:security OR abs:attack)z>abs:keyless AND (abs:relay OR abs:attack OR abs:vulnerability)zn(abs:ECU OR abs:"electronic control unit") AND (abs:attack OR abs:fuzzing) AND (abs:vehicle OR abs:automotive))zautomotive securityzvehicle securityzvehicle attackzcan bus securityzcan bus intrusionr"   zv2x securityzecu securityzconnected vehicle securityzautonomous vehicle attackzcharging station securityzvehicle intrusion detectionzautomotive fuzzingzkeyless entry attackzvehicle over-the-air securityr!   úthreat_register.jsonc                 C   sœ   zt  t| dd�¡}W n ty,   g  Y S 0 t|tƒrD| di ¡ni }tƒ g  }}| ¡ D ]:}| dg ¡D ](}| 	¡ |vrl| 
| 	¡ ¡ | |¡ qlq\|S )z`Plain-language queries per threat category, written by build_register.py (editable in the JSON).úutf-8©ÚencodingZ
categoriesZpaper_queries)ÚjsonÚloadÚopenÚ	ExceptionÚ
isinstanceÚdictÚgetÚsetÚvaluesÚlowerÚaddÚappend)ÚpathÚdataZcatsÚseenÚoutÚcÚqr   r   r   Úregister_queries‘   s    
r9   c                 C   s>   t dd„ | D ƒtdd�}t dd dd„ |D ƒ¡ d	 tj¡S )
Nc                 S   s   h | ]}|  ¡ ’qS r   )r0   )Ú.0Útr   r   r   Ú	<setcomp>¦   ó    z_rx.<locals>.<setcomp>T©ÚkeyÚreversez(?<![a-z0-9])(?:ú|c                 s   s   | ]}t  |¡V  qd S r   )ÚreÚescape)r:   Úpr   r   r   Ú	<genexpr>§   r=   z_rx.<locals>.<genexpr>z)(?![a-z0-9]))ÚsortedÚlenrB   ÚcompileÚjoinÚI)ZtermsÚpartsr   r   r   Ú_rx¥   s    rL   c                 C   s   t dd„ |  |¡D ƒƒS )Nc                 S   s   h | ]}|  d ¡ ¡ ’qS )r   )Úgroupr0   )r:   Úmr   r   r   r<   «   r=   z_hits.<locals>.<setcomp>)rF   Úfinditer)ÚrxÚtextr   r   r   Ú_hitsª   s    rR   )ZteslaZbmwZaudiZ
volkswagenZporscheZtoyotaZlexusZhyundaiZkiaZhondaZnissanz
volvo carszmercedes-benzZmercedesZjeepZ
stellantisZrenaultZpolestarZrivianZbydzgeneral motorszharley-davidson)3Z
automotiveZ
automobileZautomobilesú
in-vehicleúin vehicle networkúcan busúcontroller area networkúcan-fdÚflexrayÚecuÚecusúelectronic control unitÚobdúobd-iiú	iso 21434úiso/sae 21434ú
unece r155úwp.29ZautosarÚv2xúvehicle-to-everythingúvehicle-to-vehicleúconnected vehicleúconnected vehiclesúconnected carúconnected carsúautonomous vehicleúautonomous vehiclesúautonomous drivingúself-drivingúdriver assistanceÚadasÚ
telematicsÚinfotainmentÚkeylessÚimmobilizerÚimmobiliserúkey fobÚtpmsÚocppÚevseúev chargingúelectric vehicle chargingúcharging stationúcharging stationsÚ	vehicularzintelligent transportationz	iso 15118Zv2gzvehicle to gridzvehicle-to-grid)ZvehicleZvehiclesZcarZcarsZtruckZtrucks)+ZattackZattacksZattackerZ	attackersZvulnerabilityZvulnerabilitiesZ
vulnerableZexploitZexploitsZexploitationZ	intrusionZsecurityZsecureZcybersecurityzcyber-securityZthreatZthreatsÚadversarialÚspoofingÚjammingZmalwareÚpenetrationÚfuzzingZrelayZauthenticationZ	tamperingZhackingZhijackúrisk assessmentÚtaraÚprivacyÚsurveillanceÚ	espionageúsupply chainZ
resilienceÚ
regulationÚsovereigntyúpersonal dataÚgdprÚtrackingÚgeopoliticalÚsbomúsoftware bill of materials)rU   rV   rW   rX   rS   rT   r\   r]   )zintrusion detectionzanomaly detectionZids)rb   rc   rd   re   rf   rg   rh   r|   )rx   ry   rz   r{   rv   rw   )rq   rt   rr   rs   ru   zrelay attack)zover-the-airZotazfirmware updatezsoftware update)ri   rj   rk   rl   rn   rm   )r^   r_   r`   ra   rƒ   r‚   zthreat analysisro   rp   )r„   rŠ   r‹   rŒ   r…   )r‡   rŽ   r�   Zsupplier)zmachine learningzlarge language modelZllmzdeep learningr}   )rˆ   Zuneceztype approvalZ
compliancezcyber resilience act)ZgnssZgpsr~   r   zsoftware defined radioZsdr)r†   r‰   r�   )rY   rZ   r[   r�   r€   zreverse engineering)zIn-vehicle network / CANzIntrusion detectionzV2X / connectedzEV chargingzKeyless / hardwarezOTA / updateszAutonomous / ADASzStandards / TARAzTelematics / infotainmentzPrivacy / data protectionzSupply chain / SBOMzAI / machine learningzGovernance / regulationzWireless / GNSS / SDRzEspionage / sovereigntyzECU / fuzzing / pentestc                 C   s   i | ]\}}|t |ƒ“qS r   )rL   ©r:   ÚkÚvr   r   r   Ú
<dictcomp>Þ   r=   r“   )zusenix securityzsecurity symposiumz!symposium on security and privacyzs&pz$computer and communications securityZccsz'network and distributed system securityZndssZescarZ
vehicleseczcomputer science in carsZcscsZacsacZasiaccsZraidZdsnZeurospz*european symposium on security and privacyZchesZtchesz)ieee transactions on vehicular technologyz"intelligent transportation systemszintelligent vehicles symposiumzvehicular communicationszdependable and secure computingz"information forensics and securityzsae internationalzsae technical paperzvehicular technology conferencez\bCVE-\d{4}-\d{4,7}\bc               
   C   sZ   t  ¡ si S zt t jdd�¡W S  tyT }  zt d| ¡ i W  Y d } ~ S d } ~ 0 0 d S )Nr$   r%   zCache load failed: %s)Ú
CACHE_FILEÚexistsr'   ÚloadsÚ	read_textr*   ÚloggerÚwarning©Úer   r   r   Ú
load_cacheð   s    rœ   c                 C   s,   t  d¡}|jt | ¡dd� | t ¡ d S )Nú.tmpr$   r%   )r”   Zwith_suffixZ
write_textr'   ÚdumpsÚreplace)r   Útmpr   r   r   Ú
save_cacheú   s    
r¡   é   c                 C   s„  t tƒ}| |pi ¡ td|d ƒD �]X}|d }�z tj| ||td�}|jdkrj|r^| ¡ n|j	W   S t
 d|j| ¡ |jdk�r|j d¡}	ztdt|	ƒƒ}
W n ttfy¾   d}
Y n0 td	d
d|d   ƒ}t|
|ƒ}||krþt
 d|| ¡ W  �q€t
 d| |||¡ t |¡ W q$|jdv �r2d| }W n6 t�yj } zt
 d|| |¡ W Y d }~n
d }~0 0 ||k r$t |¡ q$d S )Né   é   )ÚparamsÚheadersZtimeoutéÈ   zHTTP %s: %si­  zRetry-Afterr   iX  é<   zARate limit persists after %s attempts (%s); giving up this query.z3Rate limit reached (%s). Retry %s/%s in %s seconds.)i“  i÷  é
   zAttempt %s failed (%s): %s)r,   ÚHEADERSÚupdateÚrangeÚrequestsr-   ÚTIMEOUTZstatus_coder'   rQ   r˜   r™   r¦   ÚmaxÚintÚ	TypeErrorÚ
ValueErrorÚminÚerrorr   r   r*   )Úurlr¥   r¦   ZretriesÚas_jsonZhdrsZattemptÚwaitÚrZretry_afterZheader_waitZbackoff_waitr›   r   r   r   Úhttp_get   sN    


ýû
&r¹   c                 C   sŒ   |   |¡}|r:t ¡ |d  td k r:t d|¡ |d S t d|¡ |ƒ }t |¡ |d u rnt d|¡ g S t ¡ |dœ| |< t| ƒ |S )NÚtsi  zCACHE HIT %sr4   zFETCH %szNo data for %s (not cached))rº   r4   )r-   r   ÚCACHE_TTL_HOURSr˜   Úinfor   r™   r¡   )r   r?   ÚfnZdelayÚitemr4   r   r   r   Úcached1  s    

r¿   c                 C   s$   | sdS t jdd|  ¡ t jd� ¡ S )Nr   z!^(https?://(dx\.)?doi\.org/|doi:))Úflags)rB   Úsubr	   rJ   r0   ©Údr   r   r   Ú	clean_doiD  s    rÄ   c                  G   s4   | D ]*}t  d|pdt j¡}|r| d¡  S qdS )NzNarxiv\.org/(?:abs|pdf)/([0-9]{4}\.[0-9]{4,5}|[a-z\-]+(?:\.[A-Z]{2})?/[0-9]{7})r   r£   )rB   ÚsearchrJ   rM   )Ztextsr;   rN   r   r   r   Úextract_arxivJ  s
    rÆ   c                  K   s~   t ddg d ddddddddd�}| dd„ |  ¡ D ƒ¡ t dd|d pJd¡ ¡  d	¡|d< t dd|d
 pnd¡ ¡ |d
< |S )Nr   r   ©ÚtitleÚabstractÚauthorsÚyearÚdateÚvenueÚdoiÚarxiv_idrµ   Úpdf_urlÚcited_byÚsourcec                 S   s   i | ]\}}|d ur||“qS r   r   r�   r   r   r   r“   U  r=   zmk.<locals>.<dictcomp>z\s+ú rÈ   Ú.rÉ   )r,   r«   ÚitemsrB   rÁ   r	   Úrstrip)ÚkwÚrecr   r   r   ÚmkR  s    
ÿ$rÙ   c                    sL   | sdS i ‰ |   ¡ D ]\}}|D ]}|ˆ |< q qd ‡ fdd„tˆ ƒD ƒ¡S )Nr   rÓ   c                 3   s   | ]}ˆ | V  qd S r   r   )r:   Úi©Úposr   r   rE   b  r=   z#inverted_to_text.<locals>.<genexpr>)rÕ   rI   rF   )ÚinvZwordZidxsrÚ   r   rÛ   r   Úinverted_to_text[  s    rÞ   zhttps://api.openalex.org/workszœid,doi,title,display_name,publication_year,publication_date,abstract_inverted_index,authorships,primary_location,best_oa_location,open_access,cited_by_countc                 C   sþ   |   d¡pi }|  d¡pi }|   d¡p(i }|   d¡p6i }t|   d¡ƒ}|  d¡pRd}|  d¡pj|  d	¡pjd}t|   d
¡p€|   d¡t|   d¡ƒdd„ |   d¡pžg D ƒ|   d¡|   d¡p¶d|  d¡pÂd|t||ƒ|rÚd|› �n|pè|   dd¡||   d¡pöddd�S )NZprimary_locationrÒ   Zbest_oa_locationÚopen_accessrÎ   Zlanding_page_urlr   rÐ   Zoa_urlrÈ   Údisplay_nameZabstract_inverted_indexc                 S   s"   g | ]}|  d ¡pi   dd¡‘qS )Úauthorrà   r   ©r-   ©r:   Úar   r   r   Ú
<listcomp>v  r=   z!norm_openalex.<locals>.<listcomp>ZauthorshipsZpublication_yearZpublication_dateúhttps://doi.org/ÚidZcited_by_countr   ÚopenalexrÇ   )r-   rÄ   rÙ   rÞ   rÆ   )ÚwZlocÚsrcZbestZoarÎ   ZlandingÚpdfr   r   r   Únorm_openalexk  s"    ørì   é   c           	      C   s¾   g d }}dt › d�g|r |gng  }t|ƒD ]Š}d|td |¡tdœ}| rT| |d< tt|ƒ}|d u rl d S | dd	„ | d
g ¡D ƒ¡ | d¡p”i  d¡}|rª| d
¡s® qºt	 
d¡ q.|S )NÚ*zfrom_publication_date:ú-01-01éd   ú,)Zper_pageÚcursorÚselectÚfilterÚapi_keyrÅ   c                 s   s   | ]}t |ƒV  qd S r   )rì   )r:   ré   r   r   r   rE   ‰  r=   z!fetch_openalex.<locals>.<genexpr>ZresultsÚmetaZnext_cursorgš™™™™™É?)ÚMIN_YEARr¬   Ú	OA_SELECTrI   ÚOPENALEX_KEYr¹   ÚOA_URLÚextendr-   r   r   )	rÅ   ÚfiltÚpagesr6   rò   ZfltÚ_r¥   r4   r   r   r   Úfetch_openalex~  s"    

ÿ
rÿ   c                 C   s^   t t› d| › �dtdœƒ}|r(| d¡s8t d| ¡ dS |d  dd¡d }td	|› �d
d�S )z*Papers that cite a seed DOI (snowballing).z/doi:rç   )ró   rõ   z"Seed DOI not found in OpenAlex: %sNú/r£   éÿÿÿÿzcites:rí   )rü   rý   )r¹   rú   rù   r-   r˜   r™   Úrsplitrÿ   )rÎ   ZworkZwidr   r   r   Úfetch_openalex_citing‘  s    r  z5https://api.semanticscholar.org/graph/v1/paper/searchz]title,abstract,year,venue,publicationDate,authors,externalIds,openAccessPdf,citationCount,urlc                 C   s  t ƒ  trdtini }tt| dt› d�tdœ|ƒ}|d u r>d S g }| dg ¡D ]Â}| d¡p^i }t| d¡ƒ}| t	| d¡| d	¡d
d„ | d¡p–g D ƒ| d¡| d¡p®d| d¡pºd|| d¡pÈd|rÖd|› �n| d¡pâd| d¡pîi  d¡pød| d¡�pddd�¡ qN|S )Nz	x-api-keyrð   ú-)ÚqueryÚlimitrË   Úfieldsr4   ZexternalIdsZDOIrÈ   rÉ   c                 S   s   g | ]}|  d d¡‘qS ©Únamer   râ   rã   r   r   r   rå   ¬  r=   zfetch_s2.<locals>.<listcomp>rÊ   rË   ZpublicationDater   rÍ   ZArXivræ   rµ   ZopenAccessPdfZcitationCountr   ÚsemanticscholarrÇ   )
r    ÚS2_KEYr¹   ÚS2_URLr÷   Ú	S2_FIELDSr-   rÄ   r2   rÙ   )r  r¦   r4   r6   rD   ZextrÎ   r   r   r   Úfetch_s2   s&    ù
r  z"https://export.arxiv.org/api/queryzhttp://www.w3.org/2005/Atomzhttp://arxiv.org/schemas/atom)rä   Zaxc           	         s4  g }t  | ¡}| dt¡D �]‰ ‡ fdd„}|dƒ}d}ˆ  dt¡D ]}| d¡dkrD| d	d¡}qDt|ƒp„t d
d| dd¡d ¡}t	|dƒƒ}| 
t|dƒ|dƒdd„ ˆ  dt¡D ƒ|dƒd d…  ¡ ràt|dƒd d… ƒnd |dƒd d… |dƒpúd|||�rd|› �nd|› �|�p$d|› �dd�¡ q|S )Nza:entryc                    s"   ˆ   | t¡}|d ur|jp dS dS )Nr   ©ÚfindÚNSrQ   )r3   Zelrš   r   r   r;   ¾  s    zparse_arxiv.<locals>.tza:idr   za:linkrÈ   rë   Zhrefzv\d+$z/abs/r£   r  zax:doiza:titlez	a:summaryc                 S   s   g | ]}|  d t¡jpd‘qS )za:namer   r  rã   r   r   r   rå   Ê  r=   zparse_arxiv.<locals>.<listcomp>za:authorza:publishedé   r©   zax:journal_refÚarXivræ   zhttps://arxiv.org/abs/zhttps://arxiv.org/pdf/Úarxiv)rÈ   rÉ   rÊ   rË   rÌ   rÍ   rÎ   rÏ   rµ   rÐ   rÒ   )ÚETZ
fromstringÚfindallr  r-   rÆ   rB   rÁ   r  rÄ   r2   rÙ   Úisdigitr°   )	Zxml_textr6   Úrootr;   Zabs_urlrë   ÚlZarxrÎ   r   rš   r   Úparse_arxivº  s(    
"*ú
r  r¤   c                 C   sª   g }t |ƒD ]˜}tt| |d ddddœdd�}|d u r< d S zt|ƒ}W n6 tjy~ } zt d|¡ W Y d }~ d S d }~0 0 | |¡ t	|ƒdk rš q¦t
 d¡ q|S )	Nrð   ZsubmittedDateZ
descending)Zsearch_queryÚstartZmax_resultsZsortByZ	sortOrderF)r¶   zarXiv XML parse error: %sçÍÌÌÌÌÌ@)r¬   r¹   Ú	ARXIV_URLr  r  Z
ParseErrorr˜   r™   rû   rG   r   r   )r  rý   r6   ZpageZtxtZbatchr›   r   r   r   Úfetch_arxivÒ  s&    ÿÿ
r  z https://dblp.org/search/publ/apic                 C   sh  t t| dddœƒ}|d u rd S | d¡p*i  d¡p4i  d¡p>g }g }|D �]}| d¡pZi }| d¡phi  d	¡prg }t|tƒr„|g}| d
¡p�d}t|tƒr¦d |¡}t| dd¡ƒ ¡ rÆt	|d ƒnd }|rØ|t
k rØqHt| d¡ƒ}	| d¡pòd}
t|
tƒ�r|
�r|
d nd}
| t| d¡dd„ |D ƒ||dk�r<dn||	t|
ƒ|	�rVd|	› �n|
dd�¡ qH|S )Nr'   iô  )r8   r   ÚhÚresultÚhitsZhitr¼   rÊ   rá   rÍ   r   z, rË   rÎ   Úeer   rÈ   c                 S   s   g | ]}|  d d¡‘qS )rQ   r   râ   rã   r   r   r   rå   þ  r=   zfetch_dblp.<locals>.<listcomp>ZCoRRr  ræ   Údblp)rÈ   rÊ   rË   rÍ   rÎ   rÏ   rµ   rÒ   )r¹   ÚDBLP_URLr-   r+   r,   ÚlistrI   Ústrr  r°   r÷   rÄ   r2   rÙ   rÆ   )r  r4   r!  r6   r  r¼   rÊ   rÍ   rË   rÎ   r"  r   r   r   Ú
fetch_dblpé  s4    "



$ý
r'  c                 C   s   t  dd| pd ¡ ¡S )Nz	[^a-z0-9]r   )rB   rÁ   r0   ©rÈ   r   r   r   Ú	title_key  s    r)  c                 C   s   | pd  ¡  ¡ dv S )Nr   )r   r  Zcorrz	arxiv.org)r	   r0   )r’   r   r   r   Úis_preprint_venue  s    r*  c                 C   s,  dD ]$}|   |¡s|  |¡r|| | |< q| d sF|d rF|d | d< t| d ƒr€t|d ƒs€|d | d< |  d¡r€|d | d< t|d ƒt| d ƒkr¤|d | d< t|d ƒt| d ƒkrÈ|d | d< |d rô| d rè|d | d k rô|d | d< t| d |d ƒ| d< tt| d	 ƒ|d
 hB ƒ| d	< d S )N)rÎ   rÏ   rµ   rÐ   rË   rÍ   rÎ   rµ   rÉ   rÊ   rÌ   rÑ   ÚsourcesrÒ   )r-   r*  rG   r¯   rF   r.   )ÚbaseÚnewr‘   r   r   r   Ú
merge_into  s"    
 r.  c                    sê   i g  ‰ }| D ]Ö}t |d ƒdk r$q|d g|d< dd„ d|d fd|d	  ¡ fdt|d ƒffD ƒ}t‡ fd
d„|D ƒd ƒ}|d u r–|}| |¡ n
t||ƒ |D ]}|ˆ |< q¤d|d fd|d	  ¡ ffD ]}|d rÎ|ˆ |< qÎq|S )NrÈ   é   rÒ   r+  c                 S   s   g | ]}|d  r|‘qS )r£   r   ©r:   r‘   r   r   r   rå   *  s   ÿzdedupe.<locals>.<listcomp>rÎ   r  rÏ   c                 3   s   | ]}|ˆ v rˆ | V  qd S r   r   r0  ©Úindexr   r   rE   ,  r=   zdedupe.<locals>.<genexpr>r£   )rG   r0   r)  Únextr2   r.  )Úrecordsr6   r¸   Úkeysr,  r‘   r   r1  r   Údedupe$  s&    
ÿ

 r6  c                 C   sÂ   | d | d d | d   }}t t|ƒ}t t|ƒ}t t|ƒ}t t|ƒ}|sNg S |rntt|ƒt|dd… ƒB ƒS |rŽtt|ƒt|dd… ƒB ƒS |r¾t t| d ƒr¾tt|ƒtt t| d ƒƒB ƒS g S )zLReturns matched terms if the paper is automotive-security relevant, else [].rÈ   rÓ   rÉ   Nrí   )rR   ÚAUTO_STRONG_RXÚAUTO_WEAK_RXÚSEC_RXrF   r.   )rØ   rÈ   rQ   ZstrongZ
weak_titleZsec_anyZ	sec_titler   r   r   Úassess=  s    



r:  c                    s`  | d d | d  ‰t dd„ t ˆ¡D ƒƒ}| d p^| d d d…  ¡ r\t| d d d… ƒnd }| d	 rvd
| d	 › �n:| d rŒd| d › �n$dt t| d ƒ ¡ ¡ 	¡ d d…  }|| d | d d d… t
| d ƒ|| d | d | d | d	 | d | d | d t| d ƒ| d | d |‡fdd„t ¡ D ƒttˆƒtt | d �pDd¡ƒ|‡ fdd„|D ƒdœS )NrÈ   rÓ   rÉ   c                 S   s   h | ]}|  ¡ ’qS r   )Úupper©r:   r7   r   r   r   r<   Q  r=   zfinalize.<locals>.<setcomp>rË   rÌ   r  rÎ   zdoi:rÏ   úarxiv:zt:r   rÊ   rÍ   rµ   rÐ   rÑ   r+  c                    s   g | ]\}}|  ˆ ¡r|‘qS r   ©rÅ   )r:   r;   rP   )rQ   r   r   rå   \  r=   zfinalize.<locals>.<listcomp>r   c                    s   g | ]}|ˆ v r|‘qS r   r   r<  )Ú
known_cvesr   r   rå   _  r=   )rç   rÈ   rÊ   Zauthor_countrË   rÌ   rÍ   rÉ   rÎ   rÏ   rµ   rÐ   rß   rÑ   r+  Zmatched_termsZtopicsZvendorsÚ	top_venueÚcvesZlinked_cves)rF   ÚCVE_RXr  r  r°   ÚhashlibZsha1r)  ÚencodeZ	hexdigestrG   ÚboolÚTOPIC_RXrÕ   rR   ÚOEM_RXÚTOP_VENUES_RXrÅ   )rØ   Úmatchedr?  rA  rË   Úpidr   )r?  rQ   r   ÚfinalizeO  s"    4,$ÿörK  c                  C   sl   zNt tdd��.} dd„ t | ¡ dg ¡D ƒW  d   ƒ W S 1 sB0    Y  W n tyf   tƒ  Y S 0 d S )Nr$   r%   c                 S   s   h | ]}|  d d¡ ¡ ’qS r  )r-   r;  ©r:   r¸   r   r   r   r<   f  r=   z"load_known_cves.<locals>.<setcomp>r4  )r)   ÚZERO_DAY_FILEr'   r(   r-   r*   r.   )Úfr   r   r   Úload_known_cvesc  s
    @rO  c               	      s�  t ƒ } g tƒ  ‰‰ ‡ ‡fdd„}t dtr:dttƒ› d�ndtrDdnd¡ tr¬tD ]&}|d	t| d
|› �|fdd„dƒƒ qRt	r¶t
D ]&}|d	t| d|› �|fdd„dƒƒ q‚n
t d¡ trút dttƒ¡ trútD ]&}|d	t| d|› �|fdd„dƒƒ qÒtt D ](}|dt| d|› �|fdd„dƒƒ �qtD ](}|dt| d|› �|fdd„dƒƒ �q0tD ](}|dt| d|› �|fdd„dƒƒ �q^ˆˆ fS ) Nc                    s"   ˆ  |¡ ˆ |   t|ƒ7  < d S r   )rû   rG   )rÒ   rÕ   ©Ú
per_sourceÚrawr   r   r1   r  s    zcollect_raw.<locals>.addz+OpenAlex key: %s | Semantic Scholar key: %szset (z chars)zNOT setr.   znot setrè   zoa:c                 S   s
   t | d�S )Nr>  ©rÿ   ©r8   r   r   r   Ú<lambda>z  r=   zcollect_raw.<locals>.<lambda>g333333Ó?z	oa-cites:c                 S   s   t | ƒS r   )r  rÂ   r   r   r   rU  }  r=   z•No OpenAlex key found - skipping OpenAlex. Set env var OPENALEX_API_KEY in the SAME terminal you run this from, or paste it into INLINE_OPENALEX_KEY.z*%s extra queries from threat_register.jsonzoa-reg:c                 S   s   t | dd�S )Nr£   )rÅ   rý   rS  rT  r   r   r   rU  †  r=   r
  zs2:c                 S   s   t | ƒS r   )r  rT  r   r   r   rU  ˆ  r=   r   r  r=  c                 S   s   t | ƒS r   )r  rT  r   r   r   rU  Š  r=   r  r#  zdblp:c                 S   s   t | ƒS r   )r'  rT  r   r   r   rU  Œ  r=   g      ø?)rœ   r   r˜   r¼   rù   rG   r  ÚOPENALEX_QUERIESr¿   ÚSNOWBALLÚ	SEED_DOISr™   ÚREGISTER_QUERIESÚ
S2_QUERIESÚARXIV_QUERIESÚDBLP_QUERIES)r   r1   r8   rÎ   r   rP  r   Úcollect_rawn  s4    
þ$&
$&&&r]  c            
      C   s  t dƒ tƒ \} }t dt|ƒ› �ƒ dd„ | D ƒ} t| ƒ}tƒ }g dd  }}}|D ]v}|d pŠ|d d d…  ¡ rˆt|d d d… ƒnd }|r¢|tk r¢|d	7 }qTt|ƒ}	|	s¸|d	7 }qT| 	t
||	|ƒ¡ qT|jd
d„ dd� t dt| ƒ› dt|ƒ› d|› d|› dt|ƒ› �
ƒ |S )Nz[PAPERS] Collecting...z![PAPERS] Raw records per source: c                 S   s   g | ]}|r|  d ¡r|‘qS r(  râ   rL  r   r   r   rå   •  r=   zrun_papers.<locals>.<listcomp>r   rË   rÌ   r  r£   c                 S   s   | d p| d pd› d�S )NrÌ   rË   r   rï   r   )r¸   r   r   r   rU  £  r=   zrun_papers.<locals>.<lambda>Tr>   z[PAPERS] raw=z unique=z	 too_old=z not_relevant=z kept=)Úprintr]  r,   r6  rO  r  r°   r÷   r:  r2   rK  ÚsortrG   )
rR  rQ  ÚuniqueZknownr4  Zdropped_yearZdropped_relr¸   rË   rI  r   r   r   Ú
run_papers�  s,    
4

ÿÿra  c               
   C   s  t ƒ } dt ¡ t tj¡ ¡ ttt	dd„ | D ƒƒ 
¡ dd�ƒtt	dd„ | D ƒƒƒtdd„ | D ƒƒtdd„ | D ƒƒtd	d„ | D ƒƒd
œ| dœ}tjtj t¡dd� td }t|ddd��"}tj||ddd� W d   ƒ n1 sà0    Y  t |t¡ tdt| ƒ› dt› �ƒ d S )Nr   c                 s   s   | ]}t |d  ƒV  qdS )rË   N)r&  rL  r   r   r   rE   ¯  r=   zmain.<locals>.<genexpr>T)r@   c                 s   s    | ]}|d  D ]
}|V  qqdS )r+  Nr   )r:   r¸   Úsr   r   r   rE   °  r=   c                 s   s   | ]}|d  V  qdS )r@  Nr   rL  r   r   r   rE   ±  r=   c                 s   s   | ]}|d  V  qdS )rß   Nr   rL  r   r   r   rE   ²  r=   c                 s   s   | ]}t |d  ƒV  qdS )rA  N)rE  rL  r   r   r   rE   ³  r=   )Zby_yearZ	by_sourcer@  rß   Z
cve_linked)ÚmoduleZ	generatedZgenerated_isoZsummaryr4  r   r�   ré   r$   r%   r¤   F)ÚindentZensure_asciiz[PAPERS] Wrote z records to )ra  r   r   Znowr   ZutcZ	isoformatr,   rF   r   rÕ   Úsumr   Úmakedirsr3   ÚdirnameÚOUTPUT_FILEr)   r'   ÚdumprŸ   r^  rG   )r4  r6   r    rN  r   r   r   Úmain©  s$     ü÷0rj  Ú__main__)r#   )NNr¢   T)NNrí   )r¤   )gÚ__doc__rC  r'   Zloggingr   rB   r   Zxml.etree.ElementTreeZetreeZElementTreer  Úcollectionsr   r   r   Úpathlibr   r­   r°   r   r÷   ZINLINE_OPENALEX_KEYZINLINE_S2_KEYZCONTACTr   rù   r  rW  r®   r»   rh  rM  Z	CACHE_DIRÚmkdirZLOG_DIRr”   rª   ZbasicConfigÚINFOZFileHandlerZStreamHandlerZ	getLoggerr˜   rX  r   r   r    rV  rZ  r[  r\  r9   rY  rL   rR   ZOEMSZAUTO_STRONGZ	AUTO_WEAKZSECURITYr7  r8  r9  rG  ZTOPIC_RULESrÕ   rF  rH  rH   rJ   rB  rœ   r¡   r¹   r¿   rÄ   rÆ   rÙ   rÞ   rú   rø   rì   rÿ   r  r  r  r  r  r  r  r  r$  r'  r)  r*  r.  r6  r:  rK  rO  r]  ra  rj  Ú__name__r   r   r   r   Ú<module>   sÜ   

ÿþ
ÿ

õ
ð


1	


"
