a
    I‰ðjô° ã                   @  sp  U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZ d	Ze  d
¡Z!e
 "de
j#¡Z$e
 "de
j#¡Z%e
 "de
j#¡Z&e
 "d¡Z'e
 "d¡Z(edƒedƒedƒedƒedƒedƒedƒedƒdœZ)h d£Z*g d¢g d¢g d¢g d¢g d¢g d¢g d ¢g d!¢g d"¢g d#¢g d$¢g d%¢g d&¢g d'¢d(œd)d*gg d+¢g d,¢g d-¢g d.¢g d/¢g d0¢g d1¢g d2¢g d3¢d4d5gg d6¢d7œg d8¢g d9¢g d:¢g d;¢g d<¢g d=¢g d>¢g d?¢g d@¢g dA¢dBœ
g dC¢g dD¢g dE¢g dF¢g dG¢g dH¢g dI¢g dJ¢g dK¢g dL¢dMœ
g dN¢g dO¢g dP¢g dQ¢g dR¢dSœdTœZ+dUe,dV< dWg dX¢dYdZid[œd\g d]¢d^d_d`daœd[œdbg dc¢dddedfœd[œdgg dh¢didjdkdlœd[œdmg dn¢dod_dkdpœd[œdqg dr¢didsdtdudvœd[œdwg dx¢dodtdyœd[œdzg d{¢d|d}d~œd[œdg d€¢d�djd‚œd[œdƒg d„¢dod_dud…œd[œd†g d‡¢d|dsd`dˆœd[œd‰g dŠ¢d‹djdŒœd[œd�g dŽ¢d�d�d`d‘œd[œd’g d“¢d‹dsdjd”œd[œd•g d–¢d}dud`d—œd[œd˜g d™¢dšdjdkd›œd[œdœg d�¢ddd�dudžœd[œdŸg d ¢d¡d¢d£œd[œd¤g d¥¢dsd�d¦œd[œd§g d¨¢d�d_d_d`d©œd[œdªg d«¢dddjd`d¬œd[œgZ-d­e,d®< g d¯¢g d°¢g d±¢g d²¢g d³¢g d´¢g dµ¢g d¶¢g d·¢g d¸¢g d¹¢g dº¢g d»¢g d¼¢g d½¢g d¾¢g d¿¢g dÀ¢g dÁ¢g dÂ¢g dÃ¢g dÄ¢g dÅ¢dÆœZ.dÇe,dÈ< eG dÉdÊ„ dÊƒƒZ/eG dËdÌ„ dÌƒƒZ0eG dÍdÎ„ dÎƒƒZ1eG dÏdÐ„ dÐƒƒZ2dÑdÒœdÓdÔ„Z3dÕdÖd×œdØdÙ„Z4dÚdÕdÛœdÜdÝ„Z5dÚdÕdÛœdÞdß„Z6dÚdÕdÛœdàdá„Z7�d”dÚdÕdãdäœdådæ„Z8dçdÕdèœdédê„Z9dÕdëdìœdídî„Z:dÕdÕdïdðœdñdò„Z;dÕdódïdôœdõdö„Z<dÕd÷dìœdødù„Z=dÚdúdÛœdûdü„Z>dÚdýdÛœdþdÿ„Z?�d dÚ�dœ�d�d„Z@dÚdód­�dœ�d�d„ZAdÕdÕdÕ�dœ�d�d	„ZBdã�d
dã�dœ�d�d„ZC�d �d �d�dœ�d�d„ZD�d�d�dœ�d�d„ZE�d�d�dœ�d�d„ZF�d�d�dœ�d�d„ZG�ddÕ�d�dœ�d�d„ZH�d�d�dœ�d�d „ZI�d•�ddÕdÕ�d"œ�d#�d$„ZJ�ddÕdÕ�d%œ�d&�d'„ZK�d–�ddÕ�d�d"œ�d(�d)„ZL�d�d�dœ�d*�d+„ZM�d�d�dœ�d,�d-„ZN�d�d�dœ�d.�d/„ZO�d �d �d �d �d �d0�d1œ�d2�d3„ZP�d4�d5�d6�d7œ�d8�d9„ZQdëdë�d:�d;�d<œ�d=�d>„ZRdëdë�d:�d;�d?œ�d@�dA„ZSdëdë�d;�dBœ�dC�dD„ZTdÌdÊdÇ�dEœ�dF�dG„ZUdÌdÊ�dH�dEœ�dI�dJ„ZVdÌdÊ�d;�dEœ�dK�dL„ZWdÌdï�dMœ�dN�dO„ZXdÌdÊ�dPdó�dH�dQœ�dR�dS„ZYdÌdÊ�d:�dT�dUœ�dV�dW„ZZdÌ�dX�dMœ�dY�dZ„Z[dÌ�dX�dMœ�d[�d\„Z\�d;�dPdódódÕ�d]œ�d^�d_„Z]�d;�d;�d;dÕ�d`œ�da�db„Z^dÌdÊ�dPdódódÕdÕ�dcœ�dd�de„Z_dÌdÊdÎdÇdãdã�d;�d;dÐ�dfœ	�dg�dh„Z`dÌdÊ�d:�d;�dUœ�di�dj„ZadÊdÕdï�dkœ�dl�dm„Zb�d4�d5�d:dÑ�dn�doœ�dp�dq„Zc�d dÕ�dœ�dr�ds„ZddÕdÒœ�dt�du„ZedÐ�d�dvœ�dw�dx„ZfdÊ�d�dyœ�dz�d{„ZgdÌ�d�dMœ�d|�d}„Zh�d4�d~�d�dœ�d€�d�„Zi�d5�d~�d�d‚œ�dƒ�d„„Zj�d~�d~�d�d…œ�d†�d‡„Zk�d dÚdÖ�dˆœ�d‰�dŠ„ZldÑdÖ�d‹œ�dŒ�d�„Zm�d4�d5dã�d7œ�dŽ�d�„Zn�d�dÒœ�d‘�d’„Zoep�d“k�rle qeoƒ ¡ dS (—  aF  
Correlate CVE, paper, incident, assessment and signal evidence with T-xxx threats.

Default inputs:
  data/threats.json
  data/taxonomy.json
  zero_days_threats/zero_day.json
  papers_threats/papers.json

Default outputs:
  evidence/threat_correlations.json
  evidence/evidence_index.json
  evidence/correlation_review.json
  evidence/correlation_run.json

The matcher is deterministic and explainable. It uses normalized lexical
similarity, phrase/feature overlap, threat-specific rules, and contradiction
penalties. No network access, external model, or embeddings are required.
é    )ÚannotationsN)ÚCounterÚdefaultdict)ÚasdictÚ	dataclassÚfield)ÚdatetimeÚtimezone)ÚPath)ÚAnyÚIterableÚMappingÚSequencez1.0Zcorrelate_evidencez	^T-\d{3}$z\bCVE-\d{4}-\d{4,7}\bzhttps?://[^\s\"'<>]+z<[^>]+>z[a-z0-9][a-z0-9+.#/_-]*zdata/threads_register.jsonzdata/taxonomy.jsonzzero_day_threats/zero_day.jsonzpapers_threats/papers.jsonzgithub/github.jsonzincidents/incidents.jsonzsignals/signals.jsonÚevidence)ÚthreatsÚtaxonomyÚcvesÚpaperszrepositories:Ú	incidentsÚsignalsÚ
output_dir>d   ÚitsÚthatZduringZacrossZeachZtheZagainstÚpaperZthroughZbutÚaZ	includingZbeÚtoZwhichÚsystemZalsoZbeforeZhaveÚfromÚdataZaboutZintoÚnotZsoftwareÚorZdoZvehiclesZsystemsÚpossibleZsuchÚitZtheirZoneZwereZattacksZareÚcanZatÚasZweZshouldZattackerZprovideZcouldÚallZmostZhadÚifZhavingZwouldZnoZwithinZfurtherZourÚattackZbecauseZbothZcyberZtheyZanZbasedZriskZunderZviaZthanZuseZbyZmultipleÚwhileZafterZusingZmayZoverZwasZofÚ
automotiveZbeenÚandÚisÚanyZonZbetweenZsecurityZtheseZusedÚthisZsomeÚvehicleZhasÚforZcybersecurityZseveralÚotherÚinZ	attackersZmoreÚwithZbeingÚnew)úev chargingúelectric vehicle chargingúcharging infrastructureúcharging stationzcharging stationsúcharge pointzcharge pointsZchargerZchargersÚevseÚcsmszcharging backend)zelectricity gridzenergy gridú
power gridz
smart gridúgrid stabilityzgrid controlúenergy management systemzload balancing)	Zbackendúbackend servicezcloud serviceúcloud platformúapi gatewayúmanagement serverzremote serviceúfleet platformzweb service)Úinfotainmentú	head unitÚheadunitÚivizin-vehicle infotainmentÚdilinkztelematics control unitZtcu)Úecuzelectronic control unitzvehicle controllerzembedded controllerzgateway ecuzcentral gateway)zautonomous drivingúautomated drivingÚadaszautomatic emergency brakingZaebzperception systemúpath planningúself-drivingzradar objectZlidarzcamera perception)z
mobile appzmobile applicationzcustomer accountzidentity providerzauthentication servicezaccount recoveryzaccess token)úoccupant monitoringúcabin camerazcabin sensorÚ
microphoneZ	biometricúdriver monitoring)zroadside unitZrsuztraffic managementztraffic lightzroad infrastructurezintelligent transport systemr   )ÚgnssZgpsZgalileozsatellite navigationúsatellite communication)úsoftware updateúfirmware updateú
ota updateúover-the-air updateúupdate serverzupdate processúcode signing)zsupply chainÚsupplierúthird-party componentZ
dependencyZdependencieszopen source componentzsoftware bill of materialsZsbom)úmodern vehicleúmodern vehiclesúmoderne autoúmoderne voertuigenúsmart vehicleúsmart vehiclesúslimme voertuigenúconnected vehicleúconnected vehicleszconnected carúvehicle dataÚvoertuigdata)úcritical infrastructurezcritical servicezcritical serviceszessential servicezessential serviceszvital infrastructurezvitale infrastructuurzkritieke infrastructuur)Úev_chargingZenergy_gridZbackend_cloudrC   Zvehicle_ecuZautonomous_adasZmobile_identityZoccupant_sensorZroad_infrastructureZsatellite_gnssÚupdate_systemÚsupply_chainZvehicle_intelligenceZcritical_infrastructureÚocppzopen charge point protocol)Úv2gúvehicle-to-gridz	iso 15118)úcan buszcontroller area networkzcan-fdzcan fd)Úudszunified diagnostic servicesz	iso 14229)Údoipz"diagnostics over internet protocolz	iso 13400)Úv2xúc-v2xZcv2xÚdsrczits-g5)Ú	bluetoothZblezbluetooth low energy)zwi-fiÚwifiz802.11)ÚcellularZlteZ4gZ5gZmodemZbaseband)Úrkezremote keyless entryúkeyless entryzkey fobÚnfcznear field communication)úautomotive ethernetÚethernetzsome/ipZsomeip)rj   rk   r#   rn   ro   rp   rs   rt   ru   rv   rx   rz   )zremote attackzremotely exploitableznetwork attackzinternet exposedzinternet-accessibleúremote code executionZrce)úauthentication bypasszbypass authenticationzbrute forceZ
bruteforcezweak credentialszdefault credentialszaccount takeoverzbroken authentication)zphysical accessÚusbÚobdúobd-iiúdiagnostic portzremovable media)
ÚwirelessZradioZrfrs   rt   ru   Úkeylesszsignal injectionZjammingZspoofing)úsupply chain attackzdependency confusionúmalicious updatezthird-party compromiseúcompromised supplier)zprivilege escalationzunauthorized accesszaccess controlzimproper authorization)Ú	injectionzcommand injectionzsql injectionzobject injectionúadversarial exampleúphantom object)zbuffer overflowzheap overflowzuse after freezout-of-boundszmemory corruption)údenial of serviceZdoszresource exhaustionúservice disruptionÚcrash)úai-assistedúai assistedú
ai-poweredz
ai poweredúartificial intelligenceúmachine learningúlarge language modelZllmúautomated exploitationzai exploitationzai-enabled attackzai enabled attack)
ZremoteZauthenticationZphysicalr�   ri   Z	privileger†   Zmemory_corruptionZdenial_of_serviceZai_assisted)zvehicle controlÚsteeringÚbrakingZaccelerationzunsafe behaviorzunsafe behaviourzloss of controlzcontrol vehicle functions)ÚavailabilityZoutageZunavailablerŠ   r‰   Z
immobilizeZ
immobiliseÚdisable)zpersonal datazoccupant dataÚprivacyÚsurveillanceZtrackingúlocation datazbehavioral datazbehavioural data)Ú	espionageúintelligence gatheringZeavesdroppingzsecretly collectzsensitive information)úvehicle theftzsteal vehicleúunlock vehicleúkey cloningúrelay attackzreplay attackÚrolljam)úgrid destabilizationúgrid destabilisationzgrid disruptionúload manipulationzpower outagezgrid-control signals)z
fleet-widez
fleet widezlarge scaleúlarge-scaleúmultiple fleetszmany vehicleszmass compromiseúshared dependency)úroad safetyÚ	collisionr‹   ZfatalityZ
fatalitieszserious injuryzemergency brakingúunsafe driving)Ú	integrityZmanipulationZ	tamperingzunauthorized modificationz
false datazmalicious command)z"critical infrastructure disruptionzcritical service disruptionzessential service disruptionúsocietal disruptionzlarge-scale disruptionzlarge scale disruptionzsystemic disruption)
Zvehicle_controlr•   r—   rš   ÚtheftZgrid_disruptionZfleet_scaleZroad_safetyrª   Zsystemic_disruption)zsecure developmentZtestingztest coverageZverificationZ
validationZfuzzingztype approval)Z
monitoringZ	detectionÚ	telemetryÚloggingúincident responseúanomaly detectionÚhoneypotúlog correlation)Zpatchingzsecurity updateÚunsupportedúend of supportzend-of-lifezvulnerability remediation)ZdisposalZsaleZrentalZ	recyclingzfactory resetzdata deletion)	Ú
regulationZ
regulatoryZ	oversightÚ
governanceZ
complianceZr155Zr156ziso/sae 21434úcyber resilience act)Zdevelopment_testingÚmonitoring_detectionZpatching_supportZdisposal_transferr¶   )ÚassetsÚ	protocolsÚattack_vectorsÚimpactsÚ	lifecyclezdict[str, dict[str, list[str]]]ÚFEATURE_TERMSÚev_charging_general)rj   r9   r8   r7   r5   r:   úT-002çR¸…ëQØ?)ÚnameÚrequires_anyÚboostZv2g_grid_manipulation)rk   rl   r<   r£   Ú
aggregatorg      Ð?çš™™™™™¹?çš™™™™™©?)úT-004úT-003rÀ   Zgrid_attack_path)r¡   r¢   úgrid-controlr=   r;   g
×£p=
Ç?çìQ¸…ëÁ?)rÉ   rÈ   Úinfotainment_headunit)rC   rD   rE   rF   rG   gHáz®GÑ?ç{®Gáz´?ç{®Gáz¤?)zT-023úT-047úT-064Zgnss_jamming_spoofing)zgnss jammingzgnss spoofingzgps jammingzgps spoofinggìQ¸…ëÑ?)zT-035úT-066úT-054Zperception_manipulation)rˆ   zradar object injectionzsensor spoofingr‡   zperception attackç¸…ëQ¸¾?çìQ¸…ë±?ç¸…ëQ¸®?)rÑ   úT-012úT-013úT-010Údigital_vehicle_theft)rœ   rw   rŸ   r    rž   r�   )zT-044zT-043Zvehicle_espionage_assessment)zdigital espionagezvehicle espionagerš   ÚspionageÚspionagerisicoÚcyberspionager›   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   zdata over inzittendenúnational securityúnationale veiligheidústate actorústatelijke actorç333333Ó?ç¸…ëQ¸Î?)úT-017úT-018Z*vehicle_architecture_difference_assessment)zvehicle architecturezvehicle architectureszdifferent vehicle architecturesz*materially different vehicle architectureszarchitecture segmentationz/critical functionality was effectively isolatedz critical functionality is onlinez)critical vehicle functionality was onlinezisolated critical functionszconnected critical functionszsystem integrationçÃõ(\�ÂÕ?)zT-051úT-026Z#remote_oem_critical_function_access)z
remote oemzremote oem accesszmanufacturer accesszmanufacturer diagnosticszdirect digital accesszremote diagnosticszcritical functionalityzcritical vehicle functionalityzbattery managementzpower supplyzremote software controlzremotely changezremotely disablezremote disablezkill switch)rÏ   úT-058úT-001Zvehicle_ota_lifecycle_control)
rU   zota updateszota software updatezota software updatesrV   zover-the-air updatesrS   zsoftware updateszpost-sale softwarezpost-sale modification)úT-060rÏ   ræ   Z!vehicle_sensor_supplier_espionage)zvehicle sensorzvehicle sensorsúsensor datarN   rO   rY   Úleverancierúforeign supplierúbuitenlandse leverancierzforeign manufacturerzchinese vehicleszchinese vehiclezchinese manufacturerç)\�Âõ(Ì?)rä   rã   Zvehicle_tracking_privacy)zlocation trackingzmovement datar™   zdriver trackinggR¸…ëQÈ?gq=
×£pÍ?)zT-016zT-040úT-036Zoccupant_monitoring)rM   rN   rP   zcabin microphone)zT-015rï   zT-041Úvehicle_update_compromise)rU   rV   rW   rT   r„   rX   )ré   úT-033rÏ   Zvehicle_communications)rp   rq   rr   ry   zcellular modemÚ
telematicsgÃõ(\�ÂÅ?)zT-059rÒ   rÏ   Zexternal_interfaces)r}   r~   r   r€   zexternal interface)zT-042úT-062rÐ   Zidentity_keys_trust)zcryptographic keyzprivate keyzhardcoded keyzhard-coded keyzcertificate validationzcertificate verificationztrust anchorzdigital certificatezcredential storagezcredential exposureZimpersonationç{®GázÄ?g
×£p=
·?)zT-065zT-063r¸   )r°   r±   r®   r­   r¯   r²   )zT-034zT-057Zsupply_chain_compromise)rƒ   r…   rZ   zdependency compromise)rñ   zT-048zT-038ró   Zbackend_services)r>   r?   rA   rB   r@   )rç   zT-067zT-011zlist[dict[str, Any]]ÚTHREAT_RULES)Úfleetr¤   zmultiple vehiclesr¦   zmobility service)Zgridzenergy managementr;   rÊ   Zelectricity)rk   rl   rÅ   r<   r£   )Zsabotageztargeted attackzphysical damagezmalicious control)rf   ZviprÝ   zprotected transport)zcyber-physicalzmoving weaponzphysical attackzremote control)Zbotnetzdistributed denialzmalicious trafficzmany endpoints)r§   r¨   r©   r”   r“   )Ú
autonomousrI   rJ   Z
perceptionrK   )r÷   rI   rL   )rš   rÚ   rÛ   rÜ   Úintelligencezsecret collectionr˜   r^   ra   rc   rd   re   rÝ   rÞ   rß   rà   )Zsensorrê   rY   rë   rì   rí   rš   rÚ   rÛ   rø   r^   ra   rc   rd   re   )r«   Z
populationzpublic servicesr¤   )Ztrustzpublic confidencezrepeated incident)r·   Zcraúregulatory uncertainty)rµ   zlegal obligationrù   )zoem supportr´   r³   zmanufacturer insolvency)zsupplier failurezsupplier insolvencyzcritical supplierzservice withdrawal)rŒ   r�   rŽ   zaai poweredr�   r‘   r�   r’   )ZunderestimationZpreparednesszcontinuity planzfleet-scale)rR   Zforeignzuntrusted partyZsovereignty)zfinancial healthZ
insolvencyZ
bankruptcyzreduced support)r¦   zsingle point of failurer¥   zmultiple providers)rè   rÉ   rÈ   zT-005zT-006zT-007zT-009rØ   rÖ   r×   rã   rä   zT-020zT-021zT-028zT-029zT-030zT-031zT-037zT-049zT-055zT-056zT-069údict[str, list[str]]ÚTHREAT_CONTEXT_REQUIREMENTSc                   @  s`  e Zd ZU ded< dZded< dZded< dZded< dZded< dZded< e	e
d	�Zd
ed< e	e
d	�Zd
ed< e	e
d	�Zd
ed< e	e
d	�Zd
ed< e	e
d	�Zd
ed< e	e
d	�Zd
ed< e	e
d	�Zd
ed< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< dZded< e	ed	�Zded< e	ed	�Zded< dS ) ÚThreatÚstrÚ	threat_idÚ ÚtitleÚtitle_nlÚcategoryÚsourceÚprimary_domain©Údefault_factoryú	list[str]Úsecondary_domainsÚtagsÚtermsÚsurfacesÚevidence_typesÚpaper_queriesÚsignal_queriesÚdescriptionÚdomain_rationaleÚimpact_rationaleÚlikelihood_rationaleÚrisk_rationaleÚcommentsÚadditional_commentsÚrisk_ratingÚsearch_textúset[str]Útokensúdict[str, set[str]]ÚfeaturesN)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r   r  r  r  r  r   Úlistr  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  r  Úsetr  Údictr  © r#  r#  úcorrelate_evidence.pyrü   î  s0   
rü   c                   @  st  e Zd ZU ded< ded< ded< ded< dZded< dZded< dZded	< d
Zded< dZded< dZ	ded< e
ed�Zded< dZded< d
Zded< dZded< e
ed�Zded< e
ed�Zded< e
ed�Zded< e
ed�Zded< e
ed�Zded< e
ed�Zded< dZded < dZded!< dZded"< e
ed�Zd#ed$< e
ed�Zd%ed&< e
edd'�Zd(ed)< d
S )*ÚEvidencerý   Úevidence_idÚevidence_typer   r  rÿ   Ú	publishedÚurlÚseverityNúfloat | NoneÚcvssÚcvss_versionFÚboolÚpending_analysisr  r  ÚauthorsÚvenueú
int | NoneÚyearÚdoiÚsourcesÚmatched_termsÚtopicsÚvendorsr  Úlinked_cvesr   ÚintÚcited_byÚopen_accessr  r  r  r  r  )r  Úreprúdict[str, Any]Ú
raw_record)r  r  r  r  r(  r)  r*  r,  r-  r/  r   r   r0  r1  r3  r4  r5  r6  r7  r8  r  r9  r;  r<  r  r!  r  r"  r  r?  r#  r#  r#  r$  r%  
  s4   
r%  c                   @  sª   e Zd ZU dZded< dZded< dZded< dZded< dZded< dZ	ded< dZ
ded	< dZded
< dZded< dZded< dZded< dZded< dZded< dS )ÚScoreBreakdownç        ÚfloatÚlexicalr   r  r¹   rº   r»   r¼   r½   ÚdomainÚ
rule_bonusÚcontext_penaltyÚgeneric_penaltyÚfinalN)r  r  r  rC  r  r   r  r¹   rº   r»   r¼   r½   rD  rE  rF  rG  rH  r#  r#  r#  r$  r@  (  s   
r@  c                   @  s–   e Zd ZU ded< ded< ded< ded< ded< ded< ded	< ded
< ded< ded< ded< ded< ded< ded< ded< ded< ded< dS )ÚMatchResultrý   r&  r'  rþ   rB  Úrelevance_scoreÚevidence_strength_scoreÚevidence_strengthÚ
confidenceÚstatusr   r)  r(  r  Úmatched_axesrú   Úmatched_featuresr6  Úrules_firedÚexplanationúdict[str, float]Úscore_breakdownN)r  r  r  r  r#  r#  r#  r$  rI  9  s"   
rI  zargparse.Namespace)Úreturnc                  C  s  t jdd�} | jdttd d� | jdttd d� | jdttd	 d� | jd
ttd d� | jdttd d� | jdttd d� | jdttd d� | jdttd d� | jdtdd� | jdtdd� | jdtdd� | jdtdd� | jddd� | jdd d� | jd!d"d#� | jd$d"d#� | jd%d"d#� | jd&d"d'd(� | jd)g d*¢d+d,� |  ¡ }d-|j  k�rxd.k�s„n |  	d/¡ d-|j
  k�ržd.k�sªn |  	d0¡ |j|j
k�rÂ|  	d1¡ |jd2k �rØ|  	d3¡ |jd2k �rî|  	d4¡ |j�r|j ¡ |_t |j¡�s|  	d5¡ |S )6NzƒCorrelate CVEs, papers, repositories, incidents, assessments and signals to T-xxx threats using explainable deterministic matching.)r  z	--threatsr   )ÚtypeÚdefaultz
--taxonomyr   z--cvesr   z--papersr   z--repositoriesÚrepositoriesz--incidentsr   z	--signalsr   z--output-dirr   z--accept-thresholdç�Âõ(\�â?z--review-thresholdrÁ   z--top-ké
   z--max-accepted-per-evidenceé   z--threat-idz#Only process one threat, e.g. T-023)Úhelpz--evidence-idzOnly process one evidence IDz--validate-onlyÚ
store_true)Úactionz--include-rejectedz	--explainz--forcezOverwrite existing output files)r^  r\  z--log-level)ÚDEBUGÚINFOZWARNINGZERRORr`  )ÚchoicesrW  rA  ç      ð?z*--review-threshold must be between 0 and 1z*--accept-threshold must be between 0 and 1z8--review-threshold must be lower than --accept-thresholdé   z--top-k must be at least 1z.--max-accepted-per-evidence must be at least 1z--threat-id must match T-xxx)ÚargparseÚArgumentParserÚadd_argumentr
   ÚDEFAULT_PATHSrB  r:  Ú
parse_argsÚreview_thresholdÚerrorÚaccept_thresholdÚtop_kÚmax_accepted_per_evidencerþ   ÚupperÚTHREAT_ID_REÚmatch)ÚparserÚargsr#  r#  r$  rh  N  sT    ÿý





rh  rý   ÚNone)ÚlevelrU  c                 C  s   t jtt | ƒddd� d S )Nz'%(asctime)s %(levelname)-8s %(message)sz%Y-%m-%d %H:%M:%S)rt  ÚformatZdatefmt)r®   ZbasicConfigÚgetattr)rt  r#  r#  r$  Úconfigure_logging~  s
    ýrw  r   )ÚvaluerU  c                 C  sH   | d u rdS t  t| ƒ¡}t d|¡}| dd¡}t dd|¡}| ¡ S )Nrÿ   ZNFKCõ   Â ú ú\s+)	ÚhtmlÚunescaperý   ÚunicodedataZ	normalizeÚreplaceÚreÚsubÚstrip©rx  Útextr#  r#  r$  Únormalize_whitespace†  s    r…  c                 C  s8   t | ƒ ¡ }| dd¡}t dd|¡}t dd|¡ ¡ S )Nzvehicle to gridrl   z[^a-z0-9+.#/_\- ]+rz  r{  )r…  Úlowerr  r€  r�  r‚  rƒ  r#  r#  r$  Únormalize_text�  s    r‡  c                 C  sV   t  t| pdƒ¡}t |¡}|r0| d¡ d¡S t d|¡}t	|ƒ}| 
d¡rR|S dS )Nrÿ   r   z.,);]rz  ©zhttp://zhttps://)r|  r}  rý   ÚURL_REÚsearchÚgroupÚrstripÚHTML_TAG_REr�  r…  Ú
startswith)rx  r„  rp  Zcleanr#  r#  r$  Úextract_url—  s    
r�  ú[;,]r  )rx  Ú
separatorsrU  c                 C  sš   | d u rg S t | tƒr| }n6t | ttfƒr4t| ƒ}nt | tƒrLt || ¡}n| g}g }tƒ }|D ]4}t|ƒ}| ¡ }|r`||vr`| 	|¡ | 
|¡ q`|S ©N)Ú
isinstancer   Útupler!  rý   r€  Úsplitr…  ÚcasefoldÚappendÚadd)rx  r‘  ÚitemsÚresultÚseenÚitemÚcleanedÚkeyr#  r#  r$  Úlistify¡  s$    



rŸ  zIterable[Any])ÚvaluesrU  c                 C  s   d  dd„ | D ƒ¡S )NÚ
c                 s  s   | ]}t |ƒrt |ƒV  qd S r’  )r…  ©Ú.0Úvr#  r#  r$  Ú	<genexpr>¸  ó    z join_nonempty.<locals>.<genexpr>)Újoin)r   r#  r#  r$  Újoin_nonempty·  s    r¨  r  )r„  rU  c                 C  s   t | ƒ}dd„ t |¡D ƒS )Nc                 S  s,   h | ]$}t |ƒd kr|tvr| ¡ s|’qS )é   )ÚlenÚ	STOPWORDSÚisdigit©r£  Útokenr#  r#  r$  Ú	<setcomp>½  s   ÿztokenize.<locals>.<setcomp>)r‡  ÚTOKEN_REÚfindall)r„  Z
normalizedr#  r#  r$  Útokenize»  s    ÿr²  r.  )r„  ÚphraserU  c                 C  sZ   dt | ƒ› d�}t |ƒ}|s dS t|ƒdkrR| ¡ rRt dt |¡› d�|¡d uS ||v S )Nrz  Fr©  z(?<![a-z0-9])z(?![a-z0-9]))r‡  rª  Úisalnumr€  rŠ  Úescape)r„  r³  ÚtÚpr#  r#  r$  Úcontains_phraseÃ  s    r¸  zSequence[str])r„  ÚphrasesrU  c                   s   t ‡ fdd„|D ƒƒS )Nc                 3  s   | ]}t ˆ |ƒV  qd S r’  )r¸  )r£  r³  ©r„  r#  r$  r¥  Î  r¦  zhas_any.<locals>.<genexpr>)r,   )r„  r¹  r#  rº  r$  Úhas_anyÍ  s    r»  r  c                 C  sN   dd„ t D ƒ}t  ¡ D ]2\}}| ¡ D ] \}}t| |ƒr&||  |¡ q&q|S )Nc                 S  s   i | ]}|t ƒ “qS r#  )r!  ©r£  Úaxisr#  r#  r$  Ú
<dictcomp>Ò  r¦  z$extract_features.<locals>.<dictcomp>)r¾   r™  r»  r˜  )r„  r  r½  ÚlabelsÚlabelr¹  r#  r#  r$  Úextract_featuresÑ  s    
rÁ  r+  c              	   C  s6   z| dv rW d S t | ƒW S  ttfy0   Y d S 0 d S ©N)Nrÿ   )rB  Ú	TypeErrorÚ
ValueError©rx  r#  r#  r$  Ú
safe_floatÚ  s    
rÆ  r2  c              	   C  s6   z| dv rW d S t | ƒW S  ttfy0   Y d S 0 d S rÂ  )r:  rÃ  rÄ  rÅ  r#  r#  r$  Úsafe_intã  s    
rÇ  r
   )ÚpathrU  c                 C  s¨   |   ¡ std| › �ƒ‚z>| jddd��}t |¡W  d   ƒ W S 1 sH0    Y  W nN tjy¢ } z4td| › d|j› d|j› d|j	› �ƒ|‚W Y d }~n
d }~0 0 d S )	NzInput file not found: Úrúutf-8©ÚencodingzInvalid JSON in z: line z	, column ú: )
ÚexistsÚFileNotFoundErrorÚopenÚjsonÚloadZJSONDecodeErrorrÄ  ÚlinenoÚcolnoÚmsg)rÈ  ÚhandleÚexcr#  r#  r$  Ú	load_jsonì  s    .rØ  )r   Úpreferred_keysrU  c                 C  s–   t | tƒrdd„ | D ƒS t | tƒr~|D ]*}|  |¡}t |tƒr&dd„ |D ƒ  S q&| r~tdd„ |  ¡ D ƒƒr~dd„ |  ¡ D ƒS tdd |¡› �ƒ‚d S )	Nc                 S  s   g | ]}t |tƒr|‘qS r#  ©r“  r"  ©r£  rœ  r#  r#  r$  Ú
<listcomp>ø  r¦  z%records_from_json.<locals>.<listcomp>c                 S  s   g | ]}t |tƒr|‘qS r#  rÚ  rÛ  r#  r#  r$  rÜ  ý  r¦  c                 s  s   | ]}t |tƒV  qd S r’  rÚ  r¢  r#  r#  r$  r¥  ÿ  r¦  z$records_from_json.<locals>.<genexpr>c                 S  s$   g | ]\}}t || d |¡d�‘qS )Úid)rÝ  )r"  Úget)r£  Úkr¤  r#  r#  r$  rÜ     r¦  z8Could not find records; expected a list or one of keys: ú, )	r“  r   r"  rÞ  r%   r   r™  rÄ  r§  )r   rÙ  rž  rx  r#  r#  r$  Úrecords_from_jsonö  s    



rá  )ÚexistingÚincomingrU  c                 C  sf   t | ƒ}t |ƒ}|s|S |r,| ¡ | ¡ kr0|S | ¡ | ¡ v rD|S | ¡ | ¡ v rX|S |› d|› �S )Nr¡  )r…  r–  )râ  rã  r   Úbr#  r#  r$  Ú
merge_text  s    rå  zIterable[str]c                 C  sT   t | ƒ}dd„ |D ƒ}|D ]4}t|ƒ}|r| ¡ |vr| |¡ | | ¡ ¡ q|S )Nc                 S  s   h | ]}|  ¡ ’qS r#  )r–  rÛ  r#  r#  r$  r¯    r¦  zmerge_unique.<locals>.<setcomp>)r   r…  r–  r—  r˜  )râ  rã  rš  r›  rœ  r�  r#  r#  r$  Úmerge_unique  s    
ræ  zlist[Threat])Úthreats_pathÚtaxonomy_pathrU  c                 C  s  t | ƒ}t |ƒ}t|dƒ}t|dƒ}i }|D �]}t| d¡ƒ ¡ }t |¡sPq,t|t| d¡ƒt| d¡ƒt| d¡ƒt| d¡ƒt| d¡p˜| d	¡ƒt| d
¡ƒt| d¡ƒt| d¡ƒdd„ t| d¡ƒD ƒt| d¡ƒt| d¡ƒt| d¡ƒt| d¡ƒt| d¡ƒt| d¡ƒt| d¡ƒd�||< q,|D �]}t| d¡ƒ ¡ }t |¡�s`�q8| 	|t|d�¡}	t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t|	jt| d	¡�pæ| d¡ƒƒ|	_t|	jt| d
¡ƒƒ|	_t|	jt| d¡ƒƒ|	_t|	jt| d¡ƒƒ|	_t|	jdd„ t| d¡ƒD ƒƒ|	_t|	jt| d¡ƒƒ|	_t|	jt| d¡ƒƒ|	_t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t
|	j| dd¡ƒ|	_t
|	j| d¡�p| d ¡�pdƒ|	_t| d!¡| d"¡| d#¡| d$¡gƒ}
t
|	j|
ƒ|	_�q8| ¡ D ]�}	t|	j|	j|	j|	jd%  |	j¡d%  |	j¡d%  |	j¡d%  |	j¡d%  |	j¡d%  |	j¡|	j|	j!|	j|	j|	j"|	j|	jgƒ|	_#t$|	j#ƒ|	_%t&|	j#ƒ|	_'�q^t(| ¡ d&d'„ d(�}|�st)d)ƒ‚|S )*N)r   Úrecords)r   r   ré  rÝ  r   r  r  r  Zsecondary_domainr  r	  r
  r  c                 S  s   g | ]}t |ƒ‘qS r#  ©r‡  ©r£  rx  r#  r#  r$  rÜ  3  r¦  z load_threats.<locals>.<listcomp>r  r  r  r  r  r  r  Z
risk_level)rþ   r   r  r  r  r  r	  r
  r  r  r  r  r  r  r  r  r  ©rþ   rÿ   r  c                 S  s   g | ]}t |ƒ‘qS r#  rê  rë  r#  r#  r$  rÜ  N  r¦  r  r  r  Zcombined_ratingr  Zphysical_safetyZeconomic_impactZsocial_impactZterritorial_impactrz  c                 S  s   t | j d¡d ƒS )Nú-rc  ©r:  rþ   r•  )r¶  r#  r#  r$  Ú<lambda>|  r¦  zload_threats.<locals>.<lambda>©rž  z6No valid T-xxx threats found in threats/taxonomy input)*rØ  rá  r…  rÞ  rn  ro  rp  rü   rŸ  Ú
setdefaultrå  r   r  r  r  ræ  r  r	  r
  r  r  r  r  r  r  r  r  r  r  r¨  r   r  r§  r  r  r  r²  r  rÁ  r  ÚsortedrÄ  )rç  rè  Z	base_dataZtaxonomy_dataZbase_recordsZtaxonomy_recordsZmergedÚrecordrþ   ÚthreatZ
dimensionsr   r#  r#  r$  Úload_threats  s¶    



ï
ÿþþü





ïrõ  r>  zEvidence | None)ró  rU  c                 C  s‚  t |  d¡p|  d¡p|  d¡ƒ ¡ }|s.d S g }|  dg ¡p@g D ]>}t|tƒrjt | d¡pd| d¡ƒ}nt |ƒ}|rB| |¡ qBd |¡}t|  d¡ƒ}t|  d	¡ƒ}t |  d
¡ƒ}|rÆt||gƒ}t|  d¡ƒ}	t	|  d¡ƒ}
t
||d |¡d |¡d |	¡gƒ}t |¡}t|d||t |  d¡ƒ|
t |  d¡ƒt|  d¡ƒt |  d¡ƒt|  dd¡ƒt||ƒ|	tg |ƒ|t|ƒt|ƒ| d�S )NrÂ   rÝ  Zcve_idÚvulnerabilitiesr„  r  rz  r6  ÚkeywordsÚkeywordr  r)  Úcver(  Úscorer,  r-  r/  F)r&  r'  r   r  r(  r)  r*  r,  r-  r/  r6  r  r9  r  r  r  r?  )r…  rÞ  rn  r“  r"  r—  r§  rŸ  ræ  r�  r¨  Ú	CVE_ID_REr±  r%  rÆ  r.  r²  rÁ  )ró  r&  Zvulnerability_textsrœ  r„  r  r6  r÷  rø  r  r)  r  r   r#  r#  r$  Únormalize_cve‚  sZ    &

û
ïrü  c           	      C  s¤  t |  d¡p(|  d¡p(|  d¡p(|  d¡ƒ}|s4d S t |  d¡ƒ}t |  d¡ƒ}t|  d¡ƒ}t|  d¡ƒ}t|  d¡ƒ}tt|  d	¡ƒt|  d
¡ƒƒ}t|dd„ t |› d|› �¡D ƒƒ}t||d |¡d |¡d |¡d |¡|  d¡gƒ}t|d||t |  d¡�p|  d¡ƒt	|  d¡�p(|  d¡ƒt|  d¡ƒt |  d¡ƒt
|  d¡ƒt |  d¡ƒt|  d¡ƒ||||t
|  d¡ƒ�p€dt|  dd¡ƒ|t|ƒt|ƒ| d�S )NrÝ  r4  Zarxiv_idr   Úabstractr6  r7  r8  r   r9  c                 S  s   g | ]}|  ¡ ‘qS r#  ©rn  )r£  Úmr#  r#  r$  rÜ  Â  r¦  z#normalize_paper.<locals>.<listcomp>rz  r1  r   Údater3  r)  Zpdf_urlr0  r5  r;  r   r<  F)r&  r'  r   r  r(  r)  r0  r1  r3  r4  r5  r6  r7  r8  r9  r;  r<  r  r  r  r?  )r…  rÞ  rŸ  ræ  rû  r±  r¨  r§  r%  r�  rÇ  r.  r²  rÁ  )	ró  r&  r   rý  r6  r7  r8  Úexplicit_cvesr  r#  r#  r$  Únormalize_paper³  s^    &ÿþ$ù	ër  c                 C  sê  t |  d¡p2|  d¡r&d|  d¡› �ndp2|  d¡ƒ}|s>d S t |  d¡ƒ}t |  d¡ph|  d¡ph|ph|ƒ}t |  d¡ƒ}t |  d	¡ƒ}t|  d
¡ƒ}|  d¡}t|tƒs®i }|  d¡}t|tƒsÆi }|  d¡}	t|	tƒsÞi }	g }
t|
t| d¡ƒƒ}
t|
t| d¡ƒƒ}
t|
t| d¡ƒƒ}
t|
t|	 d¡ƒƒ}
dd„ t t|||gƒ¡D ƒ}t|||||d 	|¡d 	|
¡d 	|¡|  d¡|  d¡g
ƒ}t
|d||t |  d¡�p¶|  d¡�p¶|  d¡ƒt|  d¡ƒdg|
|tg |ƒ|t|ƒt|ƒ| d�S )NrÝ  Ú	full_namezgithub:rÿ   r)  r   rÂ   r  Úreadmer7  ÚclassificationÚ	relevanceÚ	discoveryZautomotive_term_hitsZev_charging_term_hitsZsecurity_term_hitsZmatched_queriesc                 S  s   g | ]}|  ¡ ‘qS r#  rþ  ©r£  rp  r#  r#  r$  rÜ  ?  s   ÿz(normalize_repository.<locals>.<listcomp>rz  ZlanguageÚlicenseÚ
repositoryZ	pushed_atZ
updated_atÚ
created_atZGitHub)r&  r'  r   r  r(  r)  r5  r6  r7  r9  r  r  r  r?  )r…  rÞ  rŸ  r“  r"  ræ  rû  r±  r¨  r§  r%  r�  r²  rÁ  )ró  r&  r  r   r  r  r7  r  r  r  r6  r  r  r#  r#  r$  Únormalize_repositoryå  sò    
ÿüù
ÿ
ÿþüÿÿÿ





ÿÿþ	ÿÿþ	ÿÿþ	ÿÿþ	ýÿþöÿýÿþÿçr  zMapping[str, Any])ró  rž  rU  c                 C  s   |   |¡}t|tƒr|S i S r’  )rÞ  r“  r"  )ró  rž  rx  r#  r#  r$  Únested_dicts  s    

r  ztuple[list[str], list[str]]c           
      C  sV  g }g }|   d¡pg }t|tƒr&|g}t|tƒs4g }|D ]¨}t|tƒrtt|ƒ}| d¡rd| |¡ q8|r8| |¡ q8t|tƒs€q8t|  d¡p¨|  d¡p¨|  d¡p¨|  d¡ƒ}|rº| |¡ t|  d¡pÎ|  d¡ƒ}|r8| |¡ q8t|   d¡�p|   d¡�p|   d¡ƒ}|�r| |¡ t|   d¡�p.|   d¡ƒ}	|	�rB| |	¡ t	g |ƒt	g |ƒfS )	Nr5  rˆ  Ú	publisherrÂ   Zorganizationr  r)  Úlink)
rÞ  r“  r"  r   rý   r…  rŽ  r—  r�  ræ  )
ró  Úsource_namesÚsource_urlsr5  r  r�  r  Z
source_urlZdirect_sourceZ
direct_urlr#  r#  r$  Úcollect_source_metadata  s„    


ÿÿÿÿ

ÿþüÿ
þÿÿýÿþÿþr  rÿ   )ró  Úsource_kindrU  c                 C  sœ   t |  d¡p,|  d¡p,|  d¡p,|  d¡p,|ƒ}dddddddddddddddddd	œ}||v rh|| S d|v rtdS d|v r€dS d|v rŒdS d|v r˜dS dS )
NZrecord_typer'  Zintelligence_typerV  ÚincidentÚ
assessmentÚsignalÚ	narrative)r  zcyber incidentZcyber_incidentr  zstrategic assessmentZstrategic_assessmentzthreat assessmentZthreat_assessmentr  zstrategic signalZstrategic_signalZobservationúthreat observationZthreat_observationr  zstrategic narrativeÚstrategic_narrative)r‡  rÞ  )ró  r  Zexplicit_typeÚaliasesr#  r#  r$  Údetermine_intelligence_typeÔ  sN    
ÿþýû	ìr  )ró  r'  rU  c                 C  sú   t |  d¡pF|  d¡pF|  d¡pF|  d¡pF|  d¡pF|  d¡pF|  d¡ƒ}|rR|S t |  d¡pp|  d	¡pp|  d
¡ƒ}t |  d¡p’|  d¡p’|  d¡ƒ}|sždS t |d | ¡  d |  d¡¡ ¡ d d…  ¡ }dddddœ}| |d¡}|› d|› �S )NZincident_idZ	signal_idZnarrative_idZassessment_idr&  rÝ  Zuuidr   rÂ   Úheadliner(  r   r  rÿ   úrÊ  é   ZINCZASMZSIGZNAR)r  r  r  r  ZINTrí  )r…  rÞ  ÚhashlibÚsha256r–  ÚencodeÚ	hexdigestrn  )ró  r'  Ú	candidater   r(  ÚdigestÚprefixesÚprefixr#  r#  r$  Úintelligence_id	  sp    
ÿþýüûù

ÿý
ÿýÿþýüÿúÿøüþr'  c           8   /   C  st  t | |d�}t| |ƒ}|sd S t|  d¡p@|  d¡p@|  d¡p@|ƒ}t|  d¡p€|  d¡p€|  d¡p€|  d¡p€|  d	¡p€|  d
¡ƒ}t| dƒ}t| dƒ}t| dƒ}t| dƒ}	t| dƒ}
t| dƒ}t| dƒ}t| dƒ}t| dƒ}tt|  d¡ƒt|  d¡ƒƒ}t|t|  d¡ƒƒ}tt|  d¡ƒt|  d¡ƒƒ}t|t|  d¡ƒƒ}tt|  d¡ƒt|  d¡ƒƒ}tt|  d¡ƒt|  d¡ƒƒ}tt|  d¡ƒt|  d¡ƒƒ}t|  d ¡ƒ}tt| d!¡ƒt|  d"¡ƒƒ}t| d#¡ƒ}tt| d$¡ƒt|  d$¡ƒƒ}t| d%¡ƒ}t| d&¡ƒ}tt| d'¡ƒt|  d(¡ƒƒ}t| d)¡ƒ}t| d*¡ƒ}t| d+¡ƒ}t| d,¡ƒ}t| d-¡ƒ}tt|  d.¡ƒt|  d/¡ƒƒ} tt|  d0¡ƒt|  d1¡ƒƒ}!tt|  d2¡ƒt|  d3¡ƒƒ}"tt| d4¡ƒt|  d4¡ƒƒ}#t|#d5d6„ t t	||| d¡| d¡gƒ¡D ƒƒ}#t
| ƒ\}$}%t| d7¡�pd| d8¡�pd|  d7¡�pd|  d9¡�pd|  d:¡ƒ}&|%�rv|%d; nt|  d<¡ƒ}'t|  d=¡ƒ}(t|  d>¡ƒ})t| d?¡�p¶|  d?¡ƒ}*t| d@¡�pÐ|  d@¡ƒ}+t|  d¡�p|  dA¡�p|  dB¡�p|  dC¡ƒ},t|  dD¡�p(|  dE¡�p(|  dF¡ƒ}-t|  dG¡�pB|  dH¡ƒ}.t|  dI¡�p\|  dJ¡ƒ}/g }0| dK¡�r„|0 t| dK¡ƒ¡ dLD ]0}1|  |1¡}2|2d dMg i fv�rˆ|0 t|2ƒ¡ �qˆt||ƒ}3t|3|ƒ}3t|3|ƒ}3t|3|ƒ}4t||ƒ}5t|5|ƒ}5t||ƒ}6t	|||||,|-|(|)|*|+|.|/dN |¡dN |¡dN |¡dN |¡dN |¡dN |¡dN |¡dN |¡dN |¡dN |¡dN |¡dN |#¡t| d¡ƒdN |¡t| d¡ƒt|	 dO¡ƒt|	 dP¡ƒt|	 dQ¡ƒt|
 dR¡ƒt|
 dE¡ƒdN |¡dN |¡dN |¡t| dS¡ƒt| dT¡ƒdN |¡dN |¡dN |0¡dN |!¡dN | ¡dN |"¡dN |$¡g,ƒ}7t|||||&|'|+|$|4|5|6|#|7t|7ƒt|7ƒ| dU�S )VN©r  r   rÂ   r  Zsummaryr  rý  r  r„  ZcontentÚdatesr'   ÚimpactÚcyber_physicalÚautomotive_relevanceÚactorsÚ	geographyr   ÚcampaignÚsectorsZdomainsZindustryÚthemesZthemeZnarrative_tagsr	  r÷  Úsignal_typesZsignal_typer¹   Zaffected_assetsrº   Zattack_vectorr»   Úinitial_accessÚ
techniquesÚauthentication_issuesrö  Z
categoriesr¼   ÚnamesrV  Z
motivationÚ	countriesÚregionsZ
signal_idsÚrelated_signalsZincident_idsÚrelated_incidentsZnarrative_idsÚrelated_narrativesr   c                 S  s   g | ]}|  ¡ ‘qS r#  rþ  r  r#  r#  r$  rÜ    s   ÿz*normalize_intelligence.<locals>.<listcomp>r(  Zfirst_observedr   r  r   r)  rN  Ú
event_typerM  r*  r  ZinterpretationZanalysisÚ	rationaleÚreasonZwhy_it_mattersÚtrendÚ	directionZtime_horizonÚhorizonÚscope)ZscalerB  Zaffected_countZaffected_locationsZaffected_organizationsrÿ   rz  Zdigital_actionZphysical_effectZpotential_consequencer  ZattributionZcampaign_name)r&  r'  r   r  r(  r)  r*  r5  r6  r7  r  r9  r  r  r  r?  )r  r'  r…  rÞ  r  ræ  rŸ  rû  r±  r¨  r  r�  r—  r§  r%  r²  rÁ  )8ró  r  r'  r&  r   r  r)  r'   r*  r+  r,  r-  r.  Zevidence_metadatar/  r0  r1  r	  r2  r¹   rº   r»   r3  r4  r5  rö  Zimpact_categoriesZactor_namesZactor_typesZactor_motivationsr7  r8  r9  r:  r;  r9  r  r  r(  Zprimary_urlrN  r<  rM  r*  Znarrative_statementr=  r?  rA  Zscale_termsrž  rx  Zstructured_termsr6  r7  r  r  r#  r#  r$  Únormalize_intelligenceB  sî   þþ
ÿþü
ÿþýüú	þþþþþþþþþþþþþþþþÿÿÿü	ÿÿÿü	ÿÿÿÿÿü	ÿÿÿÿÿþþþÿÿü	üÿþþÿÿÿþýû
ÿÿýÿÿþþÿþüÿýþþÿÿ
ÿþþþþþþþÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¥^ÿÿìrC  c                 C  s   t | dd�S )Nr  r(  ©rC  ©ró  r#  r#  r$  Únormalize_incident	  s    þrF  c                 C  s   t | dd�S )Nr  r(  rD  rE  r#  r#  r$  Únormalize_signal  s    þrG  c                 C  s   t | dd�S )Nr  r(  rD  rE  r#  r#  r$  Únormalize_narrative  s    þrH  zlist[Evidence])Úcve_pathÚ
paper_pathÚrepositorie_pathÚincident_pathÚsignal_pathrU  c              	   C  sÖ  t | ƒ}t |ƒ}t tƒ}t |ƒ}t |ƒ}	t|dƒ}
t|dƒ}t|dƒ}t|dƒ}t|	dƒ}g }|
D ]}t|ƒ}|rb| |¡ qb|D ]}t|ƒ}|r‚| |¡ q‚|D ]}t|ƒ}|r¢| |¡ q¢|D ]}t|ƒ}|rÂ| |¡ qÂ|D ]}t|ƒ}|râ| |¡ qât	 
dt|
ƒt|ƒt|ƒt|ƒt|ƒ¡ i }tƒ }|D ]X}|j|j ¡ f}||  d7  < | |¡}|d u �s‚t|jƒt|jƒk�r4|||< �q4tdd	„ | ¡ D ƒƒ}|�r¶t	 d
|¡ t| ¡ ƒ}|jdd„ d� |S )N)ré  r   r™  )ré  r   r™  )rX  ré  r™  )r   ré  r™  )ré  r   r™  z\Loaded %d CVEs, %d papers, %d repositories, %d incident/intelligence records, and %d signalsrc  c                 s  s   | ]}|d kr|d  V  qdS )rc  Nr#  )r£  Úcountr#  r#  r$  r¥  á  s   þz load_evidence.<locals>.<genexpr>z%Removed %d duplicate evidence recordsc                 S  s   | j | j ¡ fS r’  ©r'  r&  r–  )Zevidence_itemr#  r#  r$  rï  ò  s    þzload_evidence.<locals>.<lambda>rð  )rØ  Zrepository_pathrá  rü  r—  r  r  rF  rG  ÚLOGGERÚinforª  r   r'  r&  r–  rÞ  r  Úsumr   Úwarningr   Úsort)rI  rJ  rK  rL  rM  Zcve_dataZ
paper_dataZrepository_dataZincident_dataZsignal_dataZcve_recordsZpaper_recordsZrepository_recordsZincident_recordsZsignal_recordsr   ró  rœ  ZdedupedÚ
duplicatesrž  ZcurrentZduplicate_countrš  r#  r#  r$  Úload_evidence$  sì    ÿÿÿÿÿþ	þþ	þ	þ	ÿÿÿÿÿÿ	ÿÿÿÿöýþþÿÿÿþþþÿÿrV  zSequence[Threat]zSequence[Evidence]rS  )r   r   rU  c                   sZ   dd„ | D ƒdd„ |D ƒ }t t|ƒdƒ‰ tƒ }|D ]}| |¡ q4‡ fdd„| ¡ D ƒS )Nc                 S  s   g | ]
}|j ‘qS r#  )r  rÛ  r#  r#  r$  rÜ  ü  r¦  zbuild_idf.<locals>.<listcomp>rc  c                   s,   i | ]$\}}|t  d ˆ  d |  ¡d “qS )rc  rb  )ÚmathÚlog)r£  r®  Z	frequency©Útotalr#  r$  r¾    s   ÿzbuild_idf.<locals>.<dictcomp>)Úmaxrª  r   Úupdater™  )r   r   Z	documentsZdocument_frequencyr  r#  rY  r$  Ú	build_idfû  s    
þr]  zMapping[str, float]rB  )ÚleftÚrightÚidfrU  c                   sX   | r|sdS | |B }| |@ }t ‡ fdd„|D ƒƒ}|dkr>dS t ‡ fdd„|D ƒƒ| S )NrA  c                 3  s   | ]}ˆ   |d ¡V  qdS ©rb  N©rÞ  r­  ©r`  r#  r$  r¥    r¦  z#weighted_jaccard.<locals>.<genexpr>r   c                 3  s   | ]}ˆ   |d ¡V  qdS ra  rb  r­  rc  r#  r$  r¥    r¦  ©rR  )r^  r_  r`  ÚunionÚintersectionÚdenominatorr#  rc  r$  Úweighted_jaccard  s    rh  )Úevidence_tokensÚthreat_tokensr`  rU  c                   sP   | r|sdS | |@ }t ‡ fdd„| D ƒƒ}|dkr6dS t ‡ fdd„|D ƒƒ| S )NrA  c                 3  s   | ]}ˆ   |d ¡V  qdS ra  rb  ©r£  r¶  rc  r#  r$  r¥    r¦  z!coverage_score.<locals>.<genexpr>r   c                 3  s   | ]}ˆ   |d ¡V  qdS ra  rb  rk  rc  r#  r$  r¥    r¦  rd  )ri  rj  r`  Úcommonrg  r#  rc  r$  Úcoverage_score  s    rm  )r^  r_  rU  c                 C  s$   | r|sdS t | |@ ƒt | |B ƒ S )NrA  ©rª  )r^  r_  r#  r#  r$  Úset_overlap  s    ro  )r   rô  rU  c                 C  sB   i }t D ]4}t| j |tƒ ¡|j |tƒ ¡@ ƒ}|r|||< q|S r’  )r¾   rò  r  rÞ  r!  )r   rô  rš  r½  Úmatchesr#  r#  r$  Úmatched_feature_map"  s    $
rq  ztuple[float, list[str]]c                 C  s`   d}g }t D ]D}|d  |jd¡}|rt| j|d ƒr|t|ƒ7 }| |d ¡ qt|dƒ|fS )NrA  rÄ   rÃ   rÂ   çffffffÖ?)rõ   rÞ  rþ   r»  r  rB  r—  Úmin)r   rô  ZbonusZfiredZruleZthreat_bonusr#  r#  r$  Úapply_rules+  s    rt  c                 C  s(   t  |j¡}|sdS t| j|ƒr$dS dS )NrA  rÓ   )rû   rÞ  rþ   r»  r  )r   rô  Úrequirementsr#  r#  r$  rF  6  s    rF  )r   rU  c                 C  s   g d¢}t | j|ƒ S )N)r.   r)   Zcarr4   r7   rj   rk   rm   rH   rC   rE   rò   rQ   rJ   Zroadsiderö   r‚   Ztractionr6   )r»  r  )r   Zautomotive_indicatorsr#  r#  r$  Úevidence_is_generic@  s    rv  zMapping[str, list[str]])r   rô  Úfeature_matchesrQ  rU  c                 C  sà  t | jƒ}|dvrdg fS t |jg ¡}t|o8t| j|ƒƒ}dd„ | ¡ D ƒ}t	|ƒ}t| d¡ƒ}	t| d¡ƒ}
t| d¡ƒ}t| d¡ƒ}t| d	¡ƒ}t|ƒ}d}g }|d
k�rv|rÖ|dkrÖt
|dƒ}| d¡ |rö|dkröt
|dƒ}| d¡ |�r|	�r|�rt
|dƒ}| d¡ |�r@|dk�r@t
|dƒ}| d¡ |�rØ|	�rØ|
�s^|�s^|�rØt
|dƒ}| d¡ �nb|dk�r"|�r¤|dk�r¤t
|dƒ}| d¡ |�rÈ|dk�rÈt
|dƒ}| d¡ |�rî|	�rî|�rît
|dƒ}| d¡ |�rØ|dk�rØ|
�s
|�rØt
|dƒ}| d¡ �n¶|dk�rÂ|�rP|dk�rPt
|dƒ}| d¡ |�rt|dk�rtt
|dƒ}| d ¡ |�r˜|dk�r˜t
|d!ƒ}| d"¡ |�rØ|	�rØ|�rØt
|dƒ}| d#¡ �n|d$k�rN|�rð|dk�rðt
|dƒ}| d%¡ |dk�r|
�s|�rt
|d&ƒ}| d'¡ |�rØ|	�rØ|
�s8|�s8|�rØt
|dƒ}| d(¡ nŠ|d)k�rØ|�r||dk�r|t
|dƒ}| d*¡ |�r |d+k�r t
|d!ƒ}| d,¡ |�rØ|	�rØ|�rØ|�sÄ|
�sÄ|�rØt
|d-ƒ}| d.¡ ||fS )/a  
    Determine a conservative minimum relevance score based on the
    evidence type.

    A type-specific floor is only activated when the evidence also
    contains threat-specific context or structured feature overlap.

    The floor is a minimum score, not an additive bonus.
    >   r  r  r  r
  r  rA  c                 S  s   h | ]\}}|r|’qS r#  r#  )r£  r½  r   r#  r#  r$  r¯  r  s   þz2type_specific_correlation_floor.<locals>.<setcomp>r¹   rº   r»   r¼   r½   r  rc  çš™™™™™Ù?Zincident_context_and_featureé   ç¸…ëQ¸Þ?Zincident_multi_axisçHáz®Gá?Zincident_asset_and_impactZincident_rule_and_featureçìQ¸…ëá?Z incident_strong_structured_matchr  Zassessment_context_and_featureZassessment_multi_axisç¤p=
×£à?Zassessment_asset_and_impactZassessment_technical_contextr  gö(\�ÂõØ?Zsignal_context_and_featureg)\�Âõ(Ü?Zsignal_multi_axisçq=
×£pÝ?Zsignal_rule_and_featureZsignal_asset_and_impactr
  Zrepository_rule_and_featureçáz®GáÚ?Zrepository_multi_axis_technicalZ"repository_strong_structured_matchr  Z narrative_context_and_multi_axisr©  Z"narrative_broad_structured_supportç      à?Znarrative_pathway_support)r‡  r'  rû   rÞ  rþ   r.  r»  r  r™  rª  r[  r—  )r   rô  rw  rQ  r'  ru  Zcontext_matchrO  Zmatched_axis_countZasset_matchZprotocol_matchZattack_vector_matchZimpact_matchZlifecycle_matchZdeterministic_rule_matchZfloorZreasonsr#  r#  r$  Útype_specific_correlation_floorH  sB   ÿþþþþÿÿÿÿÿÿÿ
ÿþþÿÿþþÿÿþýþÿÿþþÿÿþüûú	þÿ
ÿþþÿÿþþÿÿþýþÿÿþüûþÿ
ÿþþÿÿþþÿÿþþÿÿþýþÿ
ÿþþÿÿýüþÿÿþüûú	þÿ
ÿþþÿÿþþÿÿþýûúù
þÿr�  zAtuple[ScoreBreakdown, dict[str, list[str]], list[str], list[str]])r   rô  r`  rU  c                   s¾  t ˆ ˆƒ}tˆ jˆjˆƒ}tˆ jˆjˆƒ}tdd| d|  ƒ}tˆjd ˆj ƒ}tˆ j|ˆƒ}‡ ‡fdd„tD ƒ}	dd„ |	 	¡ D ƒ}
d	|	d
  d|	d   d|	d   d|	d   d|	d   dt
ttttˆjƒƒtƒ ƒdƒ  }td ˆjˆjgˆj¢¡ƒ}t|ƒ}tˆ j|ˆƒ}tˆ ˆƒ\}‰tˆ ˆƒ}tˆ ƒ�rDˆ�sDdnd}d| d|  d|  d|	d
   d|	d   d|	d   d|	d   d|	d   d|  }tdt
dt|
ƒd ƒd ƒ}|| | | | }d}ˆ�râd}ˆ jdk�rþˆ�r
| d
¡�r
t
|d ƒ}d!}d"ˆv �rLd#| d
g ¡v �rLd$| dg ¡v �sH| d¡�rLd%}d&ˆv �r„d'| d
g ¡v �r„| d¡�s€| d¡�r„d%}d(ˆv �r¼d)| d
g ¡v �r¼| d¡�s¸| d¡�r¼d%}d*ˆv �rîd+| dg ¡v �sêd,| dg ¡v �rîd%}|�rþt
|d-ƒ}tˆ ˆ|ˆd.�\}}|dk�r>t
||ƒ}ˆ ‡fd/d0„|D ƒ¡ t
dtdt
||ƒƒƒ}tˆ jˆj@ ‡fd1d2„d3�d d4… }t||||	d
 |	d |	d |	d |	d ||| |||d5�}|||ˆfS )6Nrb  gÍÌÌÌÌÌä?rr  rz  c              	     s0   i | ](}|t ˆ j |tƒ ¡ˆj |tƒ ¡ƒ“qS r#  )ro  r  rÞ  r!  r¼  )r   rô  r#  r$  r¾  ß	  s   ÿzscore_pair.<locals>.<dictcomp>c                 S  s   g | ]\}}|d kr|‘qS )r   r#  )r£  r½  rú  r#  r#  r$  rÜ  ã	  r¦  zscore_pair.<locals>.<listcomp>râ   r¹   çš™™™™™É?rº   rË   r»   rî   r¼   rÓ   r½   rÍ   rA  rô   rá   rÆ   rÕ   rÔ   ç¸…ëQ¸ž?ç{®Gáz”?r   rc  gš™™™™™™?rù  rz  Fr¿   rg   rj   TrÌ   rC   rð   rh   rÙ   rv   r¬   ç×£p=
×ã?)r   rô  rw  rQ  c                 3  s   | ]}|ˆ vr|V  qd S r’  r#  )r£  r>  )rQ  r#  r$  r¥  œ
  s   þzscore_pair.<locals>.<genexpr>c                   s   ˆ   | d¡ | fS )Nrb  rb  )r®  rc  r#  r$  rï  ²
  r¦  zscore_pair.<locals>.<lambda>rð  é   )rC  r   r  r¹   rº   r»   r¼   r½   rD  rE  rF  rG  rH  )rq  rh  r  rm  rs  r²  r   r  r¾   r™  r[  ro  r!  Úmapr‡  r  r§  r  r  rt  rF  rv  rª  r'  rÞ  r�  Úextendrò  r@  )r   rô  r`  rw  rC  Zevidence_coverageZlexical_combinedZtitle_tokensZtitle_scoreZaxis_scoresZstructured_axesZfeature_scoreZdomain_textZdomain_tokensZdomain_scorerE  Zctx_penaltyrG  ÚbaseZmulti_axis_bonusZcalculated_scoreZ
rule_floorZstrong_cve_matchZ
type_floorZtype_floor_reasonsrH  Úcommon_tokensÚ	breakdownr#  )r   r`  rQ  rô  r$  Ú
score_pairÒ	  s|   
þ

ÿ
þ
ý
üþûÿ
ÿþ
ý
ü
û
ú
ùøÿþÿþýüÿÿþþÿþþþùÿõÿþþÿù
ÿöÿþþÿù
ÿöÿþýþùþ	üÿ	
þþ
þþþ


þýórŒ  ztuple[float, str]c                 C  sv   | j }t|g d¢ƒrdS t|g d¢ƒr*dS t|g d¢ƒr<dS t|g d¢ƒrNdS t|g d	¢ƒr`d
S t|g d¢ƒrrdS dS )N)zreal-world malicious activityzreal world attackzproduction charging stationz"deployed in an exposed environment)g…ëQ¸…ë?zReal-world observation)zexperimental analysisZ
experimentzproof of conceptzproof-of-conceptZ	prototypezvalidated by detecting)g=
×£p=ê?zExperimental demonstration)z
case studyzmeasurement studyzempirical studyzdataset analysis)çffffffæ?zEmpirical study)Z
simulationZ	simulatedzdigital twin)r}  Z
Simulation)Zsurveyzsystematic reviewzliterature review)rå   zSurvey/review)Z	frameworkZmethodologyZmodelZ
conceptual)r  zConceptual or methodological)r|  zTechnical analysis)r  r»  )r   r„  r#  r#  r$  Úclassify_paper_strengthÇ
  s    rŽ  c                 C  sä  | j dkrt| ƒS | j dkrTd}d}| jr2|d7 }| jr@|d7 }tdtd|ƒƒ|fS | j dkr–d	}d
}| jrt|d7 }| jr‚|d7 }tdtd|ƒƒ|fS | j dkrØd}d}| jr¶|d7 }| jrÄ|d7 }tdtd|ƒƒ|fS | j dk�rd}d}| jrú|d7 }| j�r
|d7 }tdtd|ƒƒ|fS | j dk�r>| j}| d¡}t|t	ƒ�sJi }| d¡}t|t	ƒ�sdi }t
| d¡ƒ}d}d}|dk�r’|d7 }d}n|dk�r¨|d 7 }d!}| d"¡�r¼|d7 }| j�rÌ|d7 }| j�rÜ|d#7 }t| d$¡ƒ�pîd%}|d&k�r|d 7 }n|d'k�r|d(7 }| d)¡�r*|d8 }td*td|ƒƒ|fS d+}d,}| j�rZ|d-8 }d.}| jd/u�r¦| jd0k�r||d17 }n*| jd2k�r’|d37 }n| jd4k�r¦|d7 }| j�r¶|d37 }t| jg d5¢ƒ�rÐ|d#7 }td6td7|ƒƒ|fS )8z÷
    Determine the intrinsic strength of an evidence or intelligence record.

    This is separate from threat relevance. A record may be authoritative
    or strong evidence while still having little relevance to a specific
    T-xxx threat.
    r   r  gÃõ(\�Âå?z"Authoritative strategic assessmentrÇ   g)\�Âõ(ì?rx  r  r�  zReal-world incident reportgÍÌÌÌÌÌì?r  r{  zEmerging intelligence signalgö(\�Âõè?rá   r  rY  z"Synthesized intelligence narrativerƒ  gš™™™™™é?r
  r  Úreadme_metadatart  r~  zTechnical repositoryZdirectrÍ   z&Directly relevant technical repositoryZstrongrÎ   z&Strongly relevant technical repositoryZ	availablerÕ   Ústarsr   éd   é   r„  ZarchivedgR¸…ëQè?r…  zConfirmed vulnerability recordrÓ   zPending vulnerability analysisNg      "@rÆ   g      @rÔ   g      @)zallows attackersr|   r{   zarbitrary codegq=
×£pí?r‚  )r'  rŽ  r  r5  rs  r[  r?  rÞ  r“  r"  r‡  r7  r9  rÇ  r/  r,  r»  r  )r   rú  rÀ  Úrawr  r�  rt  r�  r#  r#  r$  Úclassify_evidence_strength×
  s,   
ÿ
þþø
þþø
þþøþþø
ÿþÿ
ÿ
ÿÿÿþ


þþø

þ	þþør”  )rú  rw  rQ  rŠ  rU  c                 C  sD   t |ƒ}| dkr |s|dkr dS | dkr@|dks<t |ƒdkr@dS dS )	Ng
×£p=
ç?ry  Zhighr}  rc  r©  ZmediumZlowrn  )rú  rw  rQ  rŠ  Zaxesr#  r#  r$  Úconfidence_forÇ  s    r•  )rú  rk  ri  rU  c                 C  s   | |krdS | |krdS dS )NÚacceptedÚreviewÚrejectedr#  )rú  rk  ri  r#  r#  r$  Ú
status_forÕ  s
    r™  )r   rô  rw  rŠ  rQ  rN  rU  c                 C  sÊ   g }|rLg }|  ¡ D ]"\}}	| |› dd |	¡› �¡ q| dd |¡ ¡ |rd| dd |¡ ¡ |r„| dd |d d… ¡ ¡ |s’| d¡ d	d
ddœ| }
|
› d| j› d|j› d�d |¡ d S )NrÍ  rà  zstructured overlap in z; zspecific rules: zshared terms: é   zlimited lexical similarityzStrong automatic matchz Plausible match requiring reviewzInsufficient match)r–  r—  r˜  z	 between z and Ú.)r™  r—  r§  r&  rþ   )r   rô  rw  rŠ  rQ  rN  Z
componentsÚreadabler½  r   r&  r#  r#  r$  Úexplain_matchÝ  s&    
ýür�  )	r   rô  r‹  rw  rŠ  rQ  rk  ri  rU  c                 C  s–   t |j||ƒ}t| ƒ\}	}
t|j|||ƒ}t|ƒ}t| j| j|jt	|jdƒt	|	dƒ|
||| j
| j| j||||t| |||||ƒdd„ t|ƒ ¡ D ƒd�S )Né   c                 S  s   i | ]\}}|t |d ƒ“qS )rž  )Úround©r£  rž  rx  r#  r#  r$  r¾    s   ÿz&build_match_result.<locals>.<dictcomp>)r&  r'  rþ   rJ  rK  rL  rM  rN  r   r)  r(  rO  rP  r6  rQ  rR  rT  )r™  rH  r”  r•  rò  rI  r&  r'  rþ   rŸ  r   r)  r(  r�  r   r™  )r   rô  r‹  rw  rŠ  rQ  rk  ri  rN  Zstrength_scoreZstrength_labelrM  rO  r#  r#  r$  Úbuild_match_resultù  s6    

ÿ
þír¡  c                 C  sT   t | j|j|ƒ}t| |ƒ}tdd„ | ¡ D ƒƒ}t| |ƒ\}}|td|d ƒ | S )Nc                 s  s   | ]}t |ƒV  qd S r’  rn  )r£  r   r#  r#  r$  r¥  (  r¦  z!candidate_rank.<locals>.<genexpr>r€  rÍ   )rm  r  rq  rR  r   rt  rs  )r   rô  r`  rC  rw  Zfeature_countrE  Ú_r#  r#  r$  Úcandidate_rank!  s
    
r£  )rô  r'  rU  c                 C  sp   dd„ | j D ƒ}|sdS t|ƒ}|dkr8d|v p6d|v S |dv rHd|v S |dkrhd|v pfd|v pfd	|v S ||v S )
zDReturn whether this evidence type is enabled for the threat profile.c                 S  s   h | ]}|rt |ƒ’qS r#  rê  rë  r#  r#  r$  r¯  3  s   þz(evidence_type_allowed.<locals>.<setcomp>Fr  r  >   r  r  r
  r   rù  )r  r‡  )rô  r'  ZallowedZnormalized_typer#  r#  r$  Úevidence_type_allowed-  s*    þÿþÿýr¤  z+tuple[list[MatchResult], list[MatchResult]])r   Úevidence_itemsr`  rr  rU  c                   s´  g }g }t |ƒ}t|dd�D �]l\}‰ ‡ fdd„| D ƒ}t|‡ ‡fdd„dd�}	|	d |j… }
g }|
D ]r}tˆ |ˆƒ\}}}}tˆ ||||||j|jƒ}| |¡ |j	rft
 d	ˆ j|j|j|j|j|j|j|j|j ¡	 qf|jd
d„ dd� d}|D ]p}|jdk�r<||jk �r$| |¡ |d7 }nd|_d|_| |¡ qô|jdk�rT| |¡ qô|jrô| |¡ qô|d dk�s|||krt
 d||¡ q|jdd„ d� |jdd„ d� ||fS )Nrc  )Ústartc                   s   g | ]}t |ˆ jƒr|‘qS r#  )r¤  r'  ©r£  rô  )r   r#  r$  rÜ  `  s   ÿzcorrelate.<locals>.<listcomp>c                   s   t ˆ | ˆƒS r’  )r£  ©rô  ©r   r`  r#  r$  rï  f  r¦  zcorrelate.<locals>.<lambda>T©rž  ÚreversezQ%s -> %s status=%s final=%.4f lexical=%.4f features=%.4f rule=%.4f penalties=%.4fc                 S  s   | j S r’  ©rJ  ©rš  r#  r#  r$  rï  †  r¦  r   r–  r—  zXAdditional plausible match moved to review because the accepted-match limit was reached.r‘  z Processed %d/%d evidence recordsc                 S  s   | j | j | j ¡ fS r’  )rþ   rJ  r&  r–  r­  r#  r#  r$  rï  š  r¦  rð  c                 S  s   | j  ¡ | j | jfS r’  )r&  r–  rJ  rþ   r­  r#  r#  r$  rï  �  r¦  )rª  Ú	enumeraterò  rl  rŒ  r¡  rk  ri  r—  ZexplainrP  Údebugr&  rþ   rN  rH  rC  r  rE  rF  rG  rT  rm  rR  Úinclude_rejectedrQ  )r   r¥  r`  rr  Zaccepted_or_reviewr˜  rZ  ÚindexZeligible_threatsZrankedZ
candidatesZitem_resultsrô  r‹  r  r
  Zrulesrš  Úaccepted_countr#  r©  r$  Ú	correlateU  s|    
ÿýø


÷

ÿÿr³  c                   s\   t  ¡ }|  d¡�2‰ t‡ fdd„dƒD ]}| |¡ q&W d   ƒ n1 sJ0    Y  | ¡ S )NÚrbc                     s
   ˆ   d¡S )Ni   )Úreadr#  ©rÖ  r#  r$  rï  ¥  r¦  zsha256_file.<locals>.<lambda>r¦  )r  r   rÐ  Úiterr\  r"  )rÈ  r$  Úchunkr#  r¶  r$  Úsha256_file¢  s
    *r¹  c                   C  s   t  tj¡ ¡  ¡ S r’  )r   Znowr	   ZutcZ
astimezoneZ	isoformatr#  r#  r#  r$  Úiso_nowª  s    rº  )rš  rU  c                 C  s   t | ƒS r’  )r   r­  r#  r#  r$  Újson_ready_match®  s    r»  )rô  rU  c                 C  s6   | j | j| j| j| j| j| j| j| j| j	| j
| jdœS )N)rÝ  r   r  r  r  r  r	  r
  r  r  r  r  )rþ   r   r  r  r  r  r	  r
  r  r  r  r  r¨  r#  r#  r$  Úthreat_summary²  s    ôr¼  c                 C  s†   | j | j| j| j| j| j| j| j| j| j	| j
dœ}| jdkrZ| | j| j| j| jdœ¡ n(| | j| j| j| j| j| j| jdœ¡ |S )N)rÝ  rV  r   r  r(  r)  r6  r7  r8  r  r9  rù  )r*  r,  r-  r/  )r0  r1  r3  r4  r5  r;  r<  )r&  r'  r   r  r(  r)  r6  r7  r8  r  r9  r\  r*  r,  r-  r/  r0  r1  r3  r4  r5  r;  r<  )r   r   r#  r#  r$  Úevidence_summaryÃ  s:    õ
ü
ù	r½  zSequence[MatchResult])r   ÚresultsrU  c           	      C  s  t tƒ}|D ]}||j  |¡ qi }| D ]Ò}| |jg ¡}dd„ |D ƒ}dd„ |D ƒ}i t|ƒ¥t|ƒt|ƒtdd„ |D ƒƒtdd„ |D ƒƒtdd„ |D ƒƒtdd„ |D ƒƒtd	d„ |D ƒƒtd
d„ |D ƒƒtdd„ |D ƒƒdœ	dd„ |D ƒdœ¥||j< q*tt	ƒ |dœS )Nc                 S  s   g | ]}|j d kr|‘qS ©r–  ©rN  r  r#  r#  r$  rÜ  ð  r¦  z-build_threat_correlations.<locals>.<listcomp>c                 S  s   g | ]}|j d kr|‘qS ©r—  rÀ  r  r#  r#  r$  rÜ  ñ  r¦  c                 s  s   | ]}|j d krdV  qdS ©rù  rc  N©r'  r  r#  r#  r$  r¥  ÷  r¦  z,build_threat_correlations.<locals>.<genexpr>c                 s  s   | ]}|j d krdV  qdS ©r   rc  NrÃ  r  r#  r#  r$  r¥  ø  r¦  c                 s  s   | ]}|j d krdV  qdS )r
  rc  NrÃ  r  r#  r#  r$  r¥  ù  s   
þc                 s  s   | ]}|j d krdV  qdS )r  rc  NrÃ  r  r#  r#  r$  r¥  þ  r¦  c                 s  s   | ]}|j d krdV  qdS )r  rc  NrÃ  r  r#  r#  r$  r¥  ÿ  r¦  c                 s  s   | ]}|j d krdV  qdS )r  rc  NrÃ  r  r#  r#  r$  r¥     r¦  c                 s  s   | ]}|j d krdV  qdS )r  rc  NrÃ  r  r#  r#  r$  r¥    r¦  )	r–  r—  r   r   rX  r   r   ZassessmentsZ
narrativesc                 S  s   g | ]}t |ƒ‘qS r#  ©r»  r  r#  r#  r$  rÜ    r¦  )Ú
statisticsr   )Úschema_versionÚgenerated_atr   )
r   r   rþ   r—  rÞ  r¼  rª  rR  ÚSCHEMA_VERSIONrº  )	r   r¾  Úgroupedrš  Z
threat_maprô  rp  r–  r—  r#  r#  r$  Úbuild_threat_correlationså  s:    ÿþóïýrË  )r¥  r¾  rU  c                 C  s’   t tƒ}|D ]}||j|j ¡ f  |¡ qg }| D ]N}t| |j|j ¡ fg ¡dd„ dd�}| i t|ƒ¥ddd„ |D ƒi¥¡ q4t	t
ƒ |dœS )	Nc                 S  s   | j S r’  r¬  )rp  r#  r#  r$  rï    r¦  z&build_evidence_index.<locals>.<lambda>Trª  rp  c              	   S  s*   g | ]"}|j |j|j|j|j|jd œ‘qS )©rþ   rJ  rK  rM  rN  rR  rÌ  r  r#  r#  r$  rÜ    s   	ùúz(build_evidence_index.<locals>.<listcomp>)rÇ  rÈ  ré  )r   r   r'  r&  r–  r—  rò  rÞ  r½  rÉ  rº  )r¥  r¾  rÊ  rš  ré  r   rp  r#  r#  r$  Úbuild_evidence_index  s*    ýÿ	÷þ
ýrÍ  )r¾  r˜  rU  c                 C  sV   dd„ | D ƒ}dd„ | D ƒ}t tƒ t|ƒt|ƒt|ƒdœdd„ |D ƒdd„ |D ƒdœS )Nc                 S  s   g | ]}|j d kr|‘qS rÁ  rÀ  ©r£  rš  r#  r#  r$  rÜ  6  r¦  z'build_review_output.<locals>.<listcomp>c                 S  s   g | ]}|j d kr|‘qS r¿  rÀ  rÎ  r#  r#  r$  rÜ  7  r¦  )r–  Úreview_requiredZrejected_includedc                 S  s   g | ]}t |ƒ‘qS r#  rÅ  rÎ  r#  r#  r$  rÜ  @  r¦  c                 S  s   g | ]}t |ƒ‘qS r#  rÅ  rÎ  r#  r#  r$  rÜ  A  r¦  )rÇ  rÈ  rÆ  rÏ  Zrejected_candidates)rÉ  rº  rª  )r¾  r˜  r—  r–  r#  r#  r$  Úbuild_review_output2  s    ý÷rÐ  )rÈ  r   rU  c              	   C  sà   | j jddd� tjd| j› d�dt| j ƒd�\}}t|ƒ}zttj|ddd��D}t	j
||d	d
d
d� | d¡ | ¡  t | ¡ ¡ W d   ƒ n1 s˜0    Y  t || ¡ W n* tyÚ   z|jdd� W ‚ n‚ 0 Y n0 d S )NT©ÚparentsÚexist_okr›  z.tmp)r&  ÚsuffixÚdirÚwrÊ  rË  ry  F)ÚindentZensure_asciiZ	allow_nanr¡  )Z
missing_ok)ÚparentÚmkdirÚtempfileZmkstemprÂ   rý   r
   ÚosÚfdopenrÑ  ÚdumpÚwriteÚflushÚfsyncÚfilenor  Ú	ExceptionÚunlink)rÈ  r   ÚfdZtemporary_nameZtemporary_pathrÖ  r#  r#  r$  Úatomic_write_jsonE  s     ÿ

,rå  )rr  rU  c                 C  sˆ   | j | j| j| j| j| j| jg}dd„ |D ƒ}| jd  ¡ | jd  ¡ | jd  ¡ | jd  ¡ h}||@ }|r„t	dt
t|ƒƒ› �ƒ‚d S )Nc                 S  s   h | ]}|  ¡ r| ¡ ’qS r#  )rÎ  Úresolve©r£  rÈ  r#  r#  r$  r¯  [  r¦  z!validate_paths.<locals>.<setcomp>úthreat_correlations.jsonúevidence_index.jsonúcorrelation_review.jsonúcorrelation_run.jsonz"Output path conflicts with input: )r   r   r   r   rX  r   r   r   ræ  rÄ  Únextr·  )rr  Zinput_pathsZresolved_inputsZoutput_filesZconflictr#  r#  r$  Úvalidate_pathsY  s     ürí  c                 C  s®   g }dd„ | D ƒ}|r^t ttt|ƒt|ƒd ƒƒt|ƒ ƒ}|r^| dd dd„ |D ƒ¡ ¡ tdd„ |D ƒƒ}td	d„ |D ƒƒ}|r–| d
|› �¡ |rª| d|› �¡ |S )Nc                 S  s    g | ]}t |j d ¡d ƒ‘qS )rí  rc  rî  r§  r#  r#  r$  rÜ  i  r¦  z(validate_loaded_data.<locals>.<listcomp>rc  zMissing threat IDs: rà  c                 s  s   | ]}d |d›�V  qdS )zT-Z03dNr#  rë  r#  r#  r$  r¥  m  r¦  z'validate_loaded_data.<locals>.<genexpr>c                 s  s"   | ]}|j d kr|jsdV  qdS rÄ  ©r'  r  rÛ  r#  r#  r$  r¥  n  s   c                 s  s"   | ]}|j d kr|jsdV  qdS rÂ  rî  rÛ  r#  r#  r$  r¥  q  s   zPapers without abstract: z(CVEs without vulnerability description: )rò  r!  Úrangers  r[  r—  r§  rR  )r   r   ÚwarningsZidsZmissingZpapers_without_abstractZcves_without_descriptionr#  r#  r$  Úvalidate_loaded_datag  s"    &ÿÿrñ  r:  c                    sÒ  t ƒ ‰ tˆ jƒ t ¡ } �z:tˆ ƒ t dˆ jˆ j¡ t	ˆ jˆ jƒ}t dt
|ƒ¡ t dˆ j¡ t dˆ j¡ t dˆ j¡ tˆ jˆ jˆ jˆ jˆ jƒ}tdd„ |D ƒƒ}tdd„ |D ƒƒ}td	d„ |D ƒƒ}td
d„ |D ƒƒ}tdd„ |D ƒƒ}tdd„ |D ƒƒ}tdd„ |D ƒƒ}	t d|||||||	¡ ˆ j�r^‡ fdd„|D ƒ}|�s^tdˆ j› �ƒ‚ˆ j�r˜ˆ j ¡ ‰‡fdd„|D ƒ}|�s˜tdˆ j› �ƒ‚t||ƒ}
|
D ]}t |¡ �q¦ˆ j�rÐt d¡ W dS ˆ jjddd� ˆ jd ˆ jd ˆ jd ˆ jd dœ}ˆ j�s8dd„ | ¡ D ƒ}|�r8tdd |¡ ƒ‚t d ¡ t||ƒ}t |||ˆ ƒ\}}td!d„ |D ƒƒ}td"d„ |D ƒƒ}d}d#d$„ |D ƒ}|D ]$}|j!|j ¡ f|v�r˜|d%7 }�q˜t"||ƒ}t#||ƒ}t$||ƒ}t%t&ƒ t't ¡ |  d&ƒt(ˆ jƒt)ˆ jƒd'œt(ˆ jƒt)ˆ jƒd'œt(ˆ jƒt)ˆ jƒd'œt(ˆ jƒt)ˆ jƒd'œt(ˆ jƒt)ˆ jƒd'œt(ˆ jƒt)ˆ jƒd'œt(ˆ jƒt)ˆ jƒd'œd(œˆ j*ˆ j+ˆ j,ˆ j-ˆ j.ˆ jˆ jd)d*œt
|ƒ|||||||	t
|ƒ||t
|ƒ|d+œ|
d,d-„ | /¡ D ƒd.œ}t0|d/ |ƒ t0|d0 |ƒ t0|d1 |ƒ t0|d2 |ƒ t d3|¡ t d4|¡ t d5|¡ t d6ˆ j¡ W dS  t1ttt2f�y� } zt 3d7|¡ W Y d }~d8S d }~0  t4�y®   t 3d9¡ Y d:S  t5�yÌ   t 6d;¡ Y d%S 0 d S )<Nz&Loading threat profiles from %s and %szLoaded %d unique threatszLoading CVEs from %szLoading papers from %szLoading repositories from %sc                 s  s   | ]}|j d kV  qdS )rù  NrÃ  rÛ  r#  r#  r$  r¥  �  r¦  zmain.<locals>.<genexpr>c                 s  s   | ]}|j d kV  qdS )r   NrÃ  rÛ  r#  r#  r$  r¥  Ž  r¦  c                 s  s   | ]}|j d kV  qdS )r
  NrÃ  rÛ  r#  r#  r$  r¥  �  s   ÿc                 s  s   | ]}|j d kV  qdS )r  NrÃ  rÛ  r#  r#  r$  r¥  “  r¦  c                 s  s   | ]}|j d kV  qdS )r  NrÃ  rÛ  r#  r#  r$  r¥  •  r¦  c                 s  s   | ]}|j d kV  qdS )r  NrÃ  rÛ  r#  r#  r$  r¥  –  r¦  c                 s  s   | ]}|j d kV  qdS )r  NrÃ  rÛ  r#  r#  r$  r¥  —  r¦  zfLoaded %d CVEs, %d papers, %d repositories, %d signals, %d incidents, %d assessments and %d narrativesc                   s   g | ]}|j ˆ j kr|‘qS r#  rì  r§  )rr  r#  r$  rÜ  ¤  r¦  zmain.<locals>.<listcomp>zThreat not found: c                   s   g | ]}|j  ¡ ˆ kr|‘qS r#  )r&  r–  rÛ  )Úneedler#  r$  rÜ  ©  s   zEvidence not found: z4Validation completed successfully; no output writtenr   TrÑ  rè  ré  rê  rë  )Úcorrelationsr±  r—  Úrunc                 S  s   g | ]}|  ¡ rt|ƒ‘qS r#  )rÎ  rý   rç  r#  r#  r$  rÜ  ¿  r¦  z6Output files already exist. Use --force to overwrite: rà  z!Building document-frequency indexc                 s  s   | ]}|j d kV  qdS )r–  NrÀ  rÎ  r#  r#  r$  r¥  É  r¦  c                 s  s   | ]}|j d kV  qdS )r—  NrÀ  rÎ  r#  r#  r$  r¥  Ê  r¦  c                 S  s   h | ]}|j |j ¡ f’qS r#  rO  )r£  rÉ  r#  r#  r$  r¯  Ì  r¦  zmain.<locals>.<setcomp>rc  r©  )rÈ  r   )r   r   r   r   rX  r   r   Zdeterministic_explainable_v1)rk  ri  rl  rm  r°  Zthreat_filterZevidence_filterZmatching_mode)Zthreats_loadedZcves_loadedZpapers_loadedZrepositories_loadedZsignals_loadedZincidents_loadedZassessments_loadedZnarratives_loadedZevidence_loadedZaccepted_matchesZreview_matchesZrejected_candidates_includedÚunmatched_evidencec                 S  s   i | ]\}}|t |ƒ“qS r#  )rý   r   r#  r#  r$  r¾     r¦  zmain.<locals>.<dictcomp>)rÇ  rÈ  Zduration_secondsZinputsZconfigurationrÆ  rð  Zoutputsró  r±  r—  rô  zAccepted matches: %dzReview matches: %dzUnmatched evidence records: %dzOutput written to %sz%sry  ZInterruptedé‚   zUnexpected failure)7rh  rw  Z	log_levelÚtimerí  rP  rQ  r   r   rõ  rª  r   r   rX  rV  r   r   rR  rþ   rÄ  r&  r–  rñ  rS  Zvalidate_onlyr   rÙ  Zforcer   ÚFileExistsErrorr§  r]  r³  r'  rË  rÍ  rÐ  rÉ  rº  rŸ  rý   r¹  rk  ri  rl  rm  r°  r™  rå  rÏ  ÚOSErrorrj  ÚKeyboardInterruptrâ  Z	exception)r¦  r   r¥  Z	cve_countZpaper_countZrepository_countZsignal_countZincident_countZassessment_countZnarrative_countrð  rS  Zoutput_pathsrâ  r`  r¾  r˜  r²  Zreview_countrõ  Zmatched_idsrœ  Zcorrelations_outputZindex_outputZreview_outputZ
run_outputr×  r#  )rr  rò  r$  Úmain{  s   
þþø

ÿ

üÿ




ÿþôøóÔ/

rû  Ú__main__)r�  )rÿ   )rÿ   )rÚ__doc__Z
__future__r   rd  r  r|  rÑ  r®   rW  rÛ  r€  ÚsysrÚ  r÷  r~  Úcollectionsr   r   Zdataclassesr   r   r   r   r	   Úpathlibr
   Útypingr   r   r   r   rÉ  Z	getLoggerrP  ÚcompileÚ
IGNORECASEro  rû  r‰  r�  r°  rg  r«  r¾   r  rõ   rû   rü   r%  r@  rI  rh  rw  r…  r‡  r�  rŸ  r¨  r²  r¸  r»  rÁ  rÆ  rÇ  rØ  rá  rå  ræ  rõ  rü  r  r  r  r  r  r'  rC  rF  rG  rH  rV  r]  rh  rm  ro  rq  rt  rF  rv  r�  rŒ  rŽ  r”  r•  r™  r�  r¡  r£  r¤  r³  r¹  rº  r»  r¼  r½  rË  rÍ  rÐ  rå  rí  rñ  rû  r  Úexitr#  r#  r#  r$  Ú<module>   sR  


øÉCôß1Û0ð€ Ð Lý
ýý
ý
ýýýöþñýíýòù

ý
ý
ý
ý
ýûýý
ý€ Ù 2Þ&0


			
e12 Wþ5;þ   J			* X
	
     v q  "((($M"(% 