a
    ßTðj$< ã                   @  sº  U d Z ddlmZ ddlZddlZddlZddlZddlZ	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlm  mZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) dZ*d	Z+e!d
ƒZ,dZ-dZ.de*› d�Z/e 0d¡Z1dddddddœdddddddœdddddddœdd dd!dd"dœgZ2d#d$d%d&dd'dd(œd)d*d%d+dd,d"d(œgZ3g d-¢g d.¢g d/¢g d0¢g d1¢g d2¢g d3¢g d4¢g d5¢g d6¢d7œ
Z4d8e5d9< g d:¢Z6g d;¢Z7g d<¢Z8g d=¢Z9g d>¢Z:g d?¢Z;g d@¢Z<g dA¢Z=e >dBej?¡Z@e >dCej?¡ZAe >dD¡ZBe >dE¡ZCg dF¢g dG¢dHdIgg dJ¢g dK¢g dL¢dMdNgg dO¢g dP¢g dQ¢g dR¢g dS¢g dT¢g dU¢g dV¢g dW¢g dX¢dYœZDg dZ¢g d[¢g d\¢g d]¢g d^¢g d_¢g d`¢g da¢g db¢g dc¢g dd¢g de¢g df¢g dg¢dhœZEg di¢g dj¢g dk¢g dl¢g dm¢g dn¢g do¢g dp¢g dq¢g dr¢g ds¢g dt¢duœZFg dv¢g dw¢g dx¢g dy¢g dz¢g d{¢g d|¢g d}¢g d~¢dœ	ZGg d€¢g d�¢d‚dƒgg d„¢g d…¢g d†¢d‡œZHg dˆ¢g d‰¢g dŠ¢g d‹¢dŒœZId�dŽd�d�d‘œZJejKG d’d“„ d“ƒƒZLG d”d•„ d•eMƒZNd–d—œd˜d™„ZO�d@dšd›dœœd�dž„ZPdŸd›dœœd d¡„ZQd›d›d¢œd£d¤„ZRd›d¥d¦œd§d¨„ZSd©dªœd›d«d›d¬œd­d®„ZTdŸdšdœœd¯d°„ZUdŸd±dœœd²d³„ZV�dAd±d«dšd´dµœd¶d·„ZWd¸d¹dºœd»d¼„ZXd½d¾„ e4 Y¡ D ƒZZeXe6ƒZ[eXe7ƒZ\eXe<ƒZ]eXe=ƒZ^eXe8ƒZ_eXe9ƒZ`eXe:ƒZaeXe;ƒZbd›d¿d«dÀœdÁdÂ„Zcd›d8d¥dÃœdÄdÅ„Zd�dBd›d«d›d«dÇdÈœdÉdÊ„ZedËdÌd›dÍœdÎdÏ„ZfdËd›dÐœdÑdÒ„ZgdÇdÓdÔdÕœdÖd×„ZhG dØdÙ„ dÙe)ƒZie >dÚejjejkB ¡Zle >dÛejjejkB ¡Zmd›dÜdÝœdÞdß„Znd›d±d¢œdàdá„Zod›d«d›dâdãœdädå„ZpdÓd«d›d«dÔdæœdçdè„ZqdÓd«d›dÔdéœdêdë„ZrdìdÔdíœdîdï„Zsd“dðdñœdòdó„Ztd“dôdñœdõdö„Zud›d›d¦œd÷dø„Zvd›d›d›dùœdúdû„Zwd›dÓd¦œdüdý„Zxd›d›dÓdþœdÿ�d „Zyd›dÓdÓ�dœ�d�d„Zzd›d¥d¥dÓdÓ�dœ�d�d„Z{�dCd“d«d›�dœ�d	�d
„Z|d“dÓd›�dœ�d�d„Z}d“�ddÓd›dÓ�dœ�d�d„Z~d›d›�d�dœ�d�d„ZdÓd›�dœ�d�d„Z€dÓdÓd›dÓ�dœ�d�d„Z��dD�d�d�d�dœ�d�d„Z‚dì�ddíœ�d �d!„ZƒdÓdÓdÓ�d"œ�d#�d$„Z„�d�dd›�d�d%œ�d&�d'„Z…�dd›dÓdÓ�d(œ�d)�d*„Z†dÓd¥�d+œ�d,�d-„Z‡�dEdìdÓd´d´�d.�d/œ�d0�d1„Zˆ�d2�d�d3œ�d4�d5„Z‰d«�d.�d6œ�d7�d8„ZŠ�dF�d9�d2�d:œ�d;�d<„Z‹�dG�d9d«�d:œ�d=�d>„ZŒe��d?k�r¶eŽeŒƒ ƒ‚dS (H  a—  Collect, classify, deduplicate and persist public cyber incident records.

The script deliberately separates real-world incidents from vulnerability-only
advisories and strategic publications. It uses deterministic rules only; it does
not invent facts that are absent from source material.

Default output:
    incidents/incidents.json

Examples:
    python3 incidents_intel.py
    python3 incidents_intel.py --days 31 --verbose
    python3 incidents_intel.py --feed https://example.org/feed.xml
    python3 incidents_intel.py --input-json incidents/raw/manual.json
    python3 incidents_intel.py --dry-run --pretty
    python3 incidents_intel.py --validate-only
é    )ÚannotationsN)ÚCounterÚdefaultdict)ÚPath)ÚAnyÚIterableÚIteratorÚOptionalÚSequence)Ú
HTMLParserz1.0.0z1.0zincidents/incidents.jsoné   é-   zNCRC-incidents-intel/z (+public-source-research)Zincidents_intelzNCSC-NL Newsz https://feeds.ncsc.nl/nieuws.rssz*Nationaal Cyber Security Centrum (NCSC-NL)Zgovernment_newsÚhighT©ÚnameÚurlÚ	publisherÚsource_typeÚauthority_levelÚenabledzCERT-EU Threat Intelligencez;https://cert.europa.eu/publications/threat-intelligence-rsszCERT-EUÚgovernment_threat_intelligencezCISA Alertsz8https://www.cisa.gov/cybersecurity-advisories/alerts.xmlz7Cybersecurity and Infrastructure Security Agency (CISA)Zgovernment_alertzCISA ICS Advisoriesz@https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xmlZgovernment_advisoryFz	AIVD Newsz"https://www.aivd.nl/actueel/nieuwsz2Algemene Inlichtingen- en Veiligheidsdienst (AIVD)Zgovernment_intelligence_newsz/actueel/nieuws/)r   r   r   r   r   Úpath_prefixr   zAIVD Documentszhttps://www.aivd.nl/documentenZ#government_intelligence_publicationz/documenten/)z\bransomware\bz\bgijzelsoftware\bz\bransom demand\bz\bdouble extortion\bz'\bencrypted (?:systems|files|servers)\b)z\bdata breach\bz\bdatalek\bz/\bdata (?:was )?(?:stolen|exfiltrated|leaked)\bz6\bgegevens (?:zijn )?(?:gestolen|buitgemaakt|gelekt)\bz)\bpersonal data (?:exposed|compromised)\b)z\bcyber[- ]physical\bz\bphysical process\bz\bphysical damage\bz,\btemperature(?:s)? (?:changed|raised|set)\bz)\btemperatuur .*?(?:gewijzigd|verhoogd)\bú\bdefrost mode\bz\bontdooi(?:modus|cyclus)\bz,\bcontroller settings (?:changed|modified)\bz\boperational technology\bz\bindustrial control system\bz7\b(?:plc|scada|ics|ot) (?:attack|incident|compromise)\bz\bkoelinstallaties?\bz\bkoelsystemen?\bz\bkoelcontrollers?\bz\bvriesinstallaties?\bz\bvriezers?\bz\bkoelkasten?\bz\brefrigeration systems?\bz\brefrigeration controllers?\bz\btemperature manipulation\bz\btemperatuurmanipulatie\b)z\bsupply[- ]chain attack\bz\bsoftware supply chain\bz\bleveranciersaanval\bz\bthird[- ]party compromise\bz'\btrusted update (?:was )?compromised\b)z\bcyber espionage\bú\bcyberspionage\bz\bespionage campaign\bz'\bsteal(?:ing)? sensitive information\bz\binformation gathering\b)z1\bcredential(?:s)? (?:stolen|compromised|theft)\bz*\binloggegevens (?:gestolen|buitgemaakt)\bz\baccount takeover\bz\bcompromised account\b)z\bservice disruption\bz\boperational disruption\bz\boperations disrupted\bz7\bsystems? (?:were )?(?:taken|knocked|forced) offline\bz)\bsystemen .*?(?:offline|uitgeschakeld)\bz"\bdenial[- ]of[- ]service attack\bz\bddos attack\bz/\bdienstverlening .*?(?:verstoord|stilgelegd)\b)z\bcyber sabotage\bz\bsabotage\bz\bdestructive attack\bz\bwiper malware\bz(\bdestroyed (?:data|systems|equipment)\b)z\bmalware attack\bz\bmalware campaign\bz\bmalware infection\bz\bmalwarebesmetting\bz\btrojan(?:ized)?\bz\bbackdoor deployed\b)z\bunauthorized access\bz\bunauthorised access\bz\bongeautoriseerde toegang\bz \bnetwork (?:was )?compromised\bz'\binfrastructure (?:was )?compromised\bz%\bsuspicious activity .*? malicious\bú\bkwaadwillende acties\b)
Ú
ransomwareÚdata_breachÚcyber_physical_attackÚsupply_chain_attackÚ	espionageÚcredential_compromiseÚservice_disruptionÚsabotageÚmalware_attackÚunauthorized_accesszdict[str, list[str]]ÚINCIDENT_SIGNAL_PATTERNS)z\bcyber ?attack(?:s)?\bz\bcyberaanval(?:len)?\bz\bcyber incident\bz\bcybersecurity incident\bz\bsecurity incident\bz\bcompromis(?:e|ed)\bz\bhack(?:ed|ing|ers?)?\bz\bintrusion\bz
\bbreach\bz.\battackers? (?:accessed|targeted|exploited)\bzM\bthreat actors? (?:accessed|targeted|compromised|exploited|deployed|stole)\b)z\bmay allow\bz\bcould allow\bz\bcan allow\bz2\bkwetsbaarheid(?:heden)? (?:gevonden|verholpen)\bz\bvulnerabilit(?:y|ies)\bz\bsecurity update\bz\bpatch(?:ing)? advisory\bz\bproof of concept\bz\bthreat landscape report\bz\bannual report\bú\bcyber brief\bz\bguidance\bz\bbest practices\bz\bwebinar\b)z$\bknown exploited vulnerabilities?\bz=\badds? (?:one|two|three|four|five|six|\d+) known exploited\bz\bkev catalog\bz\bkev-catalogus\bz\bkwetsbaarheid(?:heden)?\bz!\bzero[- ]?day[- ]kwetsbaarheid\bz\bzeroday[- ]kwetsbaarheid\bz\bupdate (?:nu|direct)\bz\bpatch(?:en|ing)?\bz\bsecurity advisory\bz\bcve-\d{4}-\d+\b)(z\bcybersecuritybeeld\bz\bthreat landscape\br&   z\bstrategic assessment\bz\bdreigingsbeeld\bz\bdreigingsanalyse\bz\bcyberadvies\bz$\bverwacht(?:en)? (?:een )?toename\bz\btoename van cyberaanvallen\bz*\bcyberaanvallen .*?(?:toenemen|toename)\bz-\bdreiging .*?(?:neemt toe|groeit|versnelt)\bz2\bdreigingen .*?(?:nemen toe|groeien|versnellen)\bz\bai versnelt\bz\bweerbaarheid\bz\bnu handelen noodzakelijk\bz\bstatelijke actoren?\bz\bstatelijke cyberactoren?\bz\bcyberactoren?\bz\bChinese cyberactoren?\bz\bChinese hackers?\bz\bRussische hackers?\bz\binlichtingendiensten?\bz\bspionagerisico(?:'s|s)?\bz\bspionage\br   z\bdigitale spionage\bz\bspionageoperatie\bz\bdigitale dreiging(?:en)?\bz\bveiligheidsrisico(?:'s|s)?\bz\bcyberrisico(?:'s|s)?\bz\bmoderne voertuigen\bz\bslimme voertuigen\bz\bconnected vehicles?\bz\bconnected cars?\bz\bvoertuigdata\bz\bdata over inzittenden\bz$\bdigitale systemen .*? voertuigen\bz\bedge devices?\bz\binternet verbonden\bz+\bverbonden (?:met|aan) (?:het )?internet\b)z\bblijft csirt\bz\bwordt csirt\bz\bsluit zich aan bij\bz%\bmaakt afspraken over samenwerking\bz,\bsamenwerking bij de cyberbeveiligingswet\b)ú\bcyberaanvallen? op\bz\bcyberaanvallen? tegen\bú\bwerd(?:en)? getroffen\bú\bgetroffen door\bú\bwas attacked\bú\bwere attacked\bú\bwas hacked\bú\bhas been hacked\bz7\bsystems? (?:were )?(?:taken|forced|knocked) offline\bz2\bsystemen? .*?(?:offline gehaald|uitgeschakeld)\bz#\boperations? (?:were )?disrupted\bz/\bbedrijfsvoering .*?(?:verstoord|stilgelegd)\bz5\bdata (?:was |were )?(?:stolen|exfiltrated|leaked)\bz/\bgegevens .*?(?:gestolen|buitgemaakt|gelekt)\bú\bfiled a police report\bz\baangifte gedaan\bú\bincident response\bú\bmalicious actions\br   z/\btemperature(?:s)? .*?(?:raised|changed|set)\bz;\btemperatu(?:ur|ren) .*?(?:verhoogd|gewijzigd|aangepast)\br   z!\bontdooi(?:modus|cyclus|cycli)\b)z\bconfirmed\bz\bbevestigd\br*   r+   r)   r-   r,   z.\bdetected (?:suspicious|malicious) activity\br0   z\bactive exploitation\bz\bactively exploited\bz\bactief misbruikt\br/   r.   z\baangifte\bz
\bcaused\bz\bresulted in\bz\bleidde tot\br'   z,\bcyberaanvallen? (?:vinden|vonden) plaats\bz(\bcyberaanvallen? (?:gericht op|tegen)\bzL\bhackers? (?:hebben|hadden|kunnen) .*?(?:toegang|aangepast|gemanipuleerd)\br(   z\bgetroffen (?:door|bij)\bz"\bincident (?:vond|heeft) plaats\bz"\bmisbruik (?:vond|heeft) plaats\b)z\bsuspected\bz\bvermoedelijk\bz\bpossibly\bz\bmogelijk\bz\breportedly\bz\bnaar verluidt\bz\bunconfirmed\bz\bniet bevestigd\bz\bCVE-\d{4}-\d{4,7}\bzhttps?://[^\s<>\"]+z<[^>]+>z\s+)ZnetherlandsZ	nederlandZdutch)ZbelgiumZbelgieu   belgiÃ«ZbelgianZ
luxembourgZ	luxemburg)ZfranceZ	frankrijkZfrench)ZgermanyZ	duitslandZgerman)ZswedenZzwedenZswedishZfinlandZfinnish)ZirelandZierlandZirish)ZestoniaZestlandZestonian)zunited kingdomZbritainZbritishz uk )zunited stateszu.s.z us Zamerican)ZukraineZ	ukrainianZoekraine)ZpolandZpolenZpolish)ZdenmarkZ
denemarkenZdanish)ZnorwayZ	noorwegenZ	norwegian)ZspainZspanjeZspanish)ZitalyZitalieu   italiÃ«Zitalian)ZNetherlandsZBelgiumZ
LuxembourgZFranceZGermanyZSwedenZFinlandZIrelandZEstoniazUnited KingdomzUnited StatesZUkraineZPolandZDenmarkZNorwayZSpainZItaly)Ú
automotiveÚvehicleÚvehiclesúcar manufacturerúconnected carúconnected vehicleúconnected vehiclesÚvoertuigÚ
voertuigenúmoderne autoúmoderne voertuigenúslimme voertuigenÚvoertuigdata)Z	transportZ	logisticsZrailwayZairportZmaritimeZshipping)ÚretailZsupermarketZshopZstoreZwinkelZ
supermarkt)zfood supplyúfood safetyZ	groceriesZrefrigerationZkoelinstallatie)ÚenergyZelectricityz
power gridZoilzgas pipeline)Ú
healthcareZhospitalZmedicalZpatientZzorgZ
ziekenhuis)Ú
governmentZministryZmunicipalityzpublic authorityZoverheidZgemeente)ZbankÚ	financialZ	insuranceZpaymentZ
financieleu   financiÃ«le)ZtelecomÚtelecommunicationszmobile operatorzinternet provider)ÚmanufacturingÚfactoryzindustrial productionZfabriek)zwater utilityzwater treatmentZ
wastewaterZ
drinkwater)Z
universityZschoolÚ	educationZuniversiteitZ	onderwijs)zsoftware providerzcloud providerztechnology companyzit provider)úoperational technologyúindustrial controlÚscadaú plc ÚicsÚ
controller)r1   Ztransportationr>   Zfoodr@   rA   rB   ZfinancerD   rE   ZwaterrG   Z
technologyÚOT)rH   rI   rJ   rK   rL   )zinternet of thingsZiotzconnected devicezinternet-connected)úremote accessúremotely accessedú
op afstandzremote control)úinternet-exposedúinternet facingúpublicly accessiblez*rechtstreeks toegankelijk via het internet)Z
credentialzaccount takeoverÚidentityÚphishingZinloggegevens)zsupply chainúthird partyÚsupplierÚleverancier)zcyber-physicalúphysical processúphysical damageÚtemperatureÚdefrostrM   )zartificial intelligencez ai zmachine learningzkunstmatige intelligentie)zstate-sponsoredzstate actorzgovernment-linkedzstatelijke actor)	r1   r2   r3   r8   r9   r:   r<   r5   r6   )r   ZspionageZspionagerisicozspionagerisico'sZcyberspionagezdigitale spionage)zdata verzamelenZdataverzamelingzvehicle datar=   zdata over inzittenden)rN   ÚIoTÚremote_accessÚinternet_exposurerU   Úsupply_chainÚcyber_physicalZAIZstate_actorr1   r   Zdata_collection)zoperational disruptionzoperations disruptedZofflineZ
stilgelegdZuitgeschakeld)zdata stolenzdata exfiltrationzgestolen gegevensZbuitgemaakt)z	data leakzdata exposedZdatalekZgelekt)zfinancial losszmillion euroszmiljoenen eurozeconomic damage)rZ   r\   r]   zcontroller settingsZontdooicyclus)úequipment damageÚfireZbrandzdestroyed equipment)zspoiled foodz	food losszproducts destroyedZbedorvenZ
weggegooid)zservice unavailablezwebsite unavailablezservices offlinezniet beschikbaar)zsafety riskr?   zpatient safetyzpublic safetyZ
veiligheid)	Úoperational_disruptionÚ
data_theftÚdata_exposureÚfinancial_lossÚphysical_process_manipulationÚequipment_damageÚproduct_lossÚservice_unavailabilityÚsafety_risk)rR   rS   rT   )rO   rP   rQ   rV   Zspearphishing)ústolen credentialszcompromised credentialsúdefault passwordúweak passwordÚstandaardwachtwoord)zexploited vulnerabilityzexploitation ofzmisbruik van een kwetsbaarheidzzero-day)rW   rX   zservice providerrY   )Úinternet_exposed_servicer_   rV   Zcredential_abuseZvulnerability_exploitationZthird_party)zdefault credentialsro   rq   )rp   zweak credentialszzwak wachtwoord)rn   zcredential theftzgestolen inloggegevens)zwithout multi-factorzno mfazmissing mfazgeen mfa)Zdefault_credentialsZweak_credentialsZstolen_credentialsZmissing_mfag)\�Âõ(Ì?g¸…ëQ¸¾?çš™™™™™©?ç        )r   ÚmediumÚlowÚunknownc                   @  sˆ   e Zd ZU ded< ded< ded< ded< ded< ded< ded< ded	< d
ed< d
ed< dZd
ed< dZd
ed< eddœdd„ƒZdS )ÚRawItemÚstrÚ	feed_namer   r   r   Útitler   ÚsummaryÚcontentúOptional[str]Ú	publishedÚupdatedNÚguidÚrecord_type©Úreturnc                 C  s$   t d dd„ | j| j| jfD ƒ¡ƒS )Nú c                 s  s   | ]}|r|V  qd S ©N© ©Ú.0Úxr‡   r‡   úincidents_intel.pyÚ	<genexpr>ì  s   úzRawItem.text.<locals>.<genexpr>)Ú
clean_textÚjoinr{   r|   r}   )Úselfr‡   r‡   r‹   Útexté  s    
ýþÿzRawItem.text)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r�   r‚   Úpropertyr�   r‡   r‡   r‡   r‹   rx   Ú  s   
rx   c                   @  s   e Zd ZdS )ÚCollectorErrorN)r‘   r’   r“   r‡   r‡   r‡   r‹   r–   ÷  s   r–   zdt.datetimerƒ   c                   C  s   t j t jj¡S r†   )ÚdtÚdatetimeÚnowÚtimezoneÚutcr‡   r‡   r‡   r‹   Úutc_nowû  s    rœ   zOptional[dt.datetime]ry   )Úvaluer„   c                 C  s,   | pt ƒ } |  tjj¡jdd� ¡  dd¡S )Nr   )Zmicrosecondú+00:00ÚZ)rœ   Z
astimezoner—   rš   r›   ÚreplaceÚ	isoformat)r�   r‡   r‡   r‹   Úiso_zÿ  s    
r¢   r   c                 C  sB   | d u rdS t  t| ƒ¡}t d|¡}t d|¡}t d|¡ ¡ S )NÚ r…   ZNFKC)	ÚhtmlÚunescapery   ÚTAG_REÚsubÚunicodedataÚ	normalizeÚSPACE_REÚstrip)r�   r�   r‡   r‡   r‹   r�     s    r�   )r   r„   c                 C  s¬   t  | p
d ¡ ¡} | sdS zvtj | ¡}tjj|jdd�}dd„ |D ƒ}t 	dd|j
pXd¡}tj |j ¡ |j ¡ | d¡p~dtj |¡df¡W S  ty¦   |  Y S 0 d S )Nr£   T)Zkeep_blank_valuesc                 S  s2   g | ]*\}}|  ¡  d ¡s|  ¡ dvr||f‘qS )Zutm_>   ZfbclidZgclid)ÚlowerÚ
startswith)r‰   ÚkÚvr‡   r‡   r‹   Ú
<listcomp>  ó    z!normalize_url.<locals>.<listcomp>z/{2,}ú/)r¤   r¥   r«   ÚurllibÚparseÚurlsplitZ	parse_qslÚqueryÚrer§   ÚpathZ
urlunsplitZschemer¬   ÚnetlocÚrstripZ	urlencodeÚ	Exception)r   Úpartsr¶   r¸   r‡   r‡   r‹   Únormalize_url  s    6r½   z	list[str])r�   r„   c                 C  s0   t  d|  ¡ ¡} d dd„ | D ƒ¡} t d| ¡S )NZNFKDr£   c                 s  s   | ]}t  |¡s|V  qd S r†   )r¨   Z	combining)r‰   Zchr‡   r‡   r‹   rŒ     r±   zslug_tokens.<locals>.<genexpr>z[a-z0-9]{3,})r¨   r©   r¬   rŽ   r·   Úfindall©r�   r‡   r‡   r‹   Úslug_tokens  s    rÀ   é   ©ÚlengthÚint)r¼   rÃ   r„   c                 G  s0   d  dd„ |D ƒ¡}t | d¡¡ ¡ d | … S )Núc                 s  s   | ]}|pd   ¡  ¡ V  qdS )r£   N)r«   r¬   ©r‰   Úpr‡   r‡   r‹   rŒ   "  r±   zstable_hash.<locals>.<genexpr>úutf-8)rŽ   ÚhashlibZsha256ÚencodeZ	hexdigest)rÃ   r¼   Zmaterialr‡   r‡   r‹   Ústable_hash!  s    rË   c              
   C  sF  | sd S t | tjƒr.| jr| S | jtjjd�S t | tjƒrTtjj| tj	j
tjjd�S t| ƒ}|sdd S z0tj |¡}|r’|jr€|n|jtjjd�W S W n tttfy¬   Y n0 | dd¡}z(tj |¡}|jrÒ|n|jtjjd�W S  tyô   Y n0 dD ]F}z(tj |d d… |¡jtjjd�W   S  t�y>   Y qúY qú0 qúd S )N)ÚtzinforŸ   rž   )z%Y-%m-%dz%d-%m-%Yz%Y/%m/%dz%d/%m/%Yé
   )Ú
isinstancer—   r˜   rÌ   r    rš   r›   ÚdateZcombineÚtimeÚminr�   ÚemailZutilsZparsedate_to_datetimeÚ	TypeErrorÚ
ValueErrorÚOverflowErrorÚfromisoformatÚstrptime)r�   r�   ÚparsedÚ	candidateZfmtr‡   r‡   r‹   Ú
parse_date&  s6     (rÚ   r~   c                 C  s   t | ƒ}|r| ¡  ¡ S d S r†   )rÚ   rÏ   r¡   )r�   rØ   r‡   r‡   r‹   Úto_iso_dateD  s    rÛ   Úbool)r�   Údaysr™   r„   c                 C  s>   |dks| sdS t | ƒ}|s dS |p(tƒ }||tj|d� kS )Nr   T)rÝ   )rÚ   rœ   r—   Z	timedelta)r�   rÝ   r™   rØ   r‡   r‡   r‹   Ú	is_recentI  s    
rÞ   zIterable[str]zlist[re.Pattern[str]])Úpatternsr„   c                 C  s   dd„ | D ƒS )Nc                 S  s   g | ]}t  |t j¡‘qS r‡   )r·   ÚcompileÚIrÆ   r‡   r‡   r‹   r°   T  r±   z$compile_patterns.<locals>.<listcomp>r‡   )rß   r‡   r‡   r‹   Úcompile_patternsS  s    râ   c                 C  s   i | ]\}}|t |ƒ“qS r‡   )râ   )r‰   ÚkeyÚvaluesr‡   r‡   r‹   Ú
<dictcomp>W  s   ÿrå   zSequence[re.Pattern[str]])r�   rß   r„   c                   s   t ‡ fdd„|D ƒƒS )Nc                 3  s   | ]}|  ˆ ¡rd V  qdS )é   N)Úsearch)r‰   Úpatternr¿   r‡   r‹   rŒ   r  r±   zmatch_count.<locals>.<genexpr>)Úsum)r�   rß   r‡   r¿   r‹   Úmatch_countq  s    rê   )r�   Úmappingr„   c                   sR   d|   ¡ › d�‰ g }| ¡ D ](\}}t‡ fdd„|D ƒƒr| |¡ qtt|ƒƒS )Nr…   c                 3  s   | ]}|  ¡ ˆ v V  qd S r†   ©r¬   ©r‰   Útermrì   r‡   r‹   rŒ   y  r±   z matched_terms.<locals>.<genexpr>)r¬   ÚitemsÚanyÚappendÚsortedÚset)r�   rë   ÚresultZlabelÚtermsr‡   rì   r‹   Úmatched_termsu  s    rö   é   Úbytes)r   ÚtimeoutÚ
user_agentÚretriesr„   c           	      C  sÚ   t jj| |ddœd�}d }t|d ƒD ]š}zBt jj||d��}| ¡ W  d   ƒ W   S 1 s`0    Y  W q& t jjt jjt	t
fy¾ } z*|}||k rªt d|d  ¡ W Y d }~q&d }~0 0 q&td| › d|› �ƒ‚d S )	Nzgapplication/rss+xml, application/atom+xml, application/xml, text/xml, application/json;q=0.9, */*;q=0.5)z
User-AgentZAccept)Zheadersræ   )rù   ç      ð?zCould not fetch ú: )r³   ÚrequestZRequestÚrangeZurlopenÚreadÚerrorZURLErrorZ	HTTPErrorÚTimeoutErrorÚOSErrorrÐ   Úsleepr–   )	r   rù   rú   rû   rþ   Z
last_errorZattemptZresponseÚexcr‡   r‡   r‹   Úfetch_bytes~  s     þþ0*r  z
ET.ElementzSequence[str])ÚnodeÚnamesr„   c                 C  sT   dd„ |D ƒ}t | ƒD ]8}|j dd¡d  ¡ }||v rtd | ¡ ¡ƒ  S qdS )Nc                 S  s   h | ]}|  ¡ ’qS r‡   rì   )r‰   r   r‡   r‡   r‹   Ú	<setcomp>“  r±   zchild_text.<locals>.<setcomp>Ú}ræ   éÿÿÿÿr£   )ÚlistÚtagÚrsplitr¬   r�   rŽ   Úitertext)r  r  ZwantedÚchildÚlocalr‡   r‡   r‹   Ú
child_text’  s    r  )r  r„   c                 C  s†   t | ƒD ]x}|j dd¡d  ¡ }|dkr,q|j dd¡}|j dd¡}|r\|d	v r\|  S td | ¡ ¡ƒ}| 	d
¡r|  S qdS )Nr
  ræ   r  ÚlinkÚhrefr£   ÚrelÚ	alternate>   r£   r  Zhttp)
r  r  r  r¬   ZattribÚgetr�   rŽ   r  r­   )r  r  r  r  r  r�   r‡   r‡   r‹   Ú
child_link›  s    

r  zdict[str, Any]zlist[RawItem])ÚdataÚfeedr„   c                 C  s®  zt  | ¡}W n@ t jyN } z&td|d › d|› �ƒ|‚W Y d }~n
d }~0 0 |j dd¡d  ¡ }|dkr’tdd	„ |D ƒ|ƒ}d
d„ |D ƒ}n*|dkrªdd„ |D ƒ}ndd„ | ¡ D ƒ}g }|D ]ä}t	|dgƒ}	t
|ƒpæt	|dgƒ}
t	|ddgƒ}t	|ddgƒ}t	|g d¢ƒ}t	|ddgƒ}t	|ddgƒ}|	�s<|
�s<qÄ| t| d| dd¡¡| d| dd¡¡| dd¡| dd¡|	�p‚d t|
ƒ||t|ƒt|ƒ|�p d d!�¡ qÄ|S )"NzInvalid XML from r   rý   r
  ræ   r  Úrssc                 s  s,   | ]$}|j  d d¡d  ¡ dkr|V  qdS )r
  ræ   r  ÚchannelN©r  r  r¬   rˆ   r‡   r‡   r‹   rŒ   ²  r±   z!parse_feed_xml.<locals>.<genexpr>c                 S  s*   g | ]"}|j  d d¡d  ¡ dkr|‘qS )r
  ræ   r  Úitemr  rˆ   r‡   r‡   r‹   r°   ³  r±   z"parse_feed_xml.<locals>.<listcomp>r  c                 S  s*   g | ]"}|j  d d¡d  ¡ dkr|‘qS )r
  ræ   r  Úentryr  rˆ   r‡   r‡   r‹   r°   µ  r±   c                 S  s*   g | ]"}|j  d d¡d  ¡ dv r|‘qS )r
  ræ   r  >   r  r  r  rˆ   r‡   r‡   r‹   r°   ·  r±   r{   r  Údescriptionr|   Zencodedr}   )Zpubdater   rÏ   Zissuedr€   Zmodifiedr�   Úidr   r   ZUnknownr   r   rw   úUntitled source item©rz   r   r   r   r{   r   r|   r}   r   r€   r�   )ÚETZ
fromstringZ
ParseErrorr–   r  r  r¬   ÚnextÚiterr  r  rñ   rx   r  r½   rÛ   )r  r  Úrootr  Z	root_namer  ZnodesZresultsr  r{   r  r|   r}   r   r€   r�   r‡   r‡   r‹   Úparse_feed_xmlª  sL    0

õÿr(  c                      s\   e Zd ZdZddddœ‡ fdd„Zddddœd	d
„Zdddœdd„Zdddœdd„Z‡  ZS )ÚAIVDOverviewParserzà
    Extract dated AIVD publication links from an overview page.

    This parser deliberately uses only Python's standard library.
    The detail pages are fetched separately for their summary and
    publication date.
    ry   ÚNone)Úbase_urlr   r„   c                   s2   t ƒ  ¡  || _|| _g | _d| _d| _g | _d S )NFr£   )ÚsuperÚ__init__r+  r   ÚlinksÚ_inside_linkÚ_current_hrefÚ_current_text)r�   r+  r   ©Ú	__class__r‡   r‹   r-  ß  s    
zAIVDOverviewParser.__init__zlist[tuple[str, Optional[str]]])r  Úattrsr„   c                 C  s    |  ¡ dkrd S t|ƒ}| d¡p$d}tj | j|¡}tj |¡}|j  ¡ dkrTd S | j	|j
vrdd S t | j	¡d }t ||j
¡s†d S d| _t|ƒ| _g | _d S )NÚar  r£   zwww.aivd.nlz\d{4}/\d{2}/\d{2}/T)r¬   Údictr  r³   r´   Zurljoinr+  rµ   r¹   r   r¸   r·   Úescaperç   r/  r½   r0  r1  )r�   r  r4  Z
attributesr  Zabsolute_urlrØ   Zdate_patternr‡   r‡   r‹   Úhandle_starttagï  s4    þÿ
ÿÿÿz"AIVDOverviewParser.handle_starttag)r  r„   c                 C  s   | j r| j |¡ d S r†   )r/  r1  rñ   )r�   r  r‡   r‡   r‹   Úhandle_data  s    zAIVDOverviewParser.handle_data)r  r„   c                 C  s^   |  ¡ dkrd S | jsd S td | j¡ƒ}|rH| jrH| j || jdœ¡ d| _d| _g | _d S )Nr5  r…   )r{   r   Fr£   )r¬   r/  r�   rŽ   r1  r0  r.  rñ   )r�   r  r{   r‡   r‡   r‹   Úhandle_endtag  s     
ÿ
þÿz AIVDOverviewParser.handle_endtag)	r‘   r’   r“   Ú__doc__r-  r8  r9  r:  Ú__classcell__r‡   r‡   r2  r‹   r)  Ö  s
   +r)  z£<meta\s+
        (?=[^>]*\b(?:name|property)=["']
        (?P<name>[^"']+)["'])
        (?=[^>]*\bcontent=["']
        (?P<content>[^"']*)["'])
        [^>]*>
    zN<time\b[^>]*datetime=["']
        (?P<datetime>[^"']+)
        ["'][^>]*>
    zdict[str, str])Ú	html_textr„   c                 C  sH   i }t  | ¡D ]4}t| d¡ƒ ¡ }t| d¡ƒ}|r|r|||< q|S )zª
    Extract useful metadata from HTML meta elements.

    Returns a dictionary indexed by the lower-case value
    of the meta element's name or property attribute.
    r   r}   )ÚMETA_TAG_REÚfinditerr�   Úgroupr¬   )r=  ÚmetadataÚmatchr   r}   r‡   r‡   r‹   Úextract_html_metadataJ  s    
ÿÿ
rC  c                 C  sl   t j | ¡j}t d|¡}|s"dS | ¡ \}}}zt t	|ƒt	|ƒt	|ƒ¡}W n t
yb   Y dS 0 | ¡ S )zÎ
    Extract the publication date from an AIVD detail-page URL.

    Expected URL structures:

        /documenten/YYYY/MM/DD/...
        /actueel/nieuws/YYYY/MM/DD/...

    Returns YYYY-MM-DD or None.
    z/(\d{4})/(\d{2})/(\d{2})/N)r³   r´   rµ   r¸   r·   rç   Úgroupsr—   rÏ   rÄ   rÔ   r¡   )r   r¸   rB  ÚyearZmonthZdayr�   r‡   r‡   r‹   Úextract_aivd_date_from_urle  s"    þýrF  zdict[str, Optional[str]])r   rù   rú   r„   c                 C  sà   t | ||d�}|jddd�}t|ƒ}| d¡p@| d¡p@| d¡}| d¡pb| d	¡pb| d
¡pbd}| d¡p€| d¡p€| d¡}| d¡p”| d¡}	|s²t |¡}
|
r²|
 d¡}t|ƒpÀt| ƒ}t|	ƒ}t	|ƒt	|ƒ||dœS )zç
    Fetch an AIVD detail page and extract basic metadata.

    Metadata in the page is preferred. If no publication date
    is available in the metadata or a time element, the date
    encoded in the URL is used as fallback.
    ©rù   rú   rÈ   r    ©Úerrorszog:titleztwitter:titler{   r   zog:descriptionztwitter:descriptionr£   zarticle:published_timerÏ   zdcterms.datezarticle:modified_timezlast-modifiedr˜   ©r{   r   r   r€   )
r  ÚdecoderC  r  ÚTIME_TAG_RErç   r@  rÛ   rF  r�   )r   rù   rú   r  r=  rA  r{   r   r   r€   Z
time_matchZpublished_dateZupdated_dater‡   r‡   r‹   Úextract_aivd_detail‹  sf    ýþÿ
ÿý
ÿþü
ÿý
þÿÿþÿürM  )Úsourcerù   rú   rÝ   r„   c                 C  s¾  t  d| d | d ¡ t| d ||d�}|jddd�}t| d | d d	�}| |¡ i }|jD ]}|||d < q^t  d
t|ƒ| d ¡ g }	| ¡ D �]}t	|d ƒ}
t
|
|ƒs°q’zt|d ||d�}W nJ t�y } z0t  d|d |¡ |d d|
d dœ}W Y d }~n
d }~0 0 | d¡�p*|d �p*d}| d¡�pD|d �pDd}|	 t| d | d | d | d t|ƒ|d t| d¡ƒd| d¡�pŒ|
| d¡|d d�¡ q’t  dt|	ƒ| d ¡ |	S )NúFetching %s: %sr   r   rG  rÈ   r    rH  r   )r+  r   z'Discovered %d AIVD detail links from %sz'Could not fetch AIVD detail page %s: %sr{   r£   rJ  zUntitled AIVD publicationr   r   r   r   r   r€   r#  ú!Collected %d source items from %s)ÚLOGÚinfor  rK  r)  r  r.  Úlenrä   rF  rÞ   rM  r»   Úwarningr  rñ   rx   r�   )rN  rù   rú   rÝ   r  r=  ÚparserZunique_linksr  rï   Zurl_dateZdetailr  r{   r‡   r‡   r‹   Úcollect_aivd_sourceÙ  s¶    ýýþþ

ýÿ
ý
ýüÿýÿýÿðÿý    ý rV  )r  rù   rú   r„   c                 C  sT   t  d|  dd¡| d ¡ t| d ||d�}t|| ƒ}t  dt|ƒ|  dd¡¡ |S )NrO  r   r  r   rG  rP  )rQ  rR  r  r  r(  rS  )r  rù   rú   r  rï   r‡   r‡   r‹   Úcollect_feedR  s
    
rW  r   )r¸   r„   c                 C  s¤  | j ddd��}t |¡}W d   ƒ n1 s.0    Y  t|tƒrN| d|¡n|}t|tƒsltd| › d�ƒ‚g }t|ƒD �]$\}}t|tƒsœt	 
d|| ¡ qx| tt| d¡p²| jƒt| d	¡pÂd
ƒt| d¡pÒdƒt| d¡pâdƒt| d¡pòdƒt| d¡�pdƒt| d¡�p"| d¡�p"dƒt| d¡�p@| d¡�p@dƒt| d¡�pX| d¡ƒt| d¡ƒt| d¡�p|| d¡ƒ�p„d t| d¡ƒ�p–d d�¡ qx|S )NÚrrÈ   ©Úencodingrï   zManual input z; must contain a JSON list or an object with an 'items' listz Skipping non-object row %d in %srz   r   zManual inputr   Úmanualr   rw   r{   r"  r   r£   r|   r   r}   Úbodyr   rÏ   r€   r�   r!  r‚   )rz   r   r   r   r{   r   r|   r}   r   r€   r�   r‚   )ÚopenÚjsonÚloadrÎ   r6  r  r  rÔ   Ú	enumeraterQ  rT  rñ   rx   r�   r   r½   rÛ   )r¸   ÚhandleÚpayloadZrowsrï   ÚindexÚrowr‡   r‡   r‹   Úload_manual_json^  s<    (

ÿòÿre  z&tuple[bool, list[str], dict[str, int]])r  r„   c                   s  | j ‰ tˆ tƒ}tˆ tƒ}tˆ tƒ}tˆ tƒ}tˆ tƒ}t‡ fdd„t 	¡ D ƒƒ}g }t
| jƒ ¡ }|dv }	|dkr†|dkr†| d¡ |dkr |dkr | d¡ |	sÆ|dkrÆ|dkrÆ|dkrÆ| d¡ |	sì|dkrì|dkrì|dkrì| d¡ | }
||||||d	œ}|
||fS )
a4  
    Determine whether a source item describes a concrete, observed incident.

    A score alone is not sufficient. Strategic publications, organizational
    news and vulnerability-only advisories must not become incident records
    unless the text also contains clear evidence of a concrete incident.
    c                 3  s   | ]}t ˆ |ƒV  qd S r†   ©rê   )r‰   rß   r¿   r‡   r‹   rŒ   ¤  s   ÿz)evaluate_incident_gate.<locals>.<genexpr>¾   Úexploitation_activityÚstrategic_assessmentÚthreat_observationr   Z$organizational_news_without_incidentZ#vulnerability_only_without_incidentZ$no_relevant_incident_or_intelligenceZ!no_incident_or_strategic_evidence)Úconcrete_matchesÚobserved_matchesÚvulnerability_matchesÚstrategic_matchesÚorganizational_matchesÚtyped_matches)r�   rê   ÚCOMPILED_CONCRETE_INCIDENTÚCOMPILED_OBSERVEDÚCOMPILED_VULNERABILITY_ONLYÚCOMPILED_STRATEGIC_CONTENTÚCOMPILED_ORGANIZATIONAL_NEWSré   ÚCOMPILED_INCIDENTrä   r�   r‚   r¬   rñ   )r  rk  rl  rm  rn  ro  rp  ÚreasonsÚexplicit_record_typeZexplicit_evidence_recordÚacceptedÚdetailsr‡   r¿   r‹   Úevaluate_incident_gate~  s~    þþþþþþÿ
ÿþ
ÿþýü
ÿþýü
ú	r{  ztuple[float, dict[str, Any]]c                   s8  | j ‰ tˆ tƒ}tˆ tƒ}tˆ tƒ}tˆ tƒ}‡ fdd„t ¡ D ƒ}dd„ | ¡ D ƒ}d}|t|dƒd 7 }|t|dƒd 7 }|tt	| 
¡ ƒdƒd	 7 }|t | jd¡7 }|t|dƒd	 8 }|t|d
ƒd 8 }|ræ|sæ|dkræ|d8 }dˆ  ¡ v �sdˆ  ¡ v �r
|d7 }tdtd|ƒƒ}|||||dœ}t|dƒ|fS )Nc                   s   i | ]\}}|t ˆ |ƒ“qS r‡   rf  ©r‰   Úkindrß   r¿   r‡   r‹   rå   â  r±   z"incident_score.<locals>.<dictcomp>c                 S  s   i | ]\}}|r||“qS r‡   r‡   )r‰   r}  Úcountr‡   r‡   r‹   rå   ã  r±   rt   é   ç
×£p=
Ç?g{®GázÄ?é   g¤p=
×£À?r÷   g{®Gáz¤?r   ç333333Ó?zactive exploitationzactief misbruiktç333333Ã?rü   )Zgeneric_matchesrl  Zspeculation_matchesZnon_incident_matchesrp  )r�   rê   ÚCOMPILED_GENERICrr  ÚCOMPILED_SPECULATIONÚCOMPILED_NON_INCIDENTrv  rï   rÑ   ré   rä   ÚSOURCE_AUTHORITY_WEIGHTSr  r   r¬   ÚmaxÚround)r  ZgenericÚobservedÚspeculativeZnon_incidentÚtypedÚscorerz  r‡   r¿   r‹   Úincident_scoreÜ  s4    



ûrŽ  c                   sN   ‡fdd„t  ¡ D ƒ‰g d¢‰ tˆ‡ ‡fdd„dd�d }ˆ| dkrJ|S d	S )
Nc                   s   i | ]\}}|t ˆ |ƒ“qS r‡   rf  r|  r¿   r‡   r‹   rå     r±   z"classify_event.<locals>.<dictcomp>)
r   r   r   r"   r   r   r    r!   r#   r$   c                   s   ˆ|  ˆ   | ¡ fS r†   )rc  )r}  )ÚpriorityÚscoresr‡   r‹   Ú<lambda>  r±   z classify_event.<locals>.<lambda>T©rã   Úreverser   Zcyber_attack)rv  rï   rò   )r�   Zbestr‡   )r�  r�  r�   r‹   Úclassify_event   s    r”  )r�   r   r„   c                 C  s@   t | tƒ}t | tƒ}|r,|dkr,|dkr,dS |r4dS |r<dS dS )Nr   r   Ú	confirmedÚreportedÚ	suspected)rê   rr  r…  )r�   r   rŠ  r‹  r‡   r‡   r‹   Úinfer_status  s    

r˜  c                   sš   d|   ¡ › d�‰ t‡ fdd„t ¡ D ƒƒ}t|ƒdkr<d}n2t|ƒdkrNd}n t‡ fdd„d	D ƒƒrjd
}nd}t‡ fdd„dD ƒƒrŠdgng }|||dœS )Nr…   c                   s*   h | ]"\}}t ‡ fd d„|D ƒƒr|’qS )c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rí   rì   r‡   r‹   rŒ     r±   z,infer_geography.<locals>.<setcomp>.<genexpr>)rð   )r‰   Úcountryrõ   rì   r‡   r‹   r	    r±   z"infer_geography.<locals>.<setcomp>ræ   Úmulti_countryr™  c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rí   rì   r‡   r‹   rŒ      r±   z"infer_geography.<locals>.<genexpr>)ÚeuropeÚeuropeanzeu-wideÚeuropaZregionalrw   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rí   rì   r‡   r‹   rŒ   $  r±   )r›  rœ  r�  ZEurope)ÚscopeÚ	countriesÚregions)r¬   rò   ÚCOUNTRY_TERMSrï   rS  rð   )r�   rŸ  rž  r   r‡   rì   r‹   Úinfer_geography  s     r¢  )r�   Ú
event_typer„   c              
     st  t | tƒ‰ |  ¡ }|dkp.t‡ fdd„dD ƒƒ}dˆ v r<dnd }dˆ v rLdnd }t‡ fdd„d	D ƒƒrjdnd }t‡ fd
d„dD ƒƒrˆdnd }t‡ fdd„dD ƒƒr¦dnd }d}	|sÆ|du sÆtˆ ƒdkrÌd}	n"tˆ ƒdksà|dv ræd}	nˆ rîd}	g }
t d| ¡D ]H}| ¡ ‰t‡fdd„t ¡ D ƒƒ�r4|
 | 	¡ ¡ t|
ƒdkrþ �qHqþˆ d 
|
¡d d… �p`d |||||||	dœ	S )Nr   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   ©Ú
categoriesr‡   r‹   rŒ   +  r±   zinfer_impact.<locals>.<genexpr>)ri   rj   rk   rh   Trm   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   r¤  r‡   r‹   rŒ   .  r±   )re   rl   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   r¤  r‡   r‹   rŒ   /  r±   )rf   rg   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   r¤  r‡   r‹   rŒ   0  r±   )ri   rf   rw   r�  r   r÷   >   r   r   r"   ru   rv   z(?<=[.!?])\s+c                 3  s    | ]}|D ]}|ˆ v V  q
qd S r†   r‡   )r‰   rõ   rî   )Ú
sentence_lr‡   r‹   rŒ   =  r±   r…   i°  )	r¥  r   ÚphysicalrC   ÚsafetyÚavailabilityÚconfidentialityÚ	integrityÚseverity)rö   ÚIMPACT_TERMSr¬   rð   rS  r·   Úspliträ   rñ   r«   rŽ   )r�   r£  r¬   r§  rC   r¨  r©  rª  r«  r¬  ZdescriptionsZsentencer‡   )r¥  r¦  r‹   Úinfer_impact(  s@    
÷r¯  )r�   Úimpactr„   c                   s¬   t | d¡pdt| tƒv ƒ}d }d }|  ¡ ‰ |ržt‡ fdd„dD ƒƒrLd}nt‡ fdd„dD ƒƒrfd	}t‡ fd
d„dD ƒƒr‚d}ndˆ v sšdˆ v sšdˆ v ržd}|||d dœS )Nr§  rb   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   rì   r‡   r‹   rŒ   U  r±   z'infer_cyber_physical.<locals>.<genexpr>)rO   ZremotelyrQ   zCReported remote access or remote manipulation of a connected systemc                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   rì   r‡   r‹   rŒ   W  r±   )rM   rJ   rK   rL   z7Reported manipulation or compromise of a control systemc                 3  s   | ]}|ˆ v V  qd S r†   r‡   rˆ   rì   r‡   r‹   rŒ   Y  r±   )r\   r]   ZontdooizBReported modification of a temperature-controlled physical processr[   rc   rd   z%Reported physical or equipment impact)Úis_cyber_physicalÚdigital_actionÚphysical_effectZpotential_consequence)rÜ   r  rö   ÚTHEME_TERMSr¬   rð   )r�   r°  Zis_cpr²  r³  r‡   rì   r‹   Úinfer_cyber_physicalO  s$    ürµ  )r�   ÚthemesÚsectorsrb   r„   c           	        s"  |   ¡ ‰ g d¢}d|v p,t‡ fdd„|D ƒƒ}d}g }|rL|d7 }| d¡ | d¡rh|d	7 }| d
¡ d|v r‚|d7 }| d¡ d|v s’d|v r¤|d7 }| d¡ d|v r¾|d7 }| d¡ tt|dƒdƒ}|rØd}n0|dkræd}n"|dkrôd}n|dk�rd}nd}|||d |¡�pd d œS )!N)r1   r2   r3   r5   r6   r7   r4   r8   r9   r:   r;   r=   r1   c                 3  s   | ]}|ˆ v V  qd S r†   r‡   rí   rì   r‡   r‹   rŒ   i  r±   z-infer_automotive_relevance.<locals>.<genexpr>rt   g333333ã?z@The source explicitly concerns vehicles or the automotive sectorr±  r€  z=The incident concerns digital influence on a physical processr_   g{®Gáz´?z0Remote access or remote manipulation is reportedrN   r^   gìQ¸…ë±?zEThe affected environment includes connected or operational technologyra   rs   z=The incident includes a supply-chain or third-party dimensionrü   r÷   Údirectg      Ð?Zstrongly_relatedrƒ  Z	analogousr   Z
contextualZnonez; )r¸  Ú	relevancer�  Úreason)r¬   rð   rñ   r  r‰  rÑ   rŽ   )	r�   r¶  r·  rb   Zdirect_termsr¸  r�  rw  r¹  r‡   rì   r‹   Úinfer_automotive_relevancee  s>    






r»  éR  )r  Ú
max_lengthr„   c                 C  sP   t | jp| jƒ}|s| jS t|ƒ|kr*|S |d |…  dd¡d }| d¡d S )Nr…   ræ   r   z ,;:z...)r�   r|   r}   r{   rS  r  rº   )r  r½  r|   Zclippedr‡   r‡   r‹   Úmake_summary‰  s    r¾  )r  Úscore_detailsr„   c           
      C  sÒ   t | jƒ ¡ }|dv r|S | di ¡}| di ¡}| dd¡}| dd¡}| dd¡}| dd¡}| dd¡}| d	d¡}	|dkrŽ|	dkrŽd
S |dkr¢|	dkr¢dS |dkr®dS |dkrÂ|dkrÂdS |dkrÎdS dS )N>   rh  Úincidentri  rj  Úincident_gaterk  r   rn  ro  rm  rl  rp  ri  ZorganizationalrÀ  rh  rj  rw   )r�   r‚   r¬   r  )
r  r¿  rx  Zgaterk  rn  ro  rm  rl  rp  r‡   r‡   r‹   Údetermine_record_type“  sr    ÿþ	þþþþþþþÿþÿþÿþrÂ  Úfloat)r  r�  r¿  Úrun_timer„   c                 C  sâ  | j }t|ƒ}t| |ƒ}|dkr$d }t|| jƒ}t|tƒ}t|tƒ}	|dkr^d|	vr^|	 d¡ t	t
|	ƒƒ}	t||ƒ}
t||
ƒ}t||	||ƒ}t|tƒ}t|tƒ}t	dd„ t |¡D ƒƒ}d|v sÈd|	v rÌdnd }d	t| j| jpè| jpè| jd
d� ¡  }d t| jƒd d… ¡}t|| j�pd| j�p&d| j�p8| j�p8ddd�}| j�pRttƒ jƒd d… }d|› dt|d
d� ¡ › �}|}| jdkd| jdv dœ}dg}|�rª| d¡ |
d �r¾| d¡ |�rÎ| d¡ |�rÞ| d¡ |d �rò| d¡ || jt| ƒ|||d d | j| j dœt!|ƒ|g g g g |g |||d d œ|
|dg g g d d!d"œ|	| jg|| j| j| j�pdd | j| j| j|d#œg||||d$œg g g g g g d%œ|d d g d&œt	t
d'd(„ d)|g|�r¼|gng  |	 D ƒƒƒ||d*|d+d d,œd-œS ).NrÀ  r   rb   c                 S  s   h | ]}|  ¡ ’qS r‡   )Úupper)r‰   rB  r‡   r‡   r‹   r	  	  r±   zbuild_record.<locals>.<setcomp>rr   r`   TzSRC-é   rÂ   r…   é   r£   é   r�  zINC-ú-r   F¾   Zthreat_intelligenceZtechnical_analysisr   )Úofficial_sourceÚmultiple_independent_sourcesÚtechnical_analysis_availableZincident_existsZtarget_sectorr¥  r°  Úattack_vectorZvulnerabilityr±  Zphysical_impact)Zfirst_observedZlast_observedr   r€   )Úinitial_accessÚ
techniquesrÎ  ÚvulnerabilitiesÚcvesÚauthentication_issuesÚinternet_exposedr   rw   )Zknownr  ÚtypeZ
motivationZattributionZattribution_confidence)Ú	source_idr   r{   r   r   r   r   Úsupports)Ú
confidenceÚconfidence_scoreÚverificationZselection_details)Zthreat_matchesZcve_matchesZpaper_matchesZgithub_matchesZsignal_matchesZnarrative_matches)Zcampaign_idZcampaign_nameZrelated_incidentsc                 s  s   | ]}|r|V  qd S r†   r‡   )r‰   r  r‡   r‡   r‹   rŒ   t  s   úzbuild_record.<locals>.<genexpr>Z
real_worldzincidents_intel.pyÚpending)Ú
created_atÚ
updated_atZdiscovered_byÚdeduplication_keyÚreview_statusÚreviewed_by)Úincident_idr{   r|   r‚   Ústatusr£  ÚdatesÚ	geographyr·  ÚtargetsÚorganizationsÚattackr°  rb   Úactorsr¶  Úsource_typesÚsourcesÚevidenceÚcorrelationZautomotive_relevanceÚcampaignÚtagsÚ
processing)"r�   r”  rÂ  r˜  r   rö   ÚSECTOR_TERMSr´  rñ   rò   ró   r¯  rµ  r»  ÚATTACK_VECTOR_TERMSÚAUTHENTICATION_TERMSÚCVE_REr¾   rË   r   r   r�   r{   rÅ  rŽ   rÀ   r   ry   rœ   rE  r   r¾  r€   r¢  )r  r�  r¿  rÄ  r�   r£  r‚   râ  r·  r¶  r°  Zcpr1   Zattack_vectorsZauth_issuesrÒ  rÔ  rÖ  Z	title_keyrÞ  rE  rá  rÙ  rÚ  r×  r‡   r‡   r‹   Úbuild_recordî  s   þþ






&

ûÿÿý






üø
ú
øÿüú
ÿþýÿú¯rô  )r5  Úbr„   c                 C  s>   t t| ƒƒt t|ƒƒ }}|r"|s&dS t||@ ƒt||B ƒ S )Nrt   )ró   rÀ   rS  )r5  rõ  ZtaÚtbr‡   r‡   r‹   Útitle_similarity‹  s    r÷  )rN  r„   c                 C  s.   |   d¡p,|   d¡p,t|   dd¡|   dd¡ƒS )Nr   rÖ  r   r£   r{   )r  rË   )rN  r‡   r‡   r‹   Úsource_identity’  s    rø  )ÚprimaryÚ	secondaryrÄ  r„   c           	      C  s‚  t  t  | ¡¡}dd„ |d D ƒ}|d D ],}t|ƒ|vr*|d  |¡ | t|ƒ¡ q*dD ](}tt| |g ¡| |g ¡ ƒƒ||< q\dD ]4}tt|d  |g ¡|d  |g ¡ ƒƒ|d |< qŠt	|d d ƒdkràd	|d d
< dD ]4}tt|d  |g ¡|d  |g ¡ ƒƒ|d |< qätt|d  dg ¡|d  dg ¡ ƒƒ|d d< dddddœ}| |d d¡| |d d¡k�r–|d |d< |d |d d< t
tt|d  d¡�p®dƒt|d  d¡�pÄdƒƒdƒ|d d< dd„ |d D ƒ}t	|ƒdk|d d d< tdd„ |d D ƒƒ|d d d< tdd„ |d D ƒƒ|d d d< t	| d d!¡ƒt	| d d!¡ƒk�rr|d  |d < ||d" d#< |S )$Nc                 S  s   h | ]}t |ƒ’qS r‡   )rø  ©r‰   rN  r‡   r‡   r‹   r	  ˜  r±   z merge_records.<locals>.<setcomp>rê  )r·  r¶  ré  rî  ræ  )rŸ  r   rä  rŸ  ræ   rš  rž  )rÎ  rÑ  rÒ  rÓ  rÏ  rÐ  rç  r°  r¥  r   r÷   r  )Ú
unverifiedr—  r–  r•  râ  rë  rØ  rÙ  r�  c                 S  s    h | ]}|  d ¡r|  d ¡’qS )r   ©r  rû  r‡   r‡   r‹   r	  ²  r±   rÚ  rÌ  c                 s  s   | ]}|  d ¡dkV  qdS )r   r   Nrý  rû  r‡   r‡   r‹   rŒ   ´  r±   z merge_records.<locals>.<genexpr>rË  c                 s  s   | ]}|  d ¡dv V  qdS )r   rÊ  Nrý  rû  r‡   r‡   r‹   rŒ   µ  s   ÿrÍ  r|   r£   rï  rÝ  )r^  ÚloadsÚdumpsrø  rñ   Úaddrò   ró   r  rS  r‰  rˆ  rÃ  rð   )	rù  rú  rÄ  ZmergedZknown_sourcesrN  ÚfieldZstatus_rankZ
publishersr‡   r‡   r‹   Úmerge_records–  sF    &220"þý"þ"r  ç¤p=
×£à?zlist[dict[str, Any]])ÚrecordsÚ	thresholdr„   c              
     sÄ  g }i ‰ i }t ƒ }| D �]¦}dd„ |d D ƒ}t‡ fdd„|D ƒd ƒ}|d  d¡}|d u rl||v rl|| }|}	|	d u �rF|d  d	¡}
t|ƒD ]´\}}|d  d	¡}|
rò|ròz"ttj |
¡tj |¡ jƒ}W n t	yæ   d
}Y n0 |dkròq�t
|d |d ƒ}t|d d ƒt|d d ƒ@ }||k�s:|r�|dkr�|}	 �qFq�|	d u �rŠ| |¡ t|ƒd }|D ]}|ˆ |< �qj|�r¾|||< qt||	 ||ƒ||	< |D ]}|	ˆ |< �q¢|r|	||< q|S )Nc                 S  s    g | ]}|  d ¡r|  d ¡‘qS )r   rý  rû  r‡   r‡   r‹   r°   Ç  r±   zdeduplicate.<locals>.<listcomp>rê  c                 3  s   | ]}|ˆ v rˆ | V  qd S r†   r‡   )r‰   r   ©Zby_urlr‡   r‹   rŒ   È  r±   zdeduplicate.<locals>.<genexpr>rï  rÞ  rã  r   iç  é   r{   rç  rÒ  r‚  ræ   )r¢   r%  r  r`  Úabsr—   rÏ   rÖ   rÝ   rÔ   r÷  ró   rñ   rS  r  )r  r  rD  Zby_keyrÄ  ÚrecordZurlsZexact_indexrã   Zcandidate_indexr   rc  ÚexistingZexisting_dateZdeltaZ
similarityZshared_cvesr   r‡   r  r‹   ÚdeduplicateÀ  sP    

"
 



r  c              
   C  s¨   |   ¡ sg S z\| jddd��}t |¡}W d   ƒ n1 s<0    Y  | dg ¡}t|tƒsdtdƒ‚|W S  ty¢ } z"td| › d|› �ƒ|‚W Y d }~n
d }~0 0 d S )NrX  rÈ   rY  Ú	incidentsz'incidents' is not a listzCould not load existing output rý   )	Úexistsr]  r^  r_  r  rÎ   r  rÔ   r»   )r¸   ra  rb  r  r  r‡   r‡   r‹   Úload_existingð  s    (
r  )ÚnewÚoldr„   c                 C  sî   t  t  | ¡¡}| d| d ¡|d< | di ¡ d¡p@| d d |d d< dD ]*}| di ¡ |¡}|dvrN||d |< qNdD ]}| |¡r~|| ||< q~dD ]4}| |¡ržtdd	„ ||  ¡ D ƒƒrž|| ||< qž| d
¡rê|d
 |d
< |S )Nrá  rï  rÜ  )rß  rà  )Nr£   rÛ  )rå  ræ  )rè  rí  c                 s  s    | ]}|d dg i dfvV  qd S )NFrw   r‡   )r‰   r¯   r‡   r‡   r‹   rŒ   
  r±   z)preserve_review_fields.<locals>.<genexpr>rì  )r^  rþ  rÿ  r  rð   rä   )r  r  rô   r  Ú	old_valueZsectionr‡   r‡   r‹   Úpreserve_review_fieldsþ  s     &
$
r  )Úfreshr
  rÄ  r„   c           	        sî   dd„ |D ƒ}i ‰ |D ],}|  dg ¡D ]}|  d¡r&|ˆ |d < q&qg }tƒ }| D ]p}|  |d   d¡¡}|d u rŽt‡ fdd„|d D ƒd ƒ}|r¸t||ƒ}||d d	< | |  d
¡¡ | |¡ qR|D ]}|  d
¡|vrÈ| |¡ qÈt|ƒS )Nc                 S  s2   i | ]*}|  d i ¡  d¡r|  d i ¡  d¡|“qS )rï  rÞ  rý  rˆ   r‡   r‡   r‹   rå     r±   z'merge_with_existing.<locals>.<dictcomp>rê  r   rï  rÞ  c                 3  s,   | ]$}|  d ¡ˆ v rˆ   |  d ¡¡V  qdS )r   Nrý  rû  ©Z
old_by_urlr‡   r‹   rŒ     r±   z&merge_with_existing.<locals>.<genexpr>rÝ  rá  )r  ró   r%  r  r   rñ   r  )	r  r
  rÄ  Z
old_by_keyr  rN  rô   Zconsumed_old_idsr	  r‡   r  r‹   Úmerge_with_existing  s*    

r  )r  rÄ  Ú
statisticsr„   c                 C  s¦   | j dd„ dd� tdd„ | D ƒƒ}tdd„ | D ƒƒ}tdd„ | D ƒƒ}td	d„ | D ƒƒ}tt|d
t| ƒi |¥tt| ¡ ƒƒtt| ¡ ƒƒt| ¡ ƒdœ¥dœ| dœS )Nc                 S  s$   |   di ¡  d¡pd|   d¡p dfS )Nrã  r   r£   r{   rý  )rŠ   r‡   r‡   r‹   r‘  -  r±   zbuild_payload.<locals>.<lambda>Tr’  c                 s  s   | ]}|  d ¡pdV  qdS )r‚   rw   Nrý  rˆ   r‡   r‡   r‹   rŒ   0  s   ÿz build_payload.<locals>.<genexpr>c                 s  s   | ]}|  d ¡pdV  qdS )r£  Znot_applicableNrý  rˆ   r‡   r‡   r‹   rŒ   5  s   ÿc                 s  s   | ]}|  d ¡pdV  qdS )râ  rw   Nrý  rˆ   r‡   r‡   r‹   rŒ   :  s   ÿc                 s  s$   | ]}|  d g ¡D ]
}|V  qqdS )r·  Nrý  )r‰   rŠ   Zsectorr‡   r‡   r‹   rŒ   ?  s   þzkReal-world cybersecurity incidents and cyber-physical events used as evidence for the NCRC Threat Register.)Zrecords_by_typeZincidents_by_typeZincidents_by_status)Zschema_versionZscript_versionZgenerated_atr   Úrecord_countr  )rA  r  )Úsortr   ÚSCHEMA_VERSIONÚSCRIPT_VERSIONrS  r6  rò   rï   )r  rÄ  r  Zby_record_typeZby_typeZ	by_statusZ	by_sectorr‡   r‡   r‹   Úbuild_payload,  s8    þþþþÿ
üúòr  )rb  r„   c                 C  sü  g }t | tƒsdgS t |  d¡tƒs.| d¡ |  d¡}t |tƒsP| d¡ |S tƒ }t|ƒD �]P\}}d|› d�}t |tƒs�| |› d�¡ q^d	D ]"}||vr”| |› d
|› d�¡ q”| d¡}||v rÚ| d|› �¡ | |¡ | d¡dv�r| |› d| d¡›�¡ t | d¡tƒ�r,| d¡�s<| |› d�¡ | di ¡ d¡}	|	d u�rzt |	tt	fƒ�sz| |› d�¡ q^t |	tt	fƒr^d|	  k�rždks^n | |› d�¡ q^|  di ¡ d¡}
|
d u�rø|
t
|ƒk�rø| d|
› dt
|ƒ› d�¡ |S )NzRoot must be a JSON objectrA  zmetadata must be an objectr  zincidents must be a listz
incidents[ú]z must be an object)rá  r{   râ  r£  rê  rï  Ú.z is requiredrá  zDuplicate incident_id: râ  >   r—  r•  r–  rü  z.status has invalid value rê  z).sources must contain at least one sourcerë  rÙ  z2.evidence.confidence_score must be numeric or nullr   ræ   z2.evidence.confidence_score must be between 0 and 1r  zmetadata.record_count=z but incidents contains ú records)rÎ   r6  r  rñ   r  ró   r`  r   rÄ   rÃ  rS  )rb  rI  r  Zseen_idsrc  r	  ÚprefixZrequiredrá  r�  r~  r‡   r‡   r‹   Úvalidate_payloadV  sF    







&r   r*  )r¸   rb  ÚprettyÚbackupr„   c                 C  s  | j jddd� |r8|  ¡ r8|  | jd ¡}t | |¡ tj| j	d dt
| j ƒd�\}}z|tj|ddd	��L}tj||d
|r~dnd d
d� | d¡ | ¡  t | ¡ ¡ W d   ƒ n1 s¼0    Y  t || ¡ W n: t�y   zt |¡ W n t�y   Y n0 ‚ Y n0 d S )NT)ÚparentsÚexist_okz.bakr  z.tmp)r  ÚsuffixÚdirÚwrÈ   rY  Fr÷   )Úensure_asciiÚindentZ	sort_keysÚ
)ÚparentÚmkdirr  Zwith_suffixr%  ÚshutilZcopy2ÚtempfileZmkstempr   ry   ÚosÚfdopenr^  ÚdumpÚwriteÚflushÚfsyncÚfilenor    r»   Úunlinkr  )r¸   rb  r!  r"  Zbackup_pathÚfdZtemporary_namera  r‡   r‡   r‹   Úatomic_write_json|  s$     
,r8  zargparse.Namespace)Úargsr„   c              
   C  s  | j r
g ndd„ tD ƒ}| jr˜t| jƒjddd��}t |¡}W d   ƒ n1 sR0    Y  t|tƒrr| 	dg ¡}t|t
ƒs„tdƒ‚| dd	„ |D ƒ¡ t| jp¢g d
d�D ]0\}}| d|› �|tj |¡jddddœ¡ qªi }|D ] }| 	d¡sôqä||t|d ƒ< qät
| ¡ ƒS )Nc                 S  s    g | ]}|  d d¡rt|ƒ‘qS )r   T)r  r6  )r‰   r  r‡   r‡   r‹   r°   ’  r±   zload_feeds.<locals>.<listcomp>rX  rÈ   rY  ÚfeedszA--feeds-file must contain a list or an object with a 'feeds' listc                 s  s(   | ] }t |tƒr| d d¡r|V  qdS )r   TN)rÎ   r6  r  rˆ   r‡   r‡   r‹   rŒ   š  r±   zload_feeds.<locals>.<genexpr>ræ   )Ústartz	CLI feed r  rw   Tr   r   )Zno_default_feedsÚDEFAULT_FEEDSZ
feeds_filer   r]  r^  r_  rÎ   r6  r  r  rÔ   Úextendr`  r  rñ   r³   r´   rµ   r¹   r½   rä   )r9  r:  ra  ZextraZnumberr   Úuniquer  r‡   r‡   r‹   Ú
load_feeds‘  s(    (

þ

r?  )Úverboser„   c                 C  s8   t j}| dkrt j}n| dkr$t j}t j|ddd� d S )Nræ   r÷   z%%(asctime)s %(levelname)s %(message)sz%Y-%m-%d %H:%M:%S)ÚlevelÚformatZdatefmt)ÚloggingZWARNINGÚINFOÚDEBUGZbasicConfig)r@  rA  r‡   r‡   r‹   Úconfigure_logging¨  s    rF  zOptional[Sequence[str]])Úargvr„   c                 C  sÒ  t jdd�}|jdttdt› d�d� |jdttdt› d�d� |jd	td
dd� |jdtddd� |jdddd� |jdtdd� |jdddd� |jddtdd� |jdddd� |jdddd� |jd dd!d� |jd"t jd#d$d%� |jd&t jd#d'd%� |jd(tt	d)t	› d�d� |jd*t
d+d,� |jd-dd.d� |jd/d0d1d2d3d%� |jd4d5d6t› �d7� | | ¡}d2|j  k�r†d8k�s’n | d9¡ d2|j  k�r¬d8k�s¸n | d:¡ |jd2k �rÎ| d;¡ |S )<Nz?Collect and normalize public real-world cybersecurity incidents)r   z--outputzOutput JSON path (default: ú))rÕ  ÚdefaultÚhelpz--dayszJInclude source items from the last N days; 0 disables filtering (default: z--min-scoreg)\�Âõ(Ü?zNMinimum deterministic incident-selection score between 0 and 1 (default: 0.44)z--dedup-thresholdr  zBTitle token similarity threshold for deduplication (default: 0.52)z--feedrñ   z(Additional RSS/Atom feed URL; repeatable)ÚactionrJ  z--feeds-filez0JSON file containing additional feed definitions)rÕ  rJ  z--no-default-feedsÚ
store_truezDo not use built-in feedsz--input-jsonz(Manual source-item JSON file; repeatable)rK  rÕ  rJ  z	--replacez<Replace existing incidents instead of preserving old recordsz--validate-onlyz%Validate the existing output and exitz	--dry-runz*Build and validate but do not write outputz--prettyTzPretty-print JSON)rK  rI  rJ  z--backupz#Create .bak before replacing outputz	--timeoutz"HTTP timeout in seconds (default: z--user-agentzHTTP User-Agent)rI  rJ  z--strictz%Abort if any feed cannot be collectedz-vz	--verboser~  r   z,Increase logging verbosity; repeat for debugz	--versionÚversionz	%(prog)s )rK  rM  ræ   z#--min-score must be between 0 and 1z)--dedup-threshold must be between 0 and 1z--days cannot be negative)ÚargparseÚArgumentParserÚadd_argumentr   ÚDEFAULT_OUTPUTrÄ   ÚDEFAULT_DAYSrÃ  ZBooleanOptionalActionÚDEFAULT_TIMEOUTÚDEFAULT_USER_AGENTr  Ú
parse_argsÚ	min_scorer  Údedup_thresholdrÝ   )rG  rU  r9  r‡   r‡   r‹   rU  ±  s6    



rU  c                    sÞ  t | ƒ‰ tˆ jƒ ˆ jr¸ˆ j ¡ s4t dˆ j¡ dS ˆ jjddd��}t	 
|¡}W d   ƒ n1 sd0    Y  t|ƒ}|r”|D ]}t d|¡ q~dS tdˆ j› d	t|d
 ƒ› d�ƒ dS tƒ }tˆ ƒ}g }g }|D ]”}	z t|	ˆ jˆ jd�}
| |
¡ W qÒ t�yd } zT| |	 d|	 dd¡¡t|ƒdœ¡ t d|¡ ˆ j�rPW Y d }~ dS W Y d }~qÒd }~0 0 qÒdd„ tD ƒ}|D ]š}z$t|ˆ jˆ jˆ jd�}
| |
¡ W nn t�y } zT| | d| dd¡¡t|ƒdœ¡ t d|¡ ˆ j�rüW Y d }~ dS W Y d }~n
d }~0 0 �qzˆ j�p g D ]|}z(t|ƒ}| |¡ t dt|ƒ|¡ W nL t�yš } z2t d||¡ ˆ j�r†W Y d }~ dS W Y d }~n
d }~0 0 �q"h d£‰‡ ‡fdd„|D ƒ}t dt|ƒ¡ t dt|ƒ¡ g }g }|D ]à}t|ƒ\}}}t |ƒ\}}||dœ|¥|d< |�sN| ||j!|d œ¡ t "d!|d" #|¡|j!¡ �qèt$||ƒ}ˆ j%d#d$d%d&œ}| |ˆ j%¡}||k �r²| ||j!||d'gd(œ¡ t "d)||||j!¡ �qè| t&||||ƒ¡ �qèt'|ˆ j(d*�}ˆ j)�ræg }n
t*ˆ jƒ}|�rt+|||ƒ}n|}t|ƒt|ƒ|t|ƒt|ƒt|ƒt|ƒt|ƒt|ƒd+œ	}t,|||ƒ}t|ƒ}|�rv|D ]}t d|¡ �q^dS tt	j-|d, d-dd.�ƒ ˆ j.�r¤t /d/¡ dS t0ˆ j|ˆ j1ˆ j2d0� td1ˆ j› d2t|d
 ƒ› d3�ƒ dS )4NzOutput does not exist: %sr÷   rX  rÈ   rY  zValidation: %sræ   zValid: z (r  z	 records)r   rG  r   r   rw   )r  r  zFeed failed: %sr  c                 S  s   g | ]}|  d d¡r|‘qS )r   Frý  rû  r‡   r‡   r‹   r°   ;  s   þþzmain.<locals>.<listcomp>)rù   rú   rÝ   zAIVD source failed: %sz%Loaded %d manual source items from %szManual input failed for %s: %srg  c                   s6   g | ].}t |jƒ ¡ ˆv s.t|jp&|jˆ jƒr|‘qS r‡   )r�   r‚   r¬   rÞ   r   r€   rÝ   )r‰   r  ©r9  Zpersistent_record_typesr‡   r‹   r°   �  s   ÿ
þûzSource items collected: %dz#Source items inside date window: %d)ry  rw  rÁ  )r�  r{   rw  zRejected by gate %.3f [%s\]: %sz, gš™™™™™É?r‚  gš™™™™™Ù?)rÀ  ri  rj  rh  Zscore_below_threshold)r�  r{   r‚   Úrequired_scorerw  z&Rejected by score %.3f < %.3f [%s]: %s)r  )	Zfeeds_configuredZaivd_sources_configuredÚfeed_failuresZsource_items_collectedZsource_items_in_date_windowZ&candidate_records_before_deduplicationZrejected_source_itemsZ#fresh_incidents_after_deduplicationZexisting_incidents_loadedrA  F)r(  r)  zDry run: output not written)r!  r"  zWrote rý   r  )3rU  rF  r@  Zvalidate_onlyÚoutputr  rQ  r  r]  r^  r_  r   ÚprintrS  r¢   r?  rW  rù   rú   r=  r»   rñ   r  ry   ÚstrictÚAIVD_SOURCESrV  rÝ   Z
input_jsonre  rR  r{  rŽ  r{   ÚdebugrŽ   rÂ  rV  rô  r  rW  r    r  r  r  rÿ  Zdry_runrT  r8  r!  r"  ) rG  ra  rb  rI  r  rÄ  r:  Z	raw_itemsrZ  r  rï   r  Zenabled_aivd_sourcesrN  Z
input_pathr[  Zrecent_itemsZ
candidatesZrejectedr  Zgate_acceptedZgate_reasonsZgate_detailsr�  rz  r‚   Zscore_thresholdsrY  r  r
  r  r  r‡   rX  r‹   ÚmainÏ  s.   ÿ
þþ(ÿþ

ÿÿÿýÿþþøÿþ(þ	üÿþþøÿþ*ÿÿýý*þþþ	ÿÿÿþý
ýÿüþüþ
ÿûÿú	üÿþ	ÿýÿÿÿÿÿÿÿã&ý
ÿþýÿÿü

ÿÿr`  Ú__main__)N)N)r÷   )r¼  )r  )TT)N)N)�r;  Z
__future__r   rN  ZcalendarZdataclassesr˜   r—   Zemail.utilsrÒ   rÉ   r¤   r^  rC  r/  r·   r-  Úsysr.  rÐ   r¨   Zurllib.errorr³   Zurllib.parseZurllib.requestZxml.etree.ElementTreeZetreeZElementTreer$  Úcollectionsr   r   Úpathlibr   Útypingr   r   r   r	   r
   Zhtml.parserr   r  r  rQ  rS  rR  rT  Z	getLoggerrQ  r<  r^  r%   r”   ZGENERIC_INCIDENT_PATTERNSZNON_INCIDENT_PATTERNSZVULNERABILITY_ONLY_PATTERNSZSTRATEGIC_CONTENT_PATTERNSZORGANIZATIONAL_NEWS_PATTERNSZCONCRETE_INCIDENT_PATTERNSZOBSERVED_PATTERNSZSPECULATION_PATTERNSrà   rá   ró  ZURL_REr¦   rª   r¡  rð  r´  r­  rñ  rò  r‡  Z	dataclassrx   ÚRuntimeErrorr–   rœ   r¢   r�   r½   rÀ   rË   rÚ   rÛ   rÞ   râ   rï   rv  r„  r†  rr  r…  rs  rt  ru  rq  rê   rö   r  r  r  r(  r)  Ú
IGNORECASEÚVERBOSEr>  rL  rC  rF  rM  rV  rW  re  r{  rŽ  r”  r˜  r¢  r¯  rµ  r»  r¾  rÂ  rô  r÷  rø  r  r  r  r  r  r  r   r8  r?  rF  rU  r`  r‘   Ú
SystemExitr‡   r‡   r‡   r‹   Ú<module>   s  
úú	ú	øä*öôóÈ?	9 

ïð	á)÷ú
ü	
þÿÿÿÿ		,_
ø
û	&Ny ^$'$
[ *"0 *&"	ÿ   Pÿ