a
    §ëjÖ3  ã                   @   s  d Z ddlZddlZddlZddlZddlmZ g d¢Ze dej	¡Z
dd„ Zdd	„ Ze d
¡Ze d¡Zdd„ Zdd„ Zd,dd„ZdZde d de d de d de d gZd-dd„Zdd„ Zdd„ Zdd „ Zd!d"„ Zg d#¢Zd$Zd%d&„ Zd'd(„ Zd)d*„ Zed+k�reƒ  dS ).a  
Turn the two free-text analyses per repo (document_pipeline + intelligence_pipeline)
into structured JSON and merge it into dataset.json.

Existing fields (name, score, confidence, surfaces) are kept untouched, so the current
dashboard pages keep working. New fields are added next to them.

Usage:
  python3 enrich_dataset.py --raw /path/to/raw_jsons --dataset dataset.json --out dataset.rich.json

--raw can be a folder with one .json per repo (file name = repo name) or a single
.json file holding a list or a {name: {...}} mapping.
é    N)ÚPath)ZlowZmediumZhighZcriticalz\b(critical|high|medium|low)\bc                 C   s"   t  dd| ¡} t  dd| ¡ d¡S )Nz	\*\*|__|`Ú z\s+ú z :-	)ÚreÚsubÚstrip©Ús© r
   ú+/mnt/bigstor/www/usecases/enrich_dataset.pyÚclean   s    r   c              	   C   sœ   g dg   }}}|   ¡ D ]f}t d|¡}|rt|dus8|rP| |pBdd |¡f¡ tt dd| d¡¡ƒ ¡ g  }}q| |¡ q| |pŠdd |¡f¡ |S )z?Split markdown on headings -> list of (normalised_title, body).Nz^\s*#{1,6}\s+(.*\S)\s*$r   Ú
z^\d+[.)]\s*é   )	Ú
splitlinesr   ÚmatchÚappendÚjoinr   r   ÚgroupÚlower)ÚtextÚpartsÚtitleÚbufÚlineÚmr
   r
   r   Úsplit_sections   s    $r   z"^\s*(?:[-*\u2022]|\d+[.)])\s+(.*)$z0^\*\*(.+?)\*\*\s*:?\s*(.*)$|^(.{2,60}?):\s+(.*)$c                 C   sê   g }|   ¡ D ]Î}t |¡}|rœ| d¡ ¡ }t |¡}|rvt| d¡pN| d¡ƒ}t| d¡rh| d¡n| d¡ƒ}ndt|ƒ }}| | d¡|dœ¡ q| ¡ r|r| 	¡  
d¡s|d	 d
 d t|ƒ  ¡ |d	 d
< qdd„ |D ƒS )z]Bullet/numbered items -> [{'title','text'}]. Empty parent bullets (pure headers) are dropped.r   é   é   é   r   ú:)r   r   ú#éÿÿÿÿr   r   c                 S   s   g | ]}|d  r|‘qS )r   r
   ©Ú.0Úir
   r
   r   Ú
<listcomp><   ó    zitems.<locals>.<listcomp>)r   ÚITEM_REr   r   r   ÚBOLD_LEAD_REr   r   ÚrstripÚlstripÚ
startswith)ÚbodyÚoutr   r   ÚrawÚbr   r   r
   r
   r   Úitems+   s    

$&r0   c                 C   s2   i }t | ƒD ] }|d r|d ||d  ¡ < q|S )Nr   r   )r0   r   )r,   Údr$   r
   r
   r   Úkv>   s
    r2   Fc                 C   s.   t  | p
d¡}|r*|r|d n|d  ¡ S d S )Nr   r!   r   )ÚLEVEL_REÚfindallÚ
capitalize)r	   Zlastr   r
   r
   r   Úlevel_inE   s    r6   z(critical|high|medium|low)zÊ(?:adjust|rais|elevat|increas|lower|reduc|downgrad|upgrad|revis|chang|updat|set|rat|escalat)(?:ed|ing)?\s+(?:the\s+)?(?:(?:risk|threat|final)\s+)?(?:score\s+|rating\s+)?(?:up\s+|down\s+)?(?:to|as)\s+\**z\bzr(?:risk|threat|final)\s+(?:score|rating|level)\s*(?:is|=|:|of|remains|stays|stands at|should be)?\s*(?:now\s+)?\**z](?:verdict|conclusion|classified as|categori[sz]ed as|assessed as|rated)\s*:?\s*(?:as\s+)?\**z@(?:remains?|stays?|stay|agree[sd]? (?:with|on)[^.]{0,40}?)\s+\**c                 C   s–   g }t D ]6}t || pdtj¡D ]}| | ¡ | d¡f¡ q q|r~t dt d | pXdtj¡D ]}| | ¡ | d¡f¡ q`|r’t|ƒd  	¡ S dS )zUExplicit verdict of the second model, or None. Bare level words in prose never count.r   r   z\*\*N)
ÚVERDICT_PATTERNSr   ÚfinditerÚIr   Ústartr   ÚLÚmaxr5   )r   Ú
allow_boldÚhitsÚpatr   r
   r
   r   Ú
verdict_inQ   s     r@   c                    s0   | D ]&\‰ }t ‡ fdd„|D ƒƒr|  S qd S )Nc                 3   s   | ]}|ˆ v V  qd S ©Nr
   )r#   Úk©Útr
   r   Ú	<genexpr>^   r&   zpick.<locals>.<genexpr>)Úany)ZsectionsÚkeysr/   r
   rC   r   Úpick\   s    
rH   c                 C   s   dd„ t  d| pd¡D ƒS )Nc                 S   s   g | ]}t |ƒrt |ƒ‘qS r
   )r   ©r#   Úxr
   r
   r   r%   c   r&   zcsv.<locals>.<listcomp>z,|;| and (?=[A-Z])r   )r   Úsplitr   r
   r
   r   Úcsvb   s    rL   c                 C   s   dd„ | D ƒS )Nc                 S   s$   g | ]}|d  p|d dd… ‘qS )r   r   Né<   r
   r"   r
   r
   r   r%   f   r&   ztitles_only.<locals>.<listcomp>r
   )Úlstr
   r
   r   Útitles_onlye   s    rO   c                 C   s  i }t | pdƒt |pdƒ }}tt|ddƒp.dƒ}| d¡|d< t| d¡ƒ|d< t| d¡pf| d	¡ƒ|d	< t|d
dƒp|d}tdd„ | ¡ D ƒ|ƒ}t|ƒp¤t|ƒ|d< t 	d|tj
tjB ¡}|rÒt| d¡ƒnt|ƒ|d< tt|dƒpìdƒ|d< dd„ tt|dƒ�p
dƒD ƒ|d< dd„ tt|dƒ�p,dƒD ƒ|d< dd„ tt|ddƒ�pPdƒD ƒ|d< tt|ddƒ�pndƒ|d< tt|d ƒ�p†dƒ|d!< tt|d"ƒ�pždƒ|d#< |d  d$d„ tt|dƒ�pÄdƒD ƒ7  < t|d%d&d'd(ƒ�päd}	|	�røt|	d)d*�nt|ƒ}
|
|d+< |
�rt|	ƒnd |d,< |
�p*|d |d-< |
�r:d.nd/|d0< |
d u �pV|
|d k|d1< |d |d-  }}d2d3„ }t|�oŠ|�oŠ||kƒ|d4< t|�o®|�o®||ƒ||ƒkƒ|d5< t|�oÒ|�oÒ||ƒ||ƒk ƒ|d6< t|d ƒt|d! ƒ t|d# ƒ |d7< t|d ƒ|d8< |S )9Nr   zprotocol classificationZclassificationzprimary focusÚprimary_focuszsecondary focusÚsecondary_focusztechnologies usedÚtechnologieszthreat scorez
risk scorec                 s   s"   | ]}t  d |t j¡r|V  qdS )zthreat score|scoreN©r   Úsearchr9   )r#   Úlr
   r
   r   rE   u   r&   zparse_repo.<locals>.<genexpr>Úscore_initialz justification\s*:?\*{0,2}\s*(.+)r   ZjustificationZvulnerabilitÚvulnerabilitiesc                 S   s$   g | ]}t  d |d t j¡r|‘qS )zchallenge|scalab|integrationr   rS   r"   r
   r
   r   r%   |   s   ÿzparse_repo.<locals>.<listcomp>zdeep analysisZimplementation_challengesc                 S   s   g | ]}t |d d�‘qS )Zinitial©Úsource©Údictr"   r
   r
   r   r%      r&   Úfuzzingc                 S   s4   g | ],}|d  |d  r"|d r"dnd |d  ‘qS )r   r   ú: r   r
   r"   r
   r
   r   r%   €   s   ÿz
next stepsZ	recommendÚ
next_stepszadditional insightsZinsightszhidden attack surfaceZhidden_attack_surfacesZweaknessZ
weaknessesc                 S   s   g | ]}t |d d�‘qS )ZadvancedrX   rZ   r"   r
   r
   r   r%   ‡   r&   zfinal risk scorezfinal scorezrisk score justificationÚverdictT)r=   Úscore_verdictZfinal_justificationÚscore_finalZintelligenceZdocumentZscore_sourceZsecond_agreedc                 S   s   t  |  ¡ ¡S rA   )ÚLEVELSÚindexr   )rJ   r
   r
   r   Ú<lambda>”   r&   zparse_repo.<locals>.<lambda>Zscore_changedÚ	escalatedÚ
downgradedZthreat_countZfuzzing_count)r   r2   rH   ÚgetrL   Únextr   r6   r   rT   r9   ÚSr   r   r0   r@   ÚboolÚlen)ÚdocÚintelr-   ZdsZintsÚclsZsbZsc_lineZjmZfbr_   r$   ÚfZrankr
   r
   r   Ú
parse_repoi   sL    ÿ
"ÿ
*$$(rp   ))zCAN Busz?\bCAN\b|\bCAN[- ]?FD\b|\bCAN[- ]?bus\b|\bSocketCAN\b|\bCANbus\bT)zUDS Diagnosticsz(\bUDS\b|ISO[- ]?14229|unified diagnosticF)zISO-TPz"ISO[- ]?TP|ISO[- ]?15765|\bISOTP\bF)ZDoIPz\bDoIP\b|ISO[- ]?13400F)zSOME/IPzSOME[/ -]?IP|\bSOMEIP\bF)zJ1939 / Heavy Vehiclez!J1939|\bISOBUS\b|heavy[- ]vehicleF)zOBD-II / J2534z)\bOBD[- ]?(?:II|2)?\b|J2534|pass[- ]?thruF)zLIN / FlexRayz\bLIN\b|\bFlexRay\bT)zECU Firmwarez3\bECUs?\b|firmware|bootloader|\bAUTOSAR\b|\bSTM32\bF)zFuzzing EngineZfuzzF)zKeyFob / RFzTkey[- ]?fob|\bRKE\b|\bPEPS\b|immobili[sz]er|\bRF\b|\bSDR\b|software[- ]defined radioF)zBluetooth / Wi-Fiz-\bBluetooth\b|\bBLE\b|blue[- ]?tap|\bWi-?Fi\bF)zCellular / V2Xz<\bV2X\b|\bV2V\b|\bV2I\b|\bDSRC\b|\bLTE\b|\b[3-6]G\b|cellularF)zTelematics / Routingz/telematics|\brouting\b|\bGPS\b|\bGNSS\b|\bTCU\bF)ZInfotainmentz3infotainment|android auto|carplay|\bIVI\b|head unitF)zOTA Updatesz%\bOTA\b|over[- ]the[- ]air|\bUptane\bF)zHardware Securityz4\bHSM\b|secure element|secure boot|\bTPM\b|trustzoneF)zDiagnostic GatewayZgatewayF)zAutomotive AIz;\bAI\b|\bML\b|\bLLM\b|machine learning|deep learning|neuralF)zAutomotive BlockchainZ
blockchainF)zDashboard Clusterz0instrument cluster|dashboard cluster|speedometerFzGeneral Automotive Surfacec           	      C   s    |   dd¡d }t dd|¡}d | d¡p.dg| d¡p<g ¢| d	¡pJg ¢¡}g }tD ]<\}}}t |||rrd
ntj¡sŠt ||tj¡rX| |¡ qX|pžt	gS )zKAttack surfaces from the analysed content, not from substrings of the name.Ú__r   r!   z[_\-.]+r   rP   r   rQ   rR   r   )
rK   r   r   r   rg   ÚSURFACE_RULESrT   r9   r   ÚGENERIC_SURFACE)	ÚnameÚrichÚrepoZ	name_textr   ÚfoundZlabelr?   Zcsr
   r
   r   Údetect_surfacesº   s     ÿ(rx   c                 C   sú   t | ƒi  }}| ¡ r²t| d¡ƒD ]Š}zt |jdd�¡}W nL tyŠ } z4td|j	› d|› �t
jd� W Y d }~q$W Y d }~n
d }~0 0 ||t|tƒr¨| d|j¡n|j< q$nDt |jdd�¡}t|tƒrÞdd	„ |D ƒ}nd
|v rò|j|i}n|}|S )Nz*.jsonúutf-8©Úencodingz! cannot read r]   )Úfilert   c                 S   s   i | ]}|d  |“qS ©rt   r
   rI   r
   r
   r   Ú
<dictcomp>Ó   r&   zload_raw.<locals>.<dictcomp>Údocument_pipeline)r   Úis_dirÚsortedÚglobÚjsonÚloadsÚ	read_textÚ	ExceptionÚprintrt   ÚsysÚstderrÚ
isinstancer[   rg   ÚstemÚlist)ÚpathÚpÚresro   r1   Úer
   r
   r   Úload_rawÇ   s    >&
r‘   c               
      sZ  t  ¡ } | jddd� | jddd� | jddd� | jd	d
dd� |  ¡ }t|jƒ}t|jƒ ¡ rzt	 
t|jƒjdd�¡ng }t|tƒrŒ|n
| dg ¡}dd„ |D ƒ}g g g   }}}| ¡ D ]€\}}	||v �rt||  d¡||  d¡ƒ}
| i |	¥d| d¡d i¥|
¥¡ q¾| i |	¥d| d¡d i¥¡ | |¡ q¾| ¡ | ¡  D ]\}t||  d¡||  d¡ƒ}
| || d¡d |
d �p”|
d dœ|
¥¡ | |¡ �qP|j�rÚ|D ]}| d¡�rº|d |d< �qºt|jƒjt	j|ddd�dd� dd„ |D ƒ‰ ‡ fd d!„}tt|ƒ› d"|j› �ƒ td#tˆ ƒ› d$t|ƒ› d%t|ƒ› �ƒ d&D ]:}||ƒ}|�rVtd'|› d(t|ƒ› d)|d d*… › �ƒ �qVtˆ ƒ�pžd+}td,d-„ ˆ D ƒƒ}td.d-„ ˆ D ƒƒ}td/d-„ ˆ D ƒƒ}td0|› d1|› d2|| › �ƒ td3|› d4|› d5|| | › d6�ƒ |j�sVd7d„ ˆ D ƒ}|�rVtd8t|ƒ› d9|d d*… › d:�ƒ d S );Nz--rawT)Úrequiredz	--datasetzdataset.json)Údefaultz--outzdataset.rich.jsonz--sync-scoreÚ
store_truezWoverwrite 'score' with the final score (second model's verdict, else the first model's))ÚactionÚhelpry   rz   Zreposc                 S   s   i | ]}|d  |“qS r}   r
   )r#   Úrr
   r
   r   r~   æ   r&   zmain.<locals>.<dictcomp>r   Úintelligence_pipelineÚownerrq   r   ra   rV   )rt   r™   Úscorerš   r   F)ÚindentÚensure_asciic                 S   s   g | ]}d |v r|‘qS )rV   r
   ©r#   r   r
   r
   r   r%   ú   r&   zmain.<locals>.<listcomp>c                    s   ‡ fdd„ˆD ƒS )Nc                    s   g | ]}|  ˆ ¡s|d  ‘qS r}   ©rg   r�   ©rB   r
   r   r%   û   r&   z&main.<locals>.miss.<locals>.<listcomp>r
   rŸ   ©Z	rich_rowsrŸ   r   Úmissû   r&   zmain.<locals>.missz repos written to z  parsed from raw: z   without raw file: z   only in raw: )rV   ra   rW   r\   r^   rP   z
  missing r]   z  e.g. r   r   c                 s   s   | ]}|  d ¡rdV  qdS )r`   r   Nrž   r�   r
   r
   r   rE     r&   zmain.<locals>.<genexpr>c                 s   s   | ]}|  d ¡rdV  qdS )re   r   Nrž   r�   r
   r
   r   rE     r&   c                 s   s   | ]}|  d ¡rdV  qdS )rf   r   Nrž   r�   r
   r
   r   rE     r&   z"  second model gave a verdict for ú/z; no verdict (agrees) for z  of those verdicts: z raised the score, z lowered it, z confirmed itc                 S   s.   g | ]&}|  d ¡r|d |d  kr|d ‘qS )ra   rš   rt   rž   r�   r
   r
   r   r%     r&   z=  ! 'score' in dataset.json differs from the final score for z repos, e.g. z  (use --sync-score to fix))ÚargparseÚArgumentParserÚadd_argumentÚ
parse_argsr‘   r.   r   ZdatasetÚexistsrƒ   r„   r…   rŠ   rŒ   rg   r0   rp   r   rK   rG   Z
sync_scorer-   Ú
write_textÚdumpsr‡   rk   Úsum)ÚapÚar.   ÚbaseZby_nameZmergedZno_rawÚnewrt   r—   ru   r   r¡   rB   ÚnÚgaveÚescZdownÚoffr
   r    r   ÚmainÚ   s\    ÿ
*
&,ÿÿ &0$r³   Ú__main__)F)F)Ú__doc__r£   rƒ   r   rˆ   Úpathlibr   rb   Úcompiler9   r3   r   r   r'   r(   r0   r2   r6   r;   r7   r@   rH   rL   rO   rp   rr   rs   rx   r‘   r³   Ú__name__r
   r
   r
   r   Ú<module>   s:    






ü
82
