a
    ¡˜ëj¯ƒ  ã                	   @  s.  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZmZ d dlZd dlZd dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, e -d¡Z.e /¡ Z0de0_1e 2d¡Z3e0 4e3¡ ddd„ e.j5D ƒv�r@e. 6e0¡ e. 7ej8¡ g d¢Z9ddddddœZ:ddgZ;eG dd„ de	ƒƒZ<eG dd„ de<ƒƒZ=eG dd „ d e<ƒƒZ>eG d!d"„ d"e<ƒƒZ?d9d#d$d#d#d%d%d&d'œd(d)„Z@d:d-d.d#d%d#d#d/d0œd1d2„ZAeG d3d4„ d4e<e	ƒƒZBG d5d6„ d6eBƒZCG d7d8„ d8eBƒZDdS );é    )ÚannotationsN)ÚABCÚabstractmethod)Ú	dataclass©ÚPath)ÚListÚOptionalÚUnion)Útqdm)Ú$calculate_element_type_percent_matchÚget_element_type_frequency)ÚObjectDetectionEvalProcessor)ÚTableEvalProcessor)Úcalculate_accuracyÚcalculate_percent_missing_text)
Ú_countÚ_displayÚ_format_grouping_outputÚ_meanÚ_prepare_output_cctÚ_pstdevÚ_read_text_fileÚ_rename_aggregated_columnsÚ_stdevÚ_write_to_filezunstructured.evalZeval_log_handlerz:%(asctime)s %(processName)-10s %(levelname)-8s %(message)sc                 C  s   g | ]
}|j ‘qS © )Úname)Ú.0Úhr   r   ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/metrics/evaluate.pyÚ
<listcomp>0   ó    r!   )ÚmetricÚaverageÚ	sample_sdÚpopulation_sdÚcountr#   r$   r%   r&   r'   )Úindexr   r   r   r   ÚjsonÚtxtc                   @  sÞ   e Zd ZU dZded< ded< dd„ Zeedd„ ƒƒZeed	d
„ ƒƒZ	edddœdd„ƒZ
d.ddd dœdd„Zd/ddddddœdd„Zedd„ ƒZedd œd!d"„ƒZd#ddd$œd%d&„Zd'd(d)œd*d+„Zed'd(d)œd,d-„ƒZdS )0ÚBaseMetricsCalculatorzÝFoundation class for specialized metrics calculators.

    It provides a common interface for calculating metrics based on outputs and ground truths.
    Those can be provided as either directories or lists of files.
    z
str | PathÚdocuments_dirÚground_truths_dirc                   s\   t ˆ jƒ ¡ ˆ _t ˆ jƒ ¡ ˆ _‡ fdd„ˆ j d¡D ƒˆ _‡ fdd„ˆ j d¡D ƒˆ _dS )z/Discover all files in the provided directories.c                   s    g | ]}|  ¡ r| ˆ j¡‘qS r   ©Úis_fileÚrelative_tor,   ©r   Úpath©Úselfr   r    r!   Q   s   þz7BaseMetricsCalculator.__post_init__.<locals>.<listcomp>Ú*c                   s    g | ]}|  ¡ r| ˆ j¡‘qS r   )r/   r0   r-   r1   r3   r   r    r!   V   s   þN)r   r,   Úresolver-   ÚglobÚ_document_pathsÚ_ground_truth_pathsr3   r   r3   r    Ú__post_init__K   s    

þ

þz#BaseMetricsCalculator.__post_init__c                 C  s   dS )z3Default name for the per-document metrics TSV file.Nr   r3   r   r   r    Údefault_tsv_name\   s    z&BaseMetricsCalculator.default_tsv_namec                 C  s   dS )z1Default name for the aggregated metrics TSV file.Nr   r3   r   r   r    Údefault_agg_tsv_namea   s    z*BaseMetricsCalculator.default_agg_tsv_nameÚlistú!tuple[pd.DataFrame, pd.DataFrame])ÚrowsÚreturnc                 C  s   dS )zåGenerates pandas DataFrames from the list of rows.

        The first DF (index 0) is a dataframe containing metrics per file.
        The second DF (index 1) is a dataframe containing the aggregated
            metrics.
        Nr   )r4   r?   r   r   r    Ú_generate_dataframesf   s    z*BaseMetricsCalculator._generate_dataframesNzOptional[list[str | Path]])Údocument_pathsÚground_truth_pathsr@   c                 C  s,   |rdd„ |D ƒ| _ |r(dd„ |D ƒ| _| S )z/Overrides the default list of files to process.c                 S  s   g | ]}t |ƒ‘qS r   r   ©r   Úpr   r   r    r!   v   r"   z2BaseMetricsCalculator.on_files.<locals>.<listcomp>c                 S  s   g | ]}t |ƒ‘qS r   r   rD   r   r   r    r!   y   r"   )r8   r9   )r4   rB   rC   r   r   r    Úon_fileso   s
    zBaseMetricsCalculator.on_filesTú%Optional[concurrent.futures.Executor]úOptional[str | Path]Úboolúpd.DataFrame©ÚexecutorÚ
export_dirÚvisualize_progressÚdisplay_agg_dfr@   c                 C  sb   |du r|   ¡ }|  ||¡}|  |¡\}}|durNt|| j|ƒ t|| j|ƒ |du r^t|ƒ |S )aò  Calculates metrics for each document using the provided executor.

        * Optionally, the results can be exported and displayed.
        * It loops through the list of structured output from all of `documents_dir` or
        selected files from `document_paths`, and compares them with gold-standard
        of the same file name under `ground_truths_dir` or selected files from `ground_truth_paths`.

        Args:
            executor: concurrent.futures.Executor instance
            export_dir: directory to export the results
            visualize_progress: whether to display progress bar
            display_agg_df: whether to display the aggregated results

        Returns:
            Metrics for each document as a pandas DataFrame
        NT)Ú_default_executorÚ_process_all_documentsrA   r   r;   r<   r   )r4   rL   rM   rN   rO   r?   ÚdfÚagg_dfr   r   r    Ú	calculate}   s    zBaseMetricsCalculator.calculatec                 C  s6   t tj dt ¡ ¡ƒ}t d|› d�¡ |  ¡ |d�S )NZMAX_PROCESSESzConfiguring a pool of z$ processors for parallel processing.)Úmax_workers)ÚintÚosÚenvironÚgetÚ	cpu_countÚloggerÚinfoÚ_get_executor_class)ÚclsZmax_processorsr   r   r    rP   ¡   s    z'BaseMetricsCalculator._default_executorzZtype[concurrent.futures.ThreadPoolExecutor] | type[concurrent.futures.ProcessPoolExecutor]©r@   c                 C  s   t jjS ©N)Ú
concurrentZfuturesÚProcessPoolExecutor)r^   r   r   r    r]   §   s    z)BaseMetricsCalculator._get_executor_classzconcurrent.futures.Executor)rL   rN   r@   c                 C  sV   |�> dd„ t | | j| j¡t| jƒd| d�D ƒW  d  ƒ S 1 sH0    Y  dS )z‚Triggers processing of all documents using the provided executor.

        Failures are omitted from the returned result.
        c                 S  s   g | ]}|d ur|‘qS r`   r   )r   Úrowr   r   r    r!   µ   s   ùz@BaseMetricsCalculator._process_all_documents.<locals>.<listcomp>F)ÚtotalZleaveÚdisableN)r   ÚmapÚ_try_process_documentr8   Úlen)r4   rL   rN   r   r   r    rQ   ­   s    üþz,BaseMetricsCalculator._process_all_documentsr   úOptional[list]©Údocr@   c              
   C  s^   t  d|› �¡ z|  |¡W S  tyX } z$t  d|› d|› �¡ W Y d}~dS d}~0 0 dS )z3Safe wrapper around the document processing method.zProcessing úFailed to process document ú: N)r[   r\   Ú_process_documentÚ	ExceptionÚerror)r4   rk   Úer   r   r    rg   À   s    z+BaseMetricsCalculator._try_process_documentc                 C  s   dS )z=Should return all metadata and metrics for a single document.Nr   )r4   rk   r   r   r    rn   É   s    z'BaseMetricsCalculator._process_document)NN)NNTT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r:   Úpropertyr   r;   r<   rA   rF   rT   ÚclassmethodrP   r]   rQ   rg   rn   r   r   r   r    r+   @   s8   

  ý    û$
	r+   c                      s‚   e Zd ZU dZdZded< dZded< dZded< ‡ fd	d
„Ze	dd„ ƒZ
e	dd„ ƒZe	dd„ ƒZdddœdd„Zdd„ Z‡  ZS )ÚTableStructureMetricsCalculatoraH  Calculates the following metrics for tables:
        - tables found accuracy
        - table-level accuracy
        - element in column index accuracy
        - element in row index accuracy
        - element's column content accuracy
        - element's row content accuracy
    It also calculates the aggregated accuracy.
    NzOptional[float]ÚcutoffTrI   Úweighted_averageÚinclude_false_positivesc                   s   t ƒ  ¡  d S r`   )Úsuperr:   r3   ©Ú	__class__r   r    r:   Þ   s    z-TableStructureMetricsCalculator.__post_init__c                 C  s   g d¢S )N)
Útotal_tablesÚtable_level_accZtable_detection_recallZtable_detection_precisionZtable_detection_f1Zcomposite_structure_accZelement_col_level_index_accZelement_row_level_index_accZelement_col_level_content_accZelement_row_level_content_accr   r3   r   r   r    Úsupported_metric_namesá   s    z6TableStructureMetricsCalculator.supported_metric_namesc                 C  s   dS )Nz%all-docs-table-structure-accuracy.tsvr   r3   r   r   r    r;   ð   s    z0TableStructureMetricsCalculator.default_tsv_namec                 C  s   dS )Nz&aggregate-table-structure-accuracy.tsvr   r3   r   r   r    r<   ô   s    z4TableStructureMetricsCalculator.default_agg_tsv_namer   ri   rj   c           
        sØ   t |ƒ}|j}t |ƒj}|d }t|jƒdkr8|jd nd }|| jv rJd S | j| }| ¡ srt 	d|› d�¡ d S | j
| }| ¡ sšt 	d|› d�¡ d S tj||| jdd�}	|	 ¡ ‰ |||ˆ jg‡ fd	d
„| jD ƒ S )Nú.jsoné   éþÿÿÿúPrediction file ú does not exist, skippingúGround truth file Úhtml)Úprediction_fileÚground_truth_filerz   Zsource_typec                   s   g | ]}t ˆ |ƒ‘qS r   ©Úgetattr©r   r#   ©Zreport_from_htmlr   r    r!     r"   zETableStructureMetricsCalculator._process_document.<locals>.<listcomp>)r   ÚstemÚsuffixrh   Úpartsr9   r,   Úexistsr[   Úwarningr-   r   Úfrom_json_filesrz   Zprocess_fileÚtotal_predicted_tablesr‚   )
r4   rk   Zdoc_pathZout_filenameÚdoctypeÚsrc_gt_filenameÚ	connectorrŠ   r‹   Zprocessor_from_text_as_htmlr   r�   r    rn   ø   s:    



üüûz1TableStructureMetricsCalculator._process_documentc           
      C  s”  g d¢| j  }tj||d�}|d |d< | jrNd|d |j d¡|j d¡@ < ||d dk }| jsz|d  	dd„ ¡|d< |j
rœt d	d
„ | j D ƒ¡ ¡ }näi }| j D ]Æ}|||  ¡  }||  tttg¡ ¡ }	| d¡rî||  ¡ |	d< nR| d¡�rt tj|| |d d�d¡|	d< n"t tj|| |d d�d¡|	d< |	j
�rdtjg d¢g d¢d�||< q¦|	||< q¦t |¡ ¡  ¡ }|jtd�}||fS )N)Úfilenamer—   r™   r–   ©Úcolumnsr€   Z_table_weightsr„   r   c                 S  s   | dkrdS dS )Nr   r„   r   )Ztable_weightr   r   r    Ú<lambda>/  r"   zFTableStructureMetricsCalculator._generate_dataframes.<locals>.<lambda>c                 S  s   g | ]}|d d d dg‘qS )Nr   r   rŽ   r   r   r    r!   4  r"   zHTableStructureMetricsCalculator._generate_dataframes.<locals>.<listcomp>r   r�   )Úweightsé   ©NNNr   ©r   r   r   r   ©Údatar(   )r‚   ÚpdÚ	DataFramer|   r€   Úeqr–   Úgtr{   ÚapplyÚemptyÚreset_indexÚnotnullÚaggr   r   r   Ú	transposeÚ
startswithÚmeanÚnpÚroundr$   ÚSeriesÚrenameÚAGG_HEADERS_MAPPING)
r4   r?   ÚheadersrR   Zhas_tables_dfrS   Úelement_metrics_resultsr#   Ú	metric_dfÚ
agg_metricr   r   r    rA     sN    û ÿÿ


þ
þÿ
z4TableStructureMetricsCalculator._generate_dataframes)rr   rs   rt   ru   rz   rv   r{   r|   r:   rw   r‚   r;   r<   rn   rA   Ú__classcell__r   r   r~   r    ry   Î   s   




"ry   c                      s¶   e Zd ZU dZdZded< dZded< dZd	ed
< ‡ fdd„Ze	d	dœdd„ƒZ
e	d	dœdd„ƒZd&ddddddœ‡ fdd„Zdd„ Zdddœdd „Zdd!dœd"d#„Zd$d%„ Z‡  ZS )'ÚTextExtractionMetricsCalculatorzŸCalculates text accuracy and percent missing between document and ground truth texts.

    It also calculates the aggregated accuracy and percent missing.
    NúOptional[str]Úgroup_by)r„   r„   r„   ztuple[int, int, int]rž   r)   ÚstrÚdocument_typec                   s   t ƒ  ¡  |  ¡  d S r`   )r}   r:   Ú_validate_inputsr3   r~   r   r    r:   _  s    
z-TextExtractionMetricsCalculator.__post_init__r_   c                 C  s   dS )Nzall-docs-cct.tsvr   r3   r   r   r    r;   c  s    z0TextExtractionMetricsCalculator.default_tsv_namec                 C  s   dS )Nzaggregate-scores-cct.tsvr   r3   r   r   r    r<   g  s    z4TextExtractionMetricsCalculator.default_agg_tsv_nameTrG   rH   rI   rJ   rK   c                   s6   t ƒ j||||d�}|dur2| jr2t| j||dƒ |S )ú0See the parent class for the method's docstring.©rL   rM   rN   rO   NÚtext_extraction©r}   rT   r¼   Úget_mean_grouping©r4   rL   rM   rN   rO   rR   r~   r   r    rT   k  s    üz)TextExtractionMetricsCalculator.calculatec              	     sÀ   ˆ j st d¡ t d¡ ˆ jtvr6tdˆ j› d�ƒ‚ˆ j D ]\}z|jd  W n( t	yv   t 
d|› d�¡ Y q<Y n0 |jd dˆ j› �kr<t d	¡ q<t‡ fd
d„ˆ j D ƒƒs¼t d	¡ d S )Nz;No output files to calculate to edit distances for, exitingr   ztSpecified file type under `documents_dir` or `output_list` should be one of `json` or `txt`. The given file type is z
, exiting.éÿÿÿÿúFile z! does not have a suffix, skippingÚ.zpThe directory contains file type inconsistent with the given input. Please note that some files will be skipped.c                 3  s$   | ]}|j d  dˆ j› �kV  qdS )rÆ   rÈ   N)Úsuffixesr¾   r1   r3   r   r    Ú	<genexpr>’  r"   zCTextExtractionMetricsCalculator._validate_inputs.<locals>.<genexpr>)r8   r[   r\   ÚsysÚexitr¾   ÚOUTPUT_TYPE_OPTIONSÚ
ValueErrorrÉ   Ú
IndexErrorrp   r”   Úall)r4   r2   r   r3   r    r¿   ~  s.    


ÿÿ

ÿÿz0TextExtractionMetricsCalculator._validate_inputsr   ri   rj   c           	      C  sž   |j }|jd }t|jƒdkr(|jd nd }|  |¡\}}dt| ¡ ƒt| ¡ ƒ   k rbdk r|n ntt||| jƒdƒ}nd}tt	||ƒdƒ}|||||gS )Nr…   r„   r   g      à?g       @rŸ   g{®Gáz„?)
r�   rÉ   rh   r’   Ú	_get_cctsÚencoder±   r   rž   r   )	r4   rk   rš   r—   r™   Ú
output_cctÚ
source_cctÚaccuracyZpercent_missingr   r   r    rn   ˜  s    
,z1TextExtractionMetricsCalculator._process_documentztuple[str, str]c                 C  s0   t | j| | jd�}t| j| d¡ ƒ}||fS )N)ZdocpathÚoutput_typeú.txt)r   r,   r¾   r   r-   Úwith_suffix)r4   rk   rÓ   rÔ   r   r   r    rÑ   ¨  s
    ÿz)TextExtractionMetricsCalculator._get_cctsc                 C  s˜   g d¢}t j||d�}|dg  ttttg¡ ¡ }|dg  ttttg¡ ¡ }|jd dkrx|jd dkrxt jt	d�}nt  
||f¡ ¡ }t	|_||fS )N)rš   r—   r™   úcct-accuracyúcct-%missingr›   rÙ   rÚ   r„   r   )r¤   r¥   r¬   r   r   r   r   r­   ÚshapeÚAGG_HEADERSÚconcatrª   rœ   )r4   r?   rµ   rR   ÚaccZmissrS   r   r   r    rA   °  s    z4TextExtractionMetricsCalculator._generate_dataframes)NNTT)rr   rs   rt   ru   r¼   rv   rž   r¾   r:   rw   r;   r<   rT   r¿   rn   rÑ   rA   r¹   r   r   r~   r    rº   T  s$   
    ûrº   c                      sz   e Zd ZU dZdZded< dddd	d	d
dœ‡ fdd„Zeddœdd„ƒZeddœdd„ƒZ	dddœdd„Z
dd„ Z‡  ZS )ÚElementTypeMetricsCalculatorz„
    Calculates element type frequency accuracy, percent missing and
    aggregated accuracy between document and ground truth.
    Nr»   r¼   TFrG   rH   rI   rJ   rK   c                   s6   t ƒ j||||d�}|dur2| jr2t| j||dƒ |S )rÀ   rÁ   NÚelement_typerÃ   rÅ   r~   r   r    rT   È  s    üz&ElementTypeMetricsCalculator.calculater½   r_   c                 C  s   dS )Nz#all-docs-element-type-frequency.tsvr   r3   r   r   r    r;   Û  s    z-ElementTypeMetricsCalculator.default_tsv_namec                 C  s   dS )Nz!aggregate-scores-element-type.tsvr   r3   r   r   r    r<   ß  s    z1ElementTypeMetricsCalculator.default_agg_tsv_namer   ri   rj   c                 C  sr   |j }|jd }t|jƒdkr(|jd nd }tt| j| ƒƒ}tt| j| d¡ ƒƒ}t	t
||ƒdƒ}||||gS )Nr…   r„   r   rƒ   rŸ   )r�   rÉ   rh   r’   r   r   r,   r-   rØ   r±   r   )r4   rk   rš   r—   r™   ÚoutputÚsourcerÕ   r   r   r    rn   ã  s    
ÿz.ElementTypeMetricsCalculator._process_documentc                 C  s`   g d¢}t j||d�}|jr0t  g d¢¡ ¡ }n"| dttttgi¡ ¡ }| 	¡ }t
|_||fS )N)rš   r—   r™   úelement-type-accuracyr›   )rã   NNNr   rã   )r¤   r¥   r©   r­   r¬   r   r   r   r   rª   rÜ   rœ   )r4   r?   rµ   rR   rS   r   r   r    rA   ï  s    z1ElementTypeMetricsCalculator._generate_dataframes)NNTF)rr   rs   rt   ru   r¼   rv   rT   rw   r;   r<   rn   rA   r¹   r   r   r~   r    rß   ¿  s   
    ûrß   r½   zUnion[pd.DataFrame, str]r»   ÚNone)r¼   Ú
data_inputrM   Ú	eval_nameÚagg_nameÚexport_filenamer@   c           
      C  s.  | dvr| dkrt dƒ‚|dkr.ddg}d}n:|dkrBd	g}d
}n&|dkrXddg}d}nt d|› d�ƒ‚t|tƒrètj |¡sŽtd|› d�ƒ‚| d¡r¨tj	|dd�}qì| d¡rÂtj	|dd�}qì| d¡rÞtj	|ddd�}qìt dƒ‚n|}|j
rütdƒ‚n8| dk�r4| |jv�s$||   ¡  ¡ �r4td| › d�ƒ‚g }| �rz| dk�rz|D ],}	| t| | ¡ |	ttttgi¡ƒ¡ �qL| dk�r¾d|d < |D ],}	| t| d ¡ |	ttttgi¡ƒ¡ �q�t|Ž }d |j d¡v �rè|jd d!dd"�}|�r| d¡�s|d }t|||ƒ nt|d#| › d$|› d�|ƒ dS )%aÍ  Aggregates accuracy and missing metrics by column name 'doctype' or 'connector',
    or 'all' for all rows. Export to TSV.
    If `all`, passing export_name is recommended.

    Args:
        group_by (str): Grouping category ('doctype' or 'connector' or 'all').
        data_input (Union[pd.DataFrame, str]): DataFrame or path to a CSV/TSV file.
        export_dir (str): Directory for the exported TSV file.
        eval_name (str): Evaluated metric ('text_extraction' or 'element_type').
        agg_name (str, optional): String to use with export filename. Default is `cct` for
            group_by `text_extraction` and `element-type` for `element_type`
        export_name (str, optional): Export filename.
    )r—   r™   rÐ   z<Invalid grouping category. Returning a non-group evaluation.rÂ   rÙ   rÚ   Zcctrà   rã   zelement-typeZobject_detectionÚf1_scoreÚm_apzobject-detectionzUnknown metric for eval zE. Expected `text_extraction` or `element_type` or `table_extraction`.rÇ   ú not found.ú.csvN©Úheaderú.tsvú	©Úsepr×   ©rò   rî   ú#Please provide a .csv or .tsv file.zData is empty. Exiting.zData cannot be aggregated by `z6`. Check if it's empty or the column is missing/empty.r   Zgrouping_keyr„   )ZaxisÚlevelzall-z-agg-)rÎ   Ú
isinstancer½   rW   r2   r“   ÚFileNotFoundErrorÚendswithr¤   Úread_csvr©   Ú
SystemExitrœ   ZisnullrÐ   Úappendr   Úgroupbyr¬   r   r   r   r   r   Zget_level_valuesZdropr   )
r¼   rå   rM   ræ   rç   rè   Z
agg_fieldsrR   Z
grouped_dfÚfieldr   r   r    rÄ   ý  sn    
ÿ





(
ÿÿÿ
ÿÿrÄ   rš   ÚmetricsÚfilezUnion[str, pd.DataFrame]zUnion[str, List[str]]zOptional[pd.DataFrame])rå   Úfilter_listÚ	filter_byrè   rM   Úreturn_typer@   c           	      C  sÀ  t | tƒr€tj | ¡s&td| › d�ƒ‚|  d¡r@tj| dd�}q„|  d¡rZtj| dd�}q„|  d	¡rvtj| ddd
�}q„t	dƒ‚n| }t |tƒ�r$tj |¡s¬td|› d�ƒ‚| d¡rÆtj|dd�}n>| d¡ràtj|dd�}n$| d	¡rütj|ddd
�}nt	dƒ‚|j
dd…df  t¡j ¡ }nt |tƒ�s8t	dƒ‚||jv�rLt	dƒ‚|||  |¡ }|j�rntdƒ‚|dk�r||S |dk�rš|�ršt|||ƒ n"|dk�r´|�s´t	dƒ‚nt	dƒ‚dS )a2  Reads the data_input file and filter only selected row available in filter_list.

    Args:
        data_input (str, dataframe): the source data, path to file or dataframe
        filter_list (str, list): the filter, path to file or list of string
        filter_by (str): data_input's column to filter the filter_list to
        export_filename (str, optional): export filename. required when return_type is "file"
        export_dir (str, optional): export directory. default to <current directory>/metrics
        return_type (str): "file" or "dataframe"
    rÇ   rë   rì   Nrí   rï   rð   rñ   r×   ró   rô   r   z1Please provide a List of strings or path to file.z5`filter_by` key does not exists in the data provided.zANo common file names between data_input and filter_list. Exiting.Z	dataframerÿ   z!Please provide `export_filename`.z1Return type must be either `dataframe` or `file`.)rö   r½   rW   r2   r“   r÷   rø   r¤   rù   rÎ   ZilocZastypeÚvaluesÚtolistr=   rœ   Úisinr©   rú   r   )	rå   r   r  rè   rM   r  rR   Z	filter_dfÚresr   r   r    Úfilter_metricsV  sH    







 

r  c                      sr   e Zd ZdZ‡ fdd„Zedd„ ƒZedd„ ƒZedd	„ ƒZd
ddœdd„Z	dddœdd„Z
ddœdd„Z‡  ZS )Ú$ObjectDetectionMetricsCalculatorBasezµ
    Calculates object detection metrics for each document:
    - f1 score
    - precision
    - recall
    - average precision (mAP)
    It also calculates aggregated metrics.
    c                   s*   t ƒ  ¡  ‡ fdd„ˆ j d¡D ƒˆ _d S )Nc                   s    g | ]}|  ¡ r| ˆ j¡‘qS r   r.   r1   r3   r   r    r!   ¤  s   þzFObjectDetectionMetricsCalculatorBase.__post_init__.<locals>.<listcomp>z,analysis/*/layout_dump/object_detection.json)r}   r:   r,   Úrglobr8   r3   r~   r3   r    r:   ¢  s    


þz2ObjectDetectionMetricsCalculatorBase.__post_init__c                 C  s   g d¢S ©N©ré   Z	precisionZrecallrê   r   r3   r   r   r    r‚   ª  s    z;ObjectDetectionMetricsCalculatorBase.supported_metric_namesc                 C  s   dS ©Nz%all-docs-object-detection-metrics.tsvr   r3   r   r   r    r;   ®  s    z5ObjectDetectionMetricsCalculatorBase.default_tsv_namec                 C  s   dS ©Nz&aggregate-object-detection-metrics.tsvr   r3   r   r   r    r<   ²  s    z9ObjectDetectionMetricsCalculatorBase.default_agg_tsv_namer½   zOptional[Path])Ú	file_stemr@   c                 C  s(   | j D ]}t|jƒj|kr|  S qdS )az  Find the file corresponding to OD model dump file among the set of ground truth files

        The files in ground truth paths keep the original extension and have .json suffix added,
        e.g.:
        some_document.pdf.json
        poster.jpg.json

        To compare to `file_stem` we need to take the prefix part of the file, thus double-stem
        is applied.
        N)r9   r   r�   )r4   r  r2   r   r   r    Ú_find_file_in_ground_truth¶  s    

z?ObjectDetectionMetricsCalculatorBase._find_file_in_ground_truthr   ztuple(str, Path, Path)rj   c                 C  s¼   t |ƒ}|jd }|  |¡}|| jvr6td|› d�ƒ‚t |jƒjdd… }| j| }| ¡ s~t	 
d|› d�¡ td|› d�ƒ‚| j| }| ¡ s²t	 
d|› d�¡ td|› d�ƒ‚|||fS )	a  Resolves ground doctype, prediction file path and ground truth path.

        As OD dump directory structure differes from other simple outputs, it needs
        a specific processing to match the output OD dump file with corresponding
        OD GT file.

        The outputs are placed in a dicrectory structure:

        analysis
        |- document_name
            |- layout_dump
                |- object_detection.json
            |- bboxes # not used in this evaluation

        and the GT file is pleced in od_gt directory for given dataset

        dataset_name
        |- od_gt
            |- document_name.pdf.json

        Args:
            doc (Path): path to the OD dump file

        Returns:
            tuple: doctype, prediction file path, ground truth path
        éýÿÿÿrˆ   z not found in list of GT filesr„   Nr†   r‡   z does not exist)r   r’   r  r9   rÎ   r�   r‘   r,   r“   r[   r”   r-   )r4   rk   Zod_dump_pathr  r˜   r—   rŠ   r‹   r   r   r    Ú
_get_pathsÆ  s    




z/ObjectDetectionMetricsCalculatorBase._get_pathsr>   r_   c           	      C  s´   g d¢| j  }tj||d�}|jr0tjtd�}nvi }| j D ]X}|||  ¡  }||  ttt	t
g¡ ¡ }|jrŠtjg d¢g d¢d�||< q:|||< q:t |¡ ¡  ¡ }t|_||fS )N)rš   r—   r™   r›   r    r¡   r¢   )r‚   r¤   r¥   r©   rÜ   r«   r¬   r   r   r   r   r­   r²   rª   rœ   )	r4   r?   rµ   rR   rS   r¶   r#   r·   r¸   r   r   r    rA   ÷  s     
ÿ
z9ObjectDetectionMetricsCalculatorBase._generate_dataframes)rr   rs   rt   ru   r:   rw   r‚   r;   r<   r  r  rA   r¹   r   r   r~   r    r  —  s   	


1r  c                      sX   e Zd Z‡ fdd„Zedd„ ƒZedd„ ƒZedd„ ƒZd	d
dœdd„Zdd„ Z	‡  Z
S )Ú(ObjectDetectionPerClassMetricsCalculatorc                   s   t ƒ  ¡  d | _|  ¡  d S r`   )r}   r:   Úper_class_metric_namesÚ_set_supported_metricsr3   r~   r   r    r:     s    
z6ObjectDetectionPerClassMetricsCalculator.__post_init__c                 C  s   | j r| j S tdƒ‚d S )Nz:per_class_metrics not initialized - cannot get class names)r  rÎ   r3   r   r   r    r‚     s    z?ObjectDetectionPerClassMetricsCalculator.supported_metric_namesc                 C  s   dS )Nz/all-docs-object-detection-metrics-per-class.tsvr   r3   r   r   r    r;     s    z9ObjectDetectionPerClassMetricsCalculator.default_tsv_namec                 C  s   dS )Nz0aggregate-object-detection-metrics-per-class.tsvr   r3   r   r   r    r<      s    z=ObjectDetectionPerClassMetricsCalculator.default_agg_tsv_namer   ri   rj   c              
   C  sÆ   z|   |¡\}}}W n< tyP } z$t d|› d|› �¡ W Y d}~dS d}~0 0 tj||d�}| ¡ \}}|j|dg}	| jD ]B}
d 	|
 
d¡dd… ¡}|
 
d¡d }t||ƒ}|	 || ¡ q~|	S )züCalculate both class-aggregated and per-class metrics for a single document.

        Args:
            doc (Path): path to the OD dump file

        Returns:
            tuple: a tuple of aggregated and per-class metrics for a single document
        rl   rm   N©Zprediction_file_pathZground_truth_file_pathÚ_rÆ   )r  rÎ   r[   rp   r   r•   Úget_metricsr�   r‚   ÚjoinÚsplitr�   rû   )r4   rk   r—   rŠ   r‹   rq   Ú	processorr  Zper_class_metricsZper_class_metrics_rowZcombined_metric_namer#   Ú
class_nameZclass_metricsr   r   r    rn   $  s(    	þý

z:ObjectDetectionPerClassMetricsCalculator._process_documentc              	   C  s¢   g d¢}t ƒ }| jD ]R}| j| }t|ƒ�,}t |¡}|d }| |¡ W d  ƒ q1 s\0    Y  qg }|D ]"}	|D ]}
| |	› d|
› �¡ qxqpt|ƒ| _	dS )a  Sets the supported metrics based on the classes found in the ground truth files.
        The difference between per class and aggregated calculator is that the list of classes
        (so the metrics) bases on the contents of the GT / prediction files.
        r  Zobject_detection_classesNr  )
Úsetr9   r-   Úopenr)   ÚloadÚupdaterû   Úsortedr  )r4   rþ   ÚclassesZgt_fileZgt_file_pathÚfr§   Z
gt_classesr  r#   r  r   r   r    r  F  s    



*z?ObjectDetectionPerClassMetricsCalculator._set_supported_metrics)rr   rs   rt   r:   rw   r‚   r;   r<   rn   r  r¹   r   r   r~   r    r    s   


"r  c                   @  sD   e Zd ZdZedd„ ƒZedd„ ƒZedd„ ƒZdd	d
œdd„ZdS )Ú*ObjectDetectionAggregatedMetricsCalculatorzSCalculates object detection metrics for each document and aggregates by all classesc                 C  s   g d¢S r
  r   r3   r   r   r    r‚   ]  s    zAObjectDetectionAggregatedMetricsCalculator.supported_metric_namesc                 C  s   dS r  r   r3   r   r   r    r;   a  s    z;ObjectDetectionAggregatedMetricsCalculator.default_tsv_namec                 C  s   dS r  r   r3   r   r   r    r<   e  s    z?ObjectDetectionAggregatedMetricsCalculator.default_agg_tsv_namer   ri   rj   c              
     sŒ   z|   |¡\}}}W n< tyP } z$t d|› d|› �¡ W Y d}~dS d}~0 0 tj||d�}| ¡ \‰ }|j|dg‡ fdd„| jD ƒ S )zìCalculate both class-aggregated and per-class metrics for a single document.

        Args:
            doc (Path): path to the OD dump file

        Returns:
            list: a list of aggregated metrics for a single document
        rl   rm   Nr  c                   s   g | ]}t ˆ |ƒ‘qS r   rŒ   rŽ   ©rþ   r   r    r!   ‚  r"   zPObjectDetectionAggregatedMetricsCalculator._process_document.<locals>.<listcomp>)	r  rÎ   r[   rp   r   r•   r  r�   r‚   )r4   rk   r—   rŠ   r‹   rq   r  r  r   r$  r    rn   i  s     	þýüz<ObjectDetectionAggregatedMetricsCalculator._process_documentN)	rr   rs   rt   ru   rw   r‚   r;   r<   rn   r   r   r   r    r#  Z  s   


r#  )NN)rš   Nrþ   rÿ   )EÚ
__future__r   Úconcurrent.futuresra   r)   ÚloggingrW   rË   Úabcr   r   Zdataclassesr   Úpathlibr   Útypingr   r	   r
   Únumpyr°   Zpandasr¤   r   Z!unstructured.metrics.element_typer   r   Z%unstructured.metrics.object_detectionr   Z%unstructured.metrics.table.table_evalr   Z$unstructured.metrics.text_extractionr   r   Zunstructured.metrics.utilsr   r   r   r   r   r   r   r   r   r   Ú	getLoggerr[   ÚStreamHandlerÚhandlerr   Ú	FormatterÚ	formatterÚsetFormatterÚhandlersÚ
addHandlerÚsetLevelÚDEBUGrÜ   r´   rÍ   r+   ry   rº   rß   rÄ   r  r  r  r#  r   r   r   r    Ú<module>   sn   0



û  jB  ú\    úAvL