a
    ¡˜ëj`  ã                   @   s  d dl Z d dlZd dlZd dlmZ d dlmZmZ d dlZd dl	Z
d dlZd dlmZmZmZmZmZ e j d¡Ze j e e¡j ¡ ddd¡Ze j ed¡Zd	Zd
ZdZdZdZdZ e
 !g d¢g d¢g d¢g d¢g d¢dœ¡Z"e
 !g d¢g d¢g d¢g d¢dœ¡Z#ej$dd„ ƒZ%e $¡ dd„ ƒZ&ej'j(edd�ej' )d¡dd„ ƒƒZ*ej' +deeeedƒd d!d"ifee eed#ƒd$i feeeed%ƒd&i fg¡d'd(„ ƒZ,ej'j(edd�ej' )dd¡ej' +d)eeeed*ƒd!d"ifg¡d+d,„ ƒƒƒZ-ej'j(edd�ej' )dd¡ej' +d)ee eed-ƒi fg¡d.d/„ ƒƒƒZ.ej'j(edd�ej' )dd¡ej' +d)eeeed0ƒi fg¡d1d2„ ƒƒƒZ/ej'j(edd�ej' )d¡d3d4„ ƒƒZ0ej'j(edd�ej' )d¡d5d6„ ƒƒZ1ej'j(edd�ej' )d¡d7d8„ ƒƒZ2ej'j(edd�ej' )d¡d9d:„ ƒƒZ3ej'j(edd�ej' )d¡d;d<„ ƒƒZ4ej'j(edd�ej' )d¡d=d>„ ƒƒZ5ej'j(edd�ej' )d¡ej' +d?d@dAg¡e6e7d?œdBdC„ƒƒƒZ8ej'j(edd�ej' )d¡dDdE„ ƒƒZ9ej'j(edd�ej' )d¡dFdG„ ƒƒZ:ej'j(edd�ej' )d¡dHdI„ ƒƒZ;ej'j(edd�ej' )d¡dJdK„ ƒƒZ<ej'j(edd�ej' )d¡dLdM„ ƒƒZ=ej'j(edd�ej' )d¡dNdO„ ƒƒZ>ej'j(edd�ej' )d¡ej' +dPd@dAg¡e6e7dPœdQdR„ƒƒƒZ?ej'j(edd�ej' )d¡dSdT„ ƒƒZ@ej'j(edd�ej' )d¡dUdV„ ƒƒZAej'j(edd�ej' )d¡dWdX„ ƒƒZBdS )Yé    N)ÚPath)Ú	MagicMockÚpatch)ÚElementTypeMetricsCalculatorÚTableStructureMetricsCalculatorÚTextExtractionMetricsCalculatorÚfilter_metricsÚget_mean_groupingz/.dockerenvz..zexample-docsZtest_evaluate_filesZunstructured_outputZgold_standard_cctZgold_standard_element_typeZgold_standard_table_structureZunstructured_output_cctZ#unstructured_output_table_structure)úBank Good Credit Loan.pptxú Performance-Audit-Discussion.pdfzcurrency.csv)ZpptxZpdfÚcsv)Ú
connector1r   Z
connector2)g–C‹lçûé?g+‡ÙÎï?gü©ñÒMbì?)gü©ñÒMbP?gü©ñÒMb`?gË¡E¶óý¤?)ÚfilenameÚdoctypeÚ	connectorzcct-accuracyzcct-%missing)r   r   r   zelement-type-accuracyc            
      c   s  t dƒ��ú} t dƒ��Î}t  td¡��ž}t dƒ��r}t dƒ��F}t dƒ��}t  td¡�ì}t dƒ�Â}t  td	¡�”}| ||||||||d
œ	}	d|	d _d|	d _ddg|	d _ddiddig|	d _d|	d _dg|	d _d|	d _ddg|	d _|	V  W d   ƒ n1 �s0    Y  W d   ƒ n1 �s,0    Y  W d   ƒ n1 �sL0    Y  W d   ƒ n1 �sl0    Y  W d   ƒ n1 �sŒ0    Y  W d   ƒ n1 �s¬0    Y  W d   ƒ n1 �sÌ0    Y  W d   ƒ n1 �sì0    Y  W d   ƒ n1 �s0    Y  d S )Nz0unstructured.metrics.evaluate.calculate_accuracyz<unstructured.metrics.evaluate.calculate_percent_missing_textZ	_get_cctsz8unstructured.metrics.evaluate.get_element_type_frequencyzBunstructured.metrics.evaluate.calculate_element_type_percent_matchz-unstructured.metrics.evaluate._read_text_fileÚexistsz@unstructured.metrics.evaluate.TableEvalProcessor.from_json_filesZsupported_metric_names)	Úmock_calculate_accuracyÚ#mock_calculate_percent_missing_textÚmock_get_cctsÚmock_get_element_type_frequencyÚmock_read_text_fileÚ)mock_calculate_element_type_percent_matchÚ)mock_table_eval_processor_from_json_filesÚmock_supported_metric_namesÚmock_path_existsg      à?r   r   Z
output_cctZ
source_cctr   Zele1é   Zele2é   r   r   Útable_level_accr   Tr   Zoutput_textZsource_textr   )r   Úobjectr   r   r   Úreturn_valueZside_effect)
r   r   r   r   r   r   r   r   r   Zmocks© r    úf/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/metrics/test_evaluate.pyÚmock_dependencies=   sn    ÿÿþÿüÿúÿø
ÿöÿôÿòÿð÷



r"   c                  c   s   dd„ } dV  | ƒ  dS )z@Fixture for removing side-effects of running tests in this file.c                  S   s@   g d¢} dd„ t  t¡D ƒ}|D ]}|j| v r t |j¡ q dS )z.Remove directories created from running tests.)Útest_evaluate_results_cctZtest_evaluate_results_cct_txtÚ"test_evaluate_results_element_typeÚ$test_evaluate_result_table_structurec                 s   s   | ]}|  ¡ r|V  qd S )N)Úis_dir)Ú.0Údr    r    r!   Ú	<genexpr>y   ó    zL_cleanup_after_test.<locals>.remove_generated_directories.<locals>.<genexpr>N)ÚosÚscandirÚTESTING_FILE_DIRÚnameÚshutilÚrmtreeÚpath)Ztarget_dir_namesÚsubdirsr(   r    r    r!   Úremove_generated_directorieso   s
    
z9_cleanup_after_test.<locals>.remove_generated_directoriesNr    )r3   r    r    r!   Ú_cleanup_after_testk   s    r4   z&Skipping this test in Docker container)Úreasonc                  C   s®   t j tt¡} t j tt¡}t j td¡}t| |d�j|ddd� t j t j |d¡¡s\J ‚t	j
t j |d¡dd�}t|ƒdks„J ‚t|jƒd	ks–J ‚|jd
 jdksªJ ‚d S )Nr#   ©Údocuments_dirÚground_truths_dirF)Ú
export_dirÚvisualize_progressZdisplay_agg_dfúall-docs-cct.tsvú	©Úsepr   é   r   r
   )r+   r1   Újoinr-   ÚUNSTRUCTURED_OUTPUT_DIRNAMEÚGOLD_CCT_DIRNAMEr   Ú	calculateÚisfileÚpdÚread_csvÚlenÚcolumnsÚilocr   ©Ú
output_dirÚ
source_dirr9   Údfr    r    r!   Útest_text_extraction_evaluationƒ   s    ÿþrN   )Úcalculator_classÚoutput_dirnameÚsource_dirnamer1   Úexpected_lengthÚkwargszBank Good Credit Loan.pptx.txtr?   Údocument_typeÚtxtzIRS-2023-Form-1095-A.pdf.jsoné   zIRS-form-1987.pdf.jsoné   c           
      C   sJ   t tƒ| }t tƒ| }| f ||dœ|¤Ž}| |¡}	t|	ƒ|ksFJ ‚d S )Nr6   )r   r-   Ú_process_documentrG   )
rO   rP   rQ   r1   rR   rS   rK   rL   Ú
calculatorÚoutput_listr    r    r!   Ú:test_process_document_returns_the_correct_amount_of_values•   s
     
r[   )rO   rP   rQ   r1   rS   z,2310.03502text_to_image_synthesis1-7.pdf.txtc                 C   sŒ   t tƒ| }t tƒ| }| d }| d }	| d }
|f ||dœ|¤Ž}| |¡}|d dks^J ‚|jdkslJ ‚|	jdkszJ ‚|
jdksˆJ ‚d S )Nr   r   r   r6   r   ú.pdf©r   r-   rX   Ú
call_count)r"   rO   rP   rQ   r1   rS   rK   rL   r   r   r   rY   rZ   r    r    r!   ÚQtest_TextExtractionMetricsCalculator_process_document_returns_the_correct_doctype½   s    
r_   z(tablib-627mTABLES-2310.07875-p7.pdf.jsonc                 C   s”   t tƒ| }t tƒ| }|f ||dœ|¤Ž}||_||_|| g|_tƒ }	d|	_d|	_| d }
|	|
jj	_| 
|¡}|d dks‚J ‚|
jdks�J ‚d S )Nr6   r   g�Âõ(\�ê?r   r   r\   )r   r-   Z_ground_truths_dirZ_documents_dirZ_ground_truth_pathsr   Ztotal_predicted_tablesZtable_evel_accr   Zprocess_filerX   r^   )r"   rO   rP   rQ   r1   rS   rK   rL   rY   Zmock_reportr   rZ   r    r    r!   ÚQtest_TableStructureMetricsCalculator_process_document_returns_the_correct_doctypeÜ   s     ÿ

r`   zIRS-form.1987.pdf.jsonc                 C   sŒ   t tƒ| }t tƒ| }|f ||dœ|¤Ž}| d }	| d }
| d }| |¡}|d dks^J ‚|
jdkslJ ‚|	jdkszJ ‚|jdksˆJ ‚d S )Nr6   r   r   r   r   r\   é   r]   )r"   rO   rP   rQ   r1   rS   rK   rL   rY   Zmock_element_type_frequencyr   r   rZ   r    r    r!   ÚNtest_ElementTypeMetricsCalculator_process_document_returns_the_correct_doctype  s    ÿ
rb   c                  C   s’   t j tt¡} t j tt¡}t j td¡}t| |dd�j|d� tj	t j |d¡dd�}t
|ƒdkshJ ‚t
|jƒd	kszJ ‚|jd
 jdksŽJ ‚d S )Nr#   rU   ©r7   r8   rT   ©r9   r;   r<   r=   r   r?   r   r
   )r+   r1   r@   r-   ÚUNSTRUCTURED_CCT_DIRNAMErB   r   rC   rE   rF   rG   rH   rI   r   rJ   r    r    r!   Ú(test_text_extraction_evaluation_type_txt)  s    ÿþrf   c                  C   s¬   t j tt¡} t j tt¡}t j td¡}t| |d�j|dd� t j t j |d¡¡sZJ ‚t	j
t j |d¡dd�}t|ƒdks‚J ‚t|jƒd	ks”J ‚|jd
 jdks¨J ‚d S )Nr#   r6   F©r9   r:   z#all-docs-element-type-frequency.tsvr<   r=   r   rW   r   zIRS-form-1987.pdf)r+   r1   r@   r-   rA   ÚGOLD_ELEMENT_TYPE_DIRNAMEr   rC   rD   rE   rF   rG   rH   rI   r   rJ   r    r    r!   Útest_element_type_evaluation:  s    þýri   c                  C   s  t j tt¡} t j tt¡}t j td¡}t| |d�j|dd� t j t j |d¡¡sZJ ‚t j t j |d¡¡stJ ‚t	j
t j |d¡dd�}t	j
t j |d¡dd� d	¡}t|ƒd
ksºJ ‚t|jƒdksÌJ ‚|jd jdksàJ ‚t tj|d |d d�d¡|jd k�sJ ‚d S )Nr%   r6   Frg   z%all-docs-table-structure-accuracy.tsvz&aggregate-table-structure-accuracy.tsvr<   r=   Zmetricra   é   r   zIRS-2023-Form-1095-A.pdfr   Ztotal_tables)Úweightsr   )r   Úaverage)r+   r1   r@   r-   Ú$UNSTRUCTURED_TABLE_STRUCTURE_DIRNAMEÚGOLD_TABLE_STRUCTURE_DIRNAMEr   rC   rD   rE   rF   Z	set_indexrG   rH   rI   r   ÚnpÚroundrl   Úloc)rK   rL   r9   rM   Zagg_dfr    r    r!   Útest_table_structure_evaluationM  s0    þýÿþÿÿrr   c                  C   s–   t j tt¡} dg}t j tt¡}t j td¡}t| |d�j|d�j|d� t j 	t j |d¡¡sfJ ‚t
jt j |d¡dd�}t|ƒt|ƒks’J ‚d S )	Nzcurrency.csv.jsonr#   r6   )Zdocument_pathsrd   r;   r<   r=   )r+   r1   r@   r-   rA   rB   r   Zon_filesrC   rD   rE   rF   rG   )rK   rZ   rL   r9   rM   r    r    r!   Útest_text_extraction_takes_listh  s    þýýrs   c                  C   sl   t j tt¡} t j tt¡}t j td¡}t| |dd�j|d� tj	t j |d¡dd�}t
|ƒdkshJ ‚d S )	Nr#   r   )r7   r8   Úgroup_byrd   úall-doctype-agg-cct.tsvr<   r=   rW   )r+   r1   r@   r-   rA   rB   r   rC   rE   rF   rG   rJ   r    r    r!   Ú"test_text_extraction_with_grouping{  s    ýürv   c                  C   sn   t j tt¡} t j tt¡}t j td¡}t t¡�& t	| |dd�j
|d� W d   ƒ n1 s`0    Y  d S )Nr#   zinvalid typerc   rd   )r+   r1   r@   r-   rA   rB   ÚpytestÚraisesÚ
ValueErrorr   rC   )rK   rL   r9   r    r    r!   Útest_text_extraction_wrong_typeŒ  s    ÿþrz   )ÚgroupingÚ	count_row)r   r   )r   ra   c                 C   sZ   t j td¡}t| t|dd� tjt j |d| › d�¡dd�}||   ¡  	¡ |ksVJ ‚d S )Nr#   Útext_extraction©rt   Ú
data_inputr9   Ú	eval_nameúall-z-agg-cct.tsvr<   r=   )
r+   r1   r@   r-   r	   ÚDUMMY_DF_CCTrE   rF   ÚdropnaÚnunique)r{   r|   r9   Ú
grouped_dfr    r    r!   Útest_get_mean_grouping_df_input˜  s    ü r†   c                  C   s�   t j tt¡} t j tt¡}t j td¡}t| |d�j|d� t j |d¡}td||dd� t	j
t j |d¡d	d
�}|d  ¡  ¡ dksŒJ ‚d S )Nr#   r6   rd   r;   r   r}   r~   ru   r<   r=   r   )r+   r1   r@   r-   rA   rB   r   rC   r	   rE   rF   rƒ   r„   )rK   rL   r9   r   r…   r    r    r!   Ú test_get_mean_grouping_tsv_input§  s$    þýür‡   c                  C   s”   t j tt¡} t j tt¡}t j td¡}t| |d�j|d� tj	t j |d¡dd�}t
 t¡�  td||dd	� W d   ƒ n1 s†0    Y  d S )
Nr#   r6   rd   r;   r<   r=   Úinvalidr}   r~   )r+   r1   r@   r-   rA   rB   r   rC   rE   rF   rw   rx   ry   r	   rJ   r    r    r!   Ú$test_get_mean_grouping_invalid_group¾  s"    þýür‰   c                  C   sF   t  ¡ } t t¡�  td| ddd� W d   ƒ n1 s80    Y  d S )Nr   Úsome_dirr}   ©r€   ©rE   Ú	DataFramerw   rx   Ú
SystemExitr	   )Zempty_dfr    r    r!   Ú&test_text_extraction_grouping_empty_dfÔ  s    r�   c                  C   sN   t  dg d¢i¡} t t¡� td| ddƒ W d   ƒ n1 s@0    Y  d S )NZsome_column)r   ra   r   r   rŠ   r}   rŒ   )Zdf_with_no_groupingr    r    r!   Ú.test_get_mean_grouping_missing_grouping_columnÜ  s    r�   c                  C   sP   t  dg d¢i¡} t t¡�  td| ddd� W d   ƒ n1 sB0    Y  d S )Nr   )NNNrŠ   r}   r‹   rŒ   )Zdf_with_null_groupingr    r    r!   Ú/test_get_mean_grouping_all_null_grouping_columnä  s    r‘   c                   C   s>   t  t¡�  tdtddd� W d   ƒ n1 s00    Y  d S )Nr   rŠ   rˆ   r‹   )rw   rx   ry   r	   ÚDUMMY_DF_ELEMENT_TYPEr    r    r    r!   Ú(test_get_mean_grouping_invalid_eval_nameì  s    r“   )rt   r|   c                 C   sZ   t j td¡}t| t|dd� tjt j |d| › d�¡dd�}||   ¡  	¡ |ksVJ ‚d S )Nr$   Zelement_typer~   r�   z-agg-element-type.tsvr<   r=   )
r+   r1   r@   r-   r	   r’   rE   rF   rƒ   r„   )rt   r|   r9   r…   r    r    r!   Ú#test_get_mean_grouping_element_typeó  s    üÿr”   c                  C   s¶   t tj td¡dƒ�$} |  d¡ |  d¡ W d   ƒ n1 s>0    Y  tj td¡}tttj td¡dd|dd	� tj	tj |d¡d
d�}t
|ƒdksœJ ‚|d jd dks²J ‚d S )Núfilter_list.txtÚwúBank Good Credit Loan.pptx
ú!Performance-Audit-Discussion.pdf
r#   r   úfiltered_metrics.tsvÚfile©r   Zfilter_listZ	filter_byÚexport_filenamer9   Úreturn_typer<   r=   ra   r   r
   )Úopenr+   r1   r@   r-   Úwriter   r‚   rE   rF   rG   rI   )rš   r9   Úfiltered_dfr    r    r!   Útest_filter_metrics  s    
(úr¡   c                  C   s  t tj td¡dƒ�$} |  d¡ |  d¡ W d   ƒ n1 s>0    Y  tj td¡}ttddgdd	|d
d� tj	tj |d	¡dd�}t
d||ddd� tj	tj |d¡dd�}t t|jd ƒd¡sÊJ ‚t t|jd ƒd¡säJ ‚t t|jd ƒd¡sþJ ‚d S )Nr•   r–   r—   r˜   r#   r
   r   r   r™   rš   r›   r<   r=   Úallr}   útwo-filename-agg-cct.tsv©rt   r   r9   r€   rœ   ©r   r   çL7‰A`åì?©r   r   çP�—nƒÀ?©r   ra   ç²�ï§ÆK·?©rž   r+   r1   r@   r-   rŸ   r   r‚   rE   rF   r	   ro   ÚiscloseÚfloatrI   ©rš   r9   r    r…   r    r    r!   Útest_get_mean_grouping_all_file  s0    
(úûr¯   c                  C   s
  t tj td¡dƒ�$} |  d¡ |  d¡ W d   ƒ n1 s>0    Y  tj td¡}tttj td¡dd|dd	� tj	tj |d¡d
d�}t
d||ddd� tj	tj |d¡d
d�}t t|jd ƒd¡sÐJ ‚t t|jd ƒd¡sêJ ‚t t|jd ƒd¡�sJ ‚d S )Nr•   r–   r—   r˜   r#   r   r™   rš   r›   r<   r=   r¢   r}   r£   r¤   r¥   r¦   r§   r¨   r©   rª   r«   r®   r    r    r!   Ú#test_get_mean_grouping_all_file_txt9  s0    
(úûr°   )Cr+   Úpathlibr/   r   Zunittest.mockr   r   Únumpyro   ZpandasrE   rw   Zunstructured.metrics.evaluater   r   r   r   r	   r1   r   Zis_in_dockerr@   Ú__file__ÚparentÚresolveZEXAMPLE_DOCS_DIRECTORYr-   rA   rB   rh   rn   re   rm   r�   r‚   r’   Zfixturer"   r4   ÚmarkZskipifZusefixturesrN   Zparametrizer[   r_   r`   rb   rf   ri   rr   rs   rv   rz   ÚstrÚintr†   r‡   r‰   r�   r�   r‘   r“   r”   r¡   r¯   r°   r    r    r    r!   Ú<module>   s>  ÿ÷ÿøÿ
-

ú	ú	úïþ
ûÿþûÿþûÿþ

















