a
    ¡˜ëjµ  ã                   @   sˆ  d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	 d dl
Z
d dlZd dlmZmZ e  d¡Zeeedœdd„Zeee d	œd
d„Zdd„ Zdd„ Zdd„ Zd+eeejeedœdd„Zdd„ Zedœdd„Zedœdd„Zd,e	ejee f ee  e	edf dœd d!„Z!d-eee  ee  e	edf dœd"d#„Z"d.eee  ee  e	edf dœd$d%„Z#eee  ed&œd'd(„Z$d)d*„ Z%dS )/é    N)ÚPath)ÚListÚOptionalÚUnion)Úelements_from_jsonÚelements_to_textzunstructured.eval)ÚdocpathÚoutput_typeÚreturnc              
   C   sz   z<|dkrt t| ƒƒ}n"|dkr*t| ƒ}ntd|› d�ƒ‚W n8 tyt } z t d| › �¡ |‚W Y d}~n
d}~0 0 |S )z�
    Convert given input document (path) into cct-ready. The function only support conversion
    from `json` or `txt` file.
    ÚjsonÚtxtzZFile type not supported. Expects one of `json` or `txt`,                     but received z	 instead.zCould not read the file N)r   r   Ú_read_text_fileÚ
ValueErrorÚloggerÚerror)r   r	   Z
output_cctÚe© r   úY/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/metrics/utils.pyÚ_prepare_output_cct   s    
ÿÿr   )Údirr
   c                 C   s^   g }t  | ¡D ]J\}}}|D ]:}t j || ¡}|dkrB| |¡ q| t j ||¡¡ qq|S )z¸
    Recursively lists all files in the given directory and its subdirectories.
    Returns a list of all files found, with each file's path relative to the
    initial directory.
    Ú.)ÚosÚwalkÚpathÚrelpathÚappendÚjoin)r   ÚlistdirÚdirpathÚ_Ú	filenamesÚfilenameÚrelative_pathr   r   r   Ú_listdir_recursive%   s    r#   c                 C   s   dddddœ}| j |d�S )a  
    Renames aggregated columns in a DataFrame based on a predefined mapping.

    Parameters:
    df (pandas.DataFrame): The DataFrame with aggregated columns to rename.

    Returns:
    pandas.DataFrame: A new DataFrame with renamed aggregated columns.
    ÚmeanÚstdevÚpstdevÚcount)Ú_meanÚ_stdevÚ_pstdevÚ_count)Úcolumns)Úrename)ÚdfZ
rename_mapr   r   r   Ú_rename_aggregated_columns7   s    
r/   c                  G   s   t j| dd� ¡ S )zs
    Concatenates multiple pandas DataFrame objects along the columns (side-by-side)
    and resets the index.
    é   )Zaxis)ÚpdÚconcatZreset_index©r.   r   r   r   Ú_format_grouping_outputE   s    r4   c              	      sä   t ˆƒdkrdS ˆj ¡ }‡fdd„|D ƒ‰ t d ‡ fdd„t|ƒD ƒ¡¡ t dtˆ ƒ dt |ƒd	   ¡ ˆ ¡ D ]f\}}g ‰|D ],}t	|t
ƒr¦ˆ |d
›¡ qˆˆ t|ƒ¡ qˆt d ‡ ‡fdd„tt ˆƒƒD ƒ¡¡ qxdS )zD
    Displays the evaluation metrics in a formatted text table.
    r   Nc                    s,   g | ]$}t t|ƒt d d„ ˆ | D ƒƒƒ‘qS )c                 s   s   | ]}t t|ƒƒV  qd S ©N)ÚlenÚstr)Ú.0Úitemr   r   r   Ú	<genexpr>U   ó    z&_display.<locals>.<listcomp>.<genexpr>)Úmaxr6   )r8   Úheaderr3   r   r   Ú
<listcomp>T   s   z_display.<locals>.<listcomp>ú c                 3   s    | ]\}}|  ˆ | ¡V  qd S r5   ©Úljust)r8   Úir=   )Ú
col_widthsr   r   r:   W   r;   z_display.<locals>.<genexpr>ú-r0   z.3fc                 3   s    | ]}ˆ|   ˆ | ¡V  qd S r5   r@   )r8   rB   )rC   Úformatted_rowr   r   r:   a   r;   )r6   r,   ÚtolistÚclickZechor   Ú	enumerateÚsumZiterrowsÚ
isinstanceÚfloatr   r7   Úrange)r.   Úheadersr   Úrowr9   r   )rC   r.   rE   r   Ú_displayM   s"    

ÿ""
 ÿrO   ÚwT©Ú	directoryr!   r.   ÚmodeÚ	overwritec                 C   sš   |dvrt dƒ‚| r$t| ƒjdd� d|jv r@|d  t¡|d< d|jv rfd|jv rf|jddgdd� |stt| |ƒ}|jt	j
 | |¡d	|d
|dkd� dS )z¾
    Save the metrics report to tsv file. The function allows an option 1) to choose `mode`
    as `w` (write) or `a` (append) and 2) to `overwrite` the file if filename existed or not.
    )rP   Úaz/Mode not supported. Mode must be one of [w, a].T)Úexist_okr'   r!   Z	connector)ZbyZinplaceú	FrP   )ÚseprS   Úindexr=   N)r   r   Úmkdirr,   ZastypeÚintZsort_valuesÚ_get_non_duplicated_filenameZto_csvr   r   r   rQ   r   r   r   Ú_write_to_filee   s    

ÿr]   c                 C   s$   t  d| ¡}|rt|d ƒS dS dS )a  
    A function that defines the sorting method for duplicated file names. For example,
    with filename.ext filename (1).ext filename (2).ext filename (10).ext - this function
    extracts the integer in the bracket and sort those numbers ascendingly.
    z(\d+)éÿÿÿÿr   N)ÚreÚfindallr[   )r!   Únumbersr   r   r   Ú_sorting_key{   s    rb   )r
   c           
         s¾   |  dd¡\}}dt |¡› dt |¡› d�‰ t‡ fdd„| D ƒtd�}g }|D ](}t d	|¡}|rP| t| d¡ƒ¡ qP| 	¡  d}|D ]}	|	|kr |d7 }qŠ q¦qŠ|d
 t
|ƒ d | S )zü
    Checks the duplicity of the file name from the list and run the numerical check
    of the minimum number needed as extension to not overwrite the exising file.
    Returns a string of file name in the format of `filename (<min number>).ext`.
    r   r0   ú^z(?: \((\d+)\))?\.ú$c                    s   g | ]}t  ˆ |¡r|‘qS r   )r_   Úmatch)r8   Úf©Úpatternr   r   r>   “   r;   z"_uniquity_file.<locals>.<listcomp>)Úkeyz	\((\d+)\)z (z).)Úrsplitr_   ÚescapeÚsortedrb   Úsearchr   r[   ÚgroupÚsortr7   )
Ú	file_listZtarget_filenameZoriginal_filenameÚ	extensionZduplicated_filesra   Úfilere   ÚcounterÚnumberr   rg   r   Ú_uniquity_file‹   s    
ru   c                 C   s   t t | ¡|ƒ}|S )zs
    Helper function to calls the `_uniquity_file` function. Takes in directory and file name
    to check on.
    )ru   r   r   )r   r!   r   r   r   r\   §   s    r\   é   )ÚscoresÚroundingr
   c                 C   s,   t | ƒdkrdS t | ¡}|s"|S t||ƒS )z¸
    Find mean from the list. Returns None if no element in the list.

    Args:
        rounding (int): optional argument that allows user to define decimal points. Default at 3.
    r   N)r6   Ú
statisticsr$   Úround)rw   rx   r$   r   r   r   r(   °   s    
r(   c                 C   s<   dd„ | D ƒ} t | ƒdkrdS |s,t | ¡S tt | ¡|ƒS )zÓ
    Find standard deviation from the list.
    Returns None if only 0 or 1 element in the list.

    Args:
        rounding (int): optional argument that allows user to define decimal points. Default at 3.
    c                 S   s   g | ]}|d ur|‘qS r5   r   ©r8   Zscorer   r   r   r>   È   r;   z_stdev.<locals>.<listcomp>r0   N)r6   ry   r%   rz   ©rw   rx   r   r   r   r)   ¿   s    	
r)   c                 C   s<   dd„ | D ƒ} t | ƒdkrdS |s,t | ¡S tt | ¡|ƒS )zÞ
    Find population standard deviation from the list.
    Returns None if only 0 or 1 element in the list.

    Args:
        rounding (int): optional argument that allows user to define decimal points. Default at 3.
    c                 S   s   g | ]}|d ur|‘qS r5   r   r{   r   r   r   r>   Ù   r;   z_pstdev.<locals>.<listcomp>r0   N)r6   ry   r&   rz   r|   r   r   r   r*   Ñ   s    
r*   )rw   r
   c                 C   s   t | ƒS )z,
    Returns the row count of the list.
    )r6   )rw   r   r   r   r+   á   s    r+   c              
   C   s”   t j | ¡std| › d�ƒ‚z:t| dd��}| ¡ }W d  ƒ n1 sH0    Y  |W S  tyŽ } z td| › d|› �ƒ‚W Y d}~n
d}~0 0 dS )zG
    Reads the contents of a text file and returns it as a string.
    zThe file at z does not exist.Úignore)ÚerrorsNz+An error occurred when reading the file at z: )r   r   ÚexistsÚFileNotFoundErrorÚopenÚreadÚOSErrorÚIOError)r   rf   Útextr   r   r   r   r   è   s    &r   )rP   T)rv   )rv   )rv   )&Úloggingr   r_   ry   Úpathlibr   Útypingr   r   r   rG   Zpandasr1   Zunstructured.staging.baser   r   Ú	getLoggerr   r7   r   r#   r/   r4   rO   Z	DataFrameÚboolr]   rb   ru   r\   ZSeriesrK   r[   r(   r)   r*   r+   r   r   r   r   r   Ú<module>   s4   
 ÿÿ	.((