a
    ¡˜ëjT'  ã                   @   sò   d dl mZmZmZ d dlmZ d dlmZmZ dee	 ee	 ee
e
e
f edœdd„Zdee	 ee	 ee
e
e
f e	eed
œdd„Ze	ee	e
f dœdd„Zee	 ee	 edœdd„Zdee	 ee	dœdd„Ze	e	dœdd„Ze	e	dœdd„ZdS ) é    )ÚDictÚOptionalÚTuple)ÚLevenshtein)Úclean_bulletsÚremove_sentence_punctuation©é   é   r
   )ÚoutputÚsourceÚweightsÚreturnc                 C   s   t | ||dd�S )z¨
    Calculates accuracy by calling calculate_edit_distance function using `return_as=score`.
    The function will return complement of the edit distance instead.
    Úscore)Ú	return_as)Úcalculate_edit_distance)r   r   r   © r   úc/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/metrics/text_extraction.pyÚcalculate_accuracy   s    	r   ÚdistanceT)r   r   r   r   Ústandardize_whitespacesr   c           	      C   sŠ   ddg}||vrt d| ƒ‚tt| |ƒƒ} tt||ƒƒ}tj| ||d�}tt|ƒdƒ}tt|| dƒdƒ}|dkrzd| S |dkr†|S dS )a%  
    Calculates edit distance using Levenshtein distance between two strings.

    Args:
        output (str): The target string to be compared.
        source (str): The reference string against which 'output' is compared.
        weights (Tuple[int, int, int], optional): A tuple containing weights
            for insertion, deletion, and substitution operations in the edit
            distance calculation. Default is (2, 1, 1).
        return_as (str, optional): The type of result to return, one of
            ["score", "distance"].
            Default is "distance".

    Returns:
        float: The calculated edit distance or similarity score between
            the 'output' and 'source' strings.

    Raises:
        ValueError: If 'return_as' is not one of the valid return types
        ["score", "distance"].

    Note:
        This function calculates the edit distance (or similarity score) between
        two strings using the Levenshtein distance algorithm. The 'weights' parameter
        allows customizing the cost of insertion, deletion, and substitution
        operations. The 'return_as' parameter determines the type of result to return:
        - "score": Returns the similarity score, where 1.0 indicates a perfect match.
        - "distance": Returns the raw edit distance value.

    r   r   z.Invalid return value type. Expected one of: %s)r   g      ð?g        r
   )Ú
ValueErrorÚstandardize_quotesÚprepare_strr   r   ÚmaxÚlenÚmin)	r   r   r   r   r   Zreturn_typesr   Zsource_char_lenZbounded_percentage_distancer   r   r   r      s    %r   )Útextr   c                 C   sø   i }d}t t|  ¡ ddgƒƒ ¡ }d}|t|ƒk rôt|| ƒdkrz|| |v rd|||   d7  < nd||| < |d7 }q&|}d}|t|ƒk r´t|| ƒdkr´||| 7 }|d7 }q‚t|ƒdkrî||  ¡ rî||v ræ||  d7  < nd||< |}q&|S )zõ
    Outputs the bag of words (BOW) found in the input text and their frequencies.

    Takes "clean, concatenated text" (CCT) from a document as input.

    Removes sentence punctuation, but not punctuation within a word (ex. apostrophes).
    Ú ú-ú'r   r
   )r   r   ÚlowerÚsplitr   Úisalnum)r   ZbowZincorrect_wordÚwordsÚiÚjr   r   r   Úbag_of_wordsJ   s*    

r'   )r   r   r   c           
      C   s’   t | ƒ} t |ƒ}t| ƒ}t|ƒ}d}d}| ¡ D ]<\}}||7 }||vrR||7 }q0|| }|t|| dƒ7 }q0|dkrzdS t|| dƒ}	t|	dƒS )aÞ  
    Creates the bag of words (BOW) found in each input text and their frequencies, then compares the
    output BOW against the source BOW to calculate the % of text from the source text missing from
    the output text.

    Takes "clean, concatenated text" (CCT) from a document output and the ground truth source text
    as inputs.

    If the output text contains all words from the source text and then some extra, result will be
    0% missing text - this calculation does not penalize duplication.

    A spaced-out word (ex. h e l l o) is considered missing; individual characters of a word
    will not be counted as separate words.

    Returns the percentage of missing text represented as a decimal between 0 and 1.
    r   é   r
   )r   r'   Úitemsr   Úroundr   )
r   r   Z
output_bowZ
source_bowZtotal_source_word_countZtotal_missing_word_countZsource_wordZsource_countZoutput_countZfraction_missingr   r   r   Úcalculate_percent_missing_texto   s     
r+   F)Ústringr   r   c                 C   s"   | sdS |rd  |  ¡ ¡S t| ƒS )Nr   ú )Újoinr"   Ústr)r,   r   r   r   r   r   �   s
    r   c                 C   s¼   ddddddddd	d
dddddddddd	dœ}dddddddddddd d!d"d#d$d%d&d'd(d)œ}d*}d+}|  ¡ D ] }t|ƒ}|| v rl|  ||¡} ql|  ¡ D ] }t|ƒ}|| v r–|  ||¡} q–| S ),zÛ
    Converts all unicode quotes to standard ASCII quotes with comprehensive coverage.

    Args:
        text (str): The input text to be standardized.

    Returns:
        str: The text with standardized quotes.
    zU+0022zU+201CzU+201DzU+201EzU+201FzU+00ABzU+00BBzU+275DzU+275EzU+2E42zU+1F676zU+1F677zU+1F678zU+2826zU+2834zU+301DzU+301EzU+301FzU+FF02)ú"r0   r0   u   â€žu   â€Ÿõ   Â«õ   Â»u   â��u   â�žu   â¹‚u   ðŸ™¶u   ðŸ™·u   ðŸ™¸u   â ¦u   â ´u   ã€�u   ã€žu   ã€Ÿu   ï¼‚z,,zU+0027zU+2018zU+2019zU+201AzU+201BzU+2039zU+203AzU+275BzU+275CzU+300CzU+300DzU+300EzU+300FzU+FE41zU+FE42zU+FE43zU+FE44zU+FF07zU+FF62zU+FF63)r    r    r    u   â€šu   â€›u   â€¹u   â€ºu   â�›u   â�œu   ã€Œu   ã€�u   ã€Žu   ã€�u   ï¹�u   ï¹‚u   ï¹ƒu   ï¹„u   ï¼‡u   ï½¢u   ï½£r0   r    )ÚvaluesÚunicode_to_charÚreplace)r   Zdouble_quotesZsingle_quotesZdouble_quote_standardZsingle_quote_standardÚunicode_valZunicode_charr   r   r   r   ¥   sj    ììr   )r6   r   c                 C   s   t t|  dd¡dƒƒS )zÃ
    Converts a Unicode value to a character.

    Args:
        unicode_val (str): The Unicode value to convert.

    Returns:
        str: The character corresponding to the Unicode value.
    zU+r   é   )ÚchrÚintr5   )r6   r   r   r   r4   ñ   s    
r4   N)r   )r   r   T)F)Útypingr   r   r   Zrapidfuzz.distancer   Zunstructured.cleaners.corer   r   r/   r9   Úfloatr   Úboolr   r'   r+   r   r   r4   r   r   r   r   Ú<module>   s8    ýü   ûú6&ý.L