a
    ¡˜ëj¢i  ã                   @   sÌ  d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	m
Z
 dd„ Zej ddd	g¡d
d„ ƒZej dg d¢¡dd„ ƒZdd„ Zej dddddddddœfddddddddœfddddddddddœfdddddddddddddœfddddddddddddddddddœfg¡d d!„ ƒZej dg d"¢¡d#d$„ ƒZej d%g d&¢¡d'd(„ ƒZej d)g d*¢¡d+d,„ ƒZej d-ejd d ddd.d/œd dddd0d/œgd d d.d1œdd d0d1œgd2d3�ejd d ddd.d/œdd ddd4d/œdd ddd5d/œd dddd6d/œddddd7d/œddddd8d/œd dddd9d/œddddd0d/œddddd:d/œg	d d d.d1œd dd4d1œd dd5d1œdd d6d1œddd7d1œddd8d1œdd d9d1œddd0d1œddd:d1œg	d;d3�ejd d ddd<d=œd dddd>d=œd d?ddd@d=œdddddAd=œdddddBd=œdd dddCd=œdddddDd=œdddddEd=œd?d dddFd=œg	d d d<d1œd dd>d1œd d?d@d1œdddAd1œdddBd1œdd dCd1œdddDd1œdddEd1œd?d dFd1œg	dGd3�g¡dHdI„ ƒZej dJejdKd d d.d1œdd d0d1œgd2d3�ejdLd d d.d1œd dd4d1œd dd5d1œdd d6d1œddd7d1œddd8d1œdd d9d1œddd0d1œddd:d1œg	d;d3�ejdMd d d<d1œd dd>d1œd d?d@d1œdddAd1œdddBd1œdd dCd1œdddDd1œdddEd1œd?d dFd1œg	dGd3�g¡dNdO„ ƒZdPdQ„ ZeedRœdSdT„Zej dUg dV¢¡dWdX„ ƒZdS )Yé    N)Útext_extraction)Údeckerd_table_to_htmlÚ!extract_cells_from_table_as_cellsÚextract_cells_from_text_as_htmlÚhtml_table_to_deckerd)Ú	partitionc                  C   s0  d} d}t  ddd | ¡¡}|  dd¡}d}d}d}d	}ttj| | d
d�dƒdksVJ ‚ttj|| d
d�dƒdkstJ ‚ttj|| d
d�dƒdks’J ‚ttj|| d
d�dƒdks°J ‚ttj|| d
d�dƒdksÎJ ‚ttj|| d
d�dƒdksìJ ‚ttj|| d
d�dƒdk�sJ ‚ttj|| d
d�dƒdk�s,J ‚d S )NzI like pizza. I like bagels.z!I like p i z z a . I like bagles.z\s+ú Ú zI like pizza.zI like pizza. I like .zI like pizza. I like beagles.z"I like pizza pizza. I like bagels.Úscore)Ú	return_asé   ç      ð?g      è?gö(\�ÂõØ?g{®Gázä?g        g=
×£p=â?g{®Gázì?gHáz®Gé?)ÚreÚsubÚjoinÚreplaceÚroundr   Úcalculate_edit_distance)Ú
source_cctZsource_cct_word_spaceZsource_cct_spacesZsource_cct_no_spaceZsource_cct_one_sentenceZsource_cct_missing_wordZsource_cct_addn_charZsource_cct_dup_word© r   úm/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/metrics/test_text_extraction.pyÚtest_calculate_edit_distance   s²    ÿÿýúøÿýúøÿýúøÿýúøÿýúøÿýúøÿýúøÿr   )ÚfilenameÚstandardize_whitespacesÚexpected_scoreÚexpected_distance)úfake-text.txtFgö(\�Âõè?é&   )r   Tgq=
×£pí?é   c           
      C   s¾   t dƒ�}| ¡ }W d   ƒ n1 s&0    Y  td| › �d�}d dd„ |D ƒ¡}tj||d|d�}tj||d	|d�}	|d
ks„J ‚|dks�J ‚|	d
ksœJ ‚t|dƒ|ks®J ‚|	|ksºJ ‚d S )Nzexample-docs/fake-text.txtzexample-docs/)r   Ú
c                 S   s   g | ]}t |ƒ‘qS r   )Ústr)Ú.0Úelr   r   r   Ú
<listcomp>z   ó    z>test_calculate_edit_distance_with_filename.<locals>.<listcomp>r
   ©r   r   Údistancer   r   r   )ÚopenÚreadr   r   r   r   r   )
r   r   r   r   Úfr   ÚelementsZ
output_cctr
   r&   r   r   r   Ú*test_calculate_edit_distance_with_filenamel   s$    

&ÿür+   ©Ztext1Ztext2))z7The  dogloved the cat, but	
    the cat	loved the
 cowú0The dog loved the cat, but the cat loved the cow)z3Hello    my	name	is H a r p e r, 
what's yourname?ú/Hello my name is H a r p e r, what's your name?©z+I have a	
	dog and a	cat,I love my



dog.ú&I have a dog and a cat, I love my dog.)zÿ
            Name    Age City           Occupation
            Alice   30  New York       Engineer
            Bob     25  Los Angeles    Designer
            Charlie 35  Chicago        Teacher
            David   40  San Francisco  Developer
            úÕ
            Name	Age	City	Occupation
            Alice	30	New York	Engineer
            Bob	25	Los Angeles	Designer
            Charlie	35	Chicago	Teacher
            David	40	San Francisco	Developer
            )r1   zŽName	Age	City	Occupation

 
Alice	30	New York	Engineer
Bob	25	Los Angeles	Designer
Charlie	35	Chicago	Teacher
David	40	San Francisco	Developerc                 C   sl   t j| |ddd�dksJ ‚t j| |ddd�dks4J ‚t j| |ddd�dk sNJ ‚t j| |ddd�dkshJ ‚d S )Nr
   Tr%   r   r&   r   F©r   r   r,   r   r   r   Ú6test_calculate_edit_distance_with_various_whitespace_1�   s0    -ÿýÿÿýÿÿýÿÿýÿr3   c                  C   s\   d} d}t j| |ddd�t j| |ddd�ks0J ‚t j| |ddd�t j| |ddd�k sXJ ‚d S )Nr1   aƒ  

            | Name    | Age | City         | Occupation     |
            |---------|-----|--------------|----------------|
            | Alice   | 30  | New York     | Engineer       |
            | Bob     | 25  | Los Angeles  | Designer       |
            | Charlie | 35  | Chicago      | Teacher        |
            | David   | 40  | San Francisco| Developer      |

            r
   Tr%   Fr&   r2   )Zsource_cct_tabsZsource_cct_with_bordersr   r   r   Ú6test_calculate_edit_distance_with_various_whitespace_2Ó   s&    
ÿÿþÿüþr4   ©ÚtextÚexpectedr-   é   r   é   )ÚtheÚcatZlovedÚdogÚbutZcowr.   )ZhelloÚmyÚnameÚiszwhat'sZyourr0   )ÚiÚhaveÚar<   Úandr;   Zlover>   z4My dog's hair is red, but the dogs' houses are blue.)r>   zdog'sZhairr@   Zredr=   r:   zdogs'ZhousesZareZbluezwSometimes sentences have a dash - like this one!
                    A hyphen connects 2 words with no gap: easy-peasy.)Z	sometimesÚ	sentencesrB   rC   ZdashÚlikeÚthisZoneZhyphenZconnectsÚ2ÚwordsÚwithÚnoZgapz
easy-peasyc                 C   s   t  | ¡|ksJ ‚d S ©N)r   Zbag_of_wordsr5   r   r   r   Útest_bag_of_wordsô   s    8rM   ))z9The  dogloved the cat, but	
    the cat	loved the
 cow

r-   )z5

Hello    my	name	is H a r p e r, 
what's yourname?r.   r/   )z¸L     is for the way you look at me
            O    is for the only one I see
            V    is very, very extraordinary
            E    is even more than anyone that you adore canz‡L is for the way you look at me O is for the only one I see V is very, very extraordinary E is even more than anyone that you adore can)a�  
            | Name    | Age | City         | Occupation     |
            |---------|-----|--------------|----------------|
            | Alice   | 30  | New York     | Engineer       |
            | Bob     | 25  | Los Angeles  | Designer       |
            | Charlie | 35  | Chicago      | Teacher        |
            | David   | 40  | San Francisco| Developer      |
            zî| Name | Age | City | Occupation | |---------|-----|--------------|----------------| | Alice | 30 | New York | Engineer | | Bob | 25 | Los Angeles | Designer | | Charlie | 35 | Chicago | Teacher | | David | 40 | San Francisco| Developer |c                 C   s,   t j| dd�|ksJ ‚t  | ¡| ks(J ‚d S )NT)r   )r   Zprepare_strr5   r   r   r   Útest_prepare_string/  s    $rN   ©Z
input_textZexpected_output))ú=She said "Hello" and then whispered 'Goodbye' before leaving.rP   )uK   â€žTo be, or not to be, that is the question" - Shakespeare's famous quote.zI"To be, or not to be, that is the question" - Shakespeare's famous quote.)uA   Â«When he said "life is beautiful," I believed himÂ» wrote Maria.z?"When he said "life is beautiful," I believed him" wrote Maria.)u@   â��Do you remember when we first met?â�ž she asked with a smile.z<"Do you remember when we first met?" she asked with a smile.)uH   ã€�The meeting starts at 10:00, don't be late!ã€Ÿ announced the manager.zD"The meeting starts at 10:00, don't be late!" announced the manager.)u>   ã€ŒHe told me "This is important" yesterdayã€�, she explained.z:'He told me "This is important" yesterday', she explained.)uC   ã€ŽThe sun was setting. The birds were singing. It was peaceful.ã€�z?'The sun was setting. The birds were singing. It was peaceful.')u*   ï¹‚Meeting #123 @ 15:00 - Don't forget!ï¹�z&'Meeting #123 @ 15:00 - Don't forget!')uA   ã€ŒHelloã€�, â��Worldâ�ž, "Test", 'Example', â€žQuote", Â«FinalÂ»z5'Hello', "World", "Test", 'Example', "Quote", "Final")úIt's John's book, isn't it?rQ   )u7   â€¹Testing the system's capability for "quoted" textâ€ºz3'Testing the system's capability for "quoted" text')u6   â�›First sentence. Second sentence. Third sentence.â�œz2'First sentence. Second sentence. Third sentence.')uH   ã€ŒChapter 1ã€�: â��The Beginningâ�ž - â€žA new story" begins Â«todayÂ».z<'Chapter 1': "The Beginning" - "A new story" begins "today".c                 C   s   t  | ¡|ksJ ‚d S rL   )r   Zstandardize_quotesrO   r   r   r   Útest_standardize_quotesW  s    ArR   ©Zoutput_textZsource_textZexpected_percentage))Úextrar	   r   )r	   zSource text has a sentence.r9   )z'The original s e n t e n c e is normal.z"The original sentence is normal...gš™™™™™É?)z'We saw 23% improvement in this quarter.z-We saw 23% improvement in sales this quarter.g      À?)rK   z4Is it possible to have more than everything missing?r9   c                 C   s   t  | |¡|ksJ ‚d S rL   )r   Zcalculate_percent_missing_textrS   r   r   r   Ú#test_calculate_percent_missing_text›  s     
ÿÿrU   )Útable_as_cellsÚexpected_extractionzMonth A.)ÚxÚyÚwÚhÚcontentZ22)Z	row_indexZ	col_indexr\   z0Simple table, 1 head cell, 1 body cell, no spans)ÚidzMonth B.zMonth C.Z11Z12Z13Z21Z23z0Simple table, 3 head cell, 5 body cell, no spansZh12col1)rY   rX   rZ   r[   r\   Zh1col23é   Zh1col4Zh2col2Zh2col34Zr3col1Zr3col2Zr34col34Zr4col12z various spans, with 2 row headerc                 C   s"   dd| idœ}t |ƒ|ksJ ‚d S )NÚTablerV   ©ÚtypeÚmetadata)r   )rV   rW   Úexample_elementr   r   r   Ú+test_cells_table_extraction_from_predictionÀ  s
     (þrd   )Útext_as_htmlrW   z½
<table>
    <thead>
        <tr>
            <th>Month A.</th>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>22</td>
        </tr>
    </tbody>
</table>"
            a€  
<table>
    <thead>
        <tr>
            <th>Month A.</th>
            <th>Month B.</th>
            <th>Month C.</th>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>11</td>
            <td>12</td>
            <td>13</td>
        </tr>
        <tr>
            <td>21</td>
            <td>22</td>
            <td>23</td>
        </tr>
    </tbody>
</table>"
áú  
<table>
    <thead>
        <tr>
            <th rowspan="2">h12col1</th>
            <th colspan="2">h1col23</th>
            <th>h1col4</th>
        </tr>
        <tr>
            <th>h2col2</th>
            <th colspan="2">h2col34</th>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>r3col1</td>
            <td>r3col2</td>
            <td colspan="2" rowspan="2">r34col34</td>
        </tr>
        <tr>
            <td colspan="2">r4col12</td>
        </tr>
    </tbody>
</table>
c                 C   s"   dd| idœ}t |ƒ|ksJ ‚d S )Nr_   re   r`   )r   )re   rW   rc   r   r   r   Ú*test_html_table_extraction_from_predictionm  s     ÿþrg   c                  C   s4   ddg dœdœ} t | ƒd u s J ‚t| ƒd u s0J ‚d S )Nr_   r	   )re   rV   r`   )r   r   )rc   r   r   r   Ú=test_cells_extraction_from_prediction_when_missing_prediction
  s    rh   )ÚhtmlÚreturnc                 C   s   dd„ |   d¡D ƒ}d |¡S )Nc                 S   s   g | ]}|r|  ¡ ‘qS r   )Ústrip)r!   Úliner   r   r   r#     r$   z_trim_html.<locals>.<listcomp>r   r	   )Úsplitr   )ri   Z
html_linesr   r   r   Ú
_trim_html  s    rn   Úhtml_to_test)z°
<table>
    <thead>
        <tr>
            <th>Month A.</th>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>22</td>
        </tr>
    </tbody>
</table>
a  
<table>
    <thead>
        <tr>
            <th>Month A.</th>
            <th>Month B.</th>
            <th>Month C.</th>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>11</td>
            <td>12</td>
            <td>13</td>
        </tr>
        <tr>
            <td>21</td>
            <td>22</td>
            <td>23</td>
        </tr>
    </tbody>
</table>
rf   c                 C   s$   t | ƒ}t|ƒ}t| ƒ|ks J ‚d S rL   )r   r   rn   )ro   Zdeckerd_tableZ
html_tabler   r   r   Útest_deckerd_html_converter  s    Drp   )r   ZpytestZunstructured.metricsr   Z+unstructured.metrics.table.table_extractionr   r   r   r   Zunstructured.partition.autor   r   ÚmarkZparametrizer+   r3   r4   rM   rN   rR   rU   Úparamrd   rg   rh   r    rn   rp   r   r   r   r   Ú<module>   s@  ]þþ
þ+
!þþþõþðýãþ7
þ#
þ@
þ
þ

þ÷÷








÷é"ûûûûûûûûûÇCýýýýýýýýý×/�Òþ &


þí








÷Ý.ýýýýýýýýý×/·¼þ 

þC