a
    ¡˜ëjËF  ã                   @   s  d dl mZ d dlmZmZ d dlZd dlmZmZ d dl	m
Z
 d dlmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd	„ d	eƒZG d
d„ deƒZ ej!edœdd„ƒZ"ej!eedœdd„ƒZ#ej!eedœdd„ƒZ$ej!eedœdd„ƒZ%ej!eedœdd„ƒZ&ej!eedœdd„ƒZ'ej!eedœdd„ƒZ(ej!e)e dœdd„ƒZ*ej!e)e e)e dœdd „ƒZ+ej!e)e dœd!d"„ƒZ,edd#œd$d%„Z-edd#œd&d'„Z.edd#œd(d)„Z/edd#œd*d+„Z0edd#œd,d-„Z1edd#œd.d/„Z2edd0œd1d2„Z3edd3œd4d5„Z4edd3œd6d7„Z5edd8œd9d:„Z6edd;œd<d=„Z7edd>œd?d@„Z8e)e ddœdAdB„Z9e
e)e ddCœdDdE„Z:e
e)e ddFœdGdH„Z;e
e)e ddFœdIdJ„Z<ej=e)e ddKœdLdM„Z>e
e)e ddNœdOdP„Z?e
e)e ddNœdQdR„Z@e
e)e ddFœdSdT„ZAdS )Ué    )Údefaultdict)ÚAnyÚOptionalN)ÚBeautifulSoupÚTag)ÚMockerFixture)ÚElementÚElementMetadataÚElementType)ÚHTML_PARSERÚElementHtmlÚImageElementHtmlÚLinkElementHtmlÚListItemElementHtmlÚTableElementHtmlÚTextElementHtmlÚTitleElementHtmlÚUnorderedListElementHtmlÚ_elements_to_html_tagsÚ_elements_to_html_tags_by_pageÚ _elements_to_html_tags_by_parentÚ_group_element_childrenÚelements_to_htmlÚgroup_elements_by_pagec                   @   s(   e Zd Zdeee eeddœdd„ZdS )ÚMockElementÚ N)ÚtextÚmetadataÚcategoryÚidÚreturnc                 C   s"   || _ |ptƒ | _|| _|| _d S ©N)r   r	   r   r   Z_element_id)Úselfr   r   r   r   © r#   úl/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/html/test_convert.pyÚ__init__   s    zMockElement.__init__)r   Nr   r   )Ú__name__Ú
__module__Ú__qualname__Ústrr   r	   r%   r#   r#   r#   r$   r      s       ûúr   c                
   @   sF   e Zd Zdee ee ee ee ee ee ee ddœdd„ZdS )ÚMockElementMetadataN)Útext_as_htmlÚcategory_depthÚimage_base64Úimage_mime_typeÚurlÚ	parent_idÚpage_numberr    c                 C   s.   || _ || _|| _|| _|| _|| _|| _d S r!   )r+   r,   r-   r.   r/   r0   r1   )r"   r+   r,   r-   r.   r/   r0   r1   r#   r#   r$   r%   -   s    
zMockElementMetadata.__init__)NNNNNNN)r&   r'   r(   r   r)   Úintr%   r#   r#   r#   r$   r*   ,   s"          ø÷r*   )r    c                  C   s   t dd�} td| ddd�S )Nz<p>Test Text</p>©r+   ú	Test Textútest-categoryútest-id©r   r   r   r   )r*   r   )r   r#   r#   r$   Úmock_element@   s    
r8   )r8   r    c                 C   s   t | ƒS r!   )r   ©r8   r#   r#   r$   Úelement_htmlF   s    r:   c                 C   s"   t dd�}td|ddd� t| ƒS )Nz<p>Test HTML</p>r3   r4   r5   r6   r7   )r*   r   r   )r8   r   r#   r#   r$   Útitle_element_htmlK   s    
r;   c                 C   s   t | ƒS r!   )r   r9   r#   r#   r$   Úimage_element_htmlR   s    r<   c                 C   s   t | ƒS r!   )r   r9   r#   r#   r$   Útable_element_htmlW   s    r=   c                 C   s   t | ƒS r!   )r   r9   r#   r#   r$   Úlink_element_html\   s    r>   c                 C   s   t | ƒS r!   )r   r9   r#   r#   r$   Úunordered_list_element_htmla   s    r?   c                   C   s  t tdtdd�tjdd�ƒttdtd d�tjdd�ƒttdtdd�tjd	d�ƒt tdtd	dd
�tjdd�ƒt tdtd	dd
�tjdd�ƒttdtdd�tjdd�ƒt tdtddd
�tjdd�ƒt tdtddd
�tjdd�ƒt tdtddd
�tjdd�ƒttdtddd
�tjdd�ƒg
S )NzTest List Itemé   ©r1   útest-element-1r7   r4   útest-element-2é   útest-element-3)r0   r1   útest-element-4útest-element-5é   útest-element-6zTest List Item Otherútest-element-7útest-element-8útest-element-9útest-element-10)r   r   r*   r
   Ú	LIST_ITEMr   ÚTEXTÚLIST_ITEM_OTHERr#   r#   r#   r$   Úelements_htmlf   s¢    üÿüÿüÿ
üÿ
üÿüÿ
üÿ
üÿ
üÿ
üÿ·rQ   )rQ   r    c                 C   s   dd„ | D ƒS )Nc                 S   s   g | ]
}|j ‘qS r#   )Úelement)Ú.0Úelr#   r#   r$   Ú
<listcomp>¾   ó    zelements.<locals>.<listcomp>r#   )rQ   r#   r#   r$   Úelements¼   s    rW   c                   C   s0   t dtjdtdd�d�t dtjdtdd�d�gS )	NzTest Text 1rB   r@   rA   )r   r   r   r   zTest Text 2rC   rD   )r   r
   rO   r*   r#   r#   r#   r$   Úelements_smallÁ   s    üüùrX   )r:   r    c                 C   s0   t dtƒ}| d¡}|  |¡ |jdks,J ‚d S )Nr   Údivr4   )r   r   Únew_tagÚ_inject_html_element_contentÚstring)r:   ÚsoupÚtagr#   r#   r$   Ú test_inject_html_element_contentÓ   s    


r_   c                 C   s6   |   ¡ }t|tƒsJ ‚|jdks$J ‚|jdks2J ‚d S )NÚpr4   )Zget_text_as_htmlÚ
isinstancer   Únamer\   ©r:   r^   r#   r#   r$   Útest_get_text_as_htmlÚ   s    rd   c                 C   sv   t dtƒ}| d¡}tdd�}t|ƒ}|  |g¡ |  ||¡}|jdksLJ ‚t|j	ƒdks^J ‚|j	d j
dksrJ ‚d S )Nr   rY   ú
Child Text©r   rD   r@   )r   r   rZ   r   r   Úset_childrenÚ_get_children_htmlrb   ÚlenÚcontentsr\   )r:   r]   Z
parent_tagÚchild_elementÚchild_element_htmlÚ
result_tagr#   r#   r$   Útest_get_children_htmlá   s    


rn   c                 C   sV   |   ¡ }t|tƒsJ ‚|jdks$J ‚|jdks2J ‚|d dksBJ ‚|d dksRJ ‚d S )Nr`   r4   Úclassr5   r   r6   )Úget_html_elementra   r   rb   r\   rc   r#   r#   r$   Ú"test_get_html_element_text_as_htmlí   s    rq   c                 C   s`   d | j j_|  ¡ }t|tƒs J ‚|jdks.J ‚|jdks<J ‚|d dksLJ ‚|d dks\J ‚d S )NrY   r4   ro   r5   r   r6   )rR   r   r+   rp   ra   r   rb   r\   rc   r#   r#   r$   Ú%test_get_html_element_no_text_as_htmlö   s    
rr   c                 C   sJ   t dd�}t|ƒ}|  |g¡ t| jƒdks0J ‚| jd jjdksFJ ‚d S )Nre   rf   r@   r   )r   r   rg   ri   ÚchildrenrR   r   )r:   rk   rl   r#   r#   r$   Útest_set_children   s
    
rt   )r;   r    c                 C   s   | j dksJ ‚d S )NÚh1)Zhtml_tag)r;   r#   r#   r$   Útest_title_element_html_tag  s    rv   )r<   r    c                 C   s2   t dtƒ}| d¡}|  |¡ |d dks.J ‚d S )Nr   ÚimgÚaltr4   )r   r   rZ   r[   ©r<   r]   r^   r#   r#   r$   Útest_image_element_html_content  s    


rz   c                 C   sV   d| j j_d| j j_tdtƒ}| d¡}|  |¡ |d dksBJ ‚|d dksRJ ‚d S )	NZ
base64dataz	image/pngr   rw   Úsrcz data:image/png;base64,base64datarx   r4   )rR   r   r-   r.   r   r   rZ   r[   ry   r#   r#   r$   Ú+test_image_element_html_content_with_base64  s    




r|   )r=   r    c                 C   s2   t dtƒ}| d¡}|  |¡ |d dks.J ‚d S )Nr   ÚtableÚstylez3border: 1px solid black; border-collapse: collapse;)r   r   rZ   Ú_inject_html_element_attrs)r=   r]   r^   r#   r#   r$   Útest_table_element_html_attrs  s    


r€   )r>   r    c                 C   s<   d| j j_tdtƒ}| d¡}|  |¡ |d dks8J ‚d S )Nzhttp://example.comr   ÚaÚhref)rR   r   r/   r   r   rZ   r   )r>   r]   r^   r#   r#   r$   Útest_link_element_html_attrs$  s
    



rƒ   )r?   r    c                 C   sŠ   t dtƒ}| d¡}tdd�}t|ƒ}|  |g¡ |  ||¡}|jdksLJ ‚t|j	ƒdks^J ‚|j	d jdksrJ ‚|j	d j
dks†J ‚d S )Nr   Úulre   rf   r@   r   Úli)r   r   rZ   r   r   rg   rh   rb   ri   rj   r\   )r?   r]   r^   rk   rl   rm   r#   r#   r$   Ú test_unordered_list_element_html,  s    


r†   c                 C   s  t | ƒ}t|ƒdksJ ‚t|d jƒdks.J ‚|d jd jjtjksJJ ‚t|d jƒdks`J ‚|d jd jjtjks|J ‚|d jd jjtjks˜J ‚t|d jƒdks®J ‚|d jd jjtjksÊJ ‚|d jd jjtjksæJ ‚|d jd jjtjk�sJ ‚d S )Né   r   r@   rH   rD   é   )r   ri   rs   rR   r   r
   rN   rP   )rQ   Zgrouped_childrenr#   r#   r$   Útest_group_element_children9  s    r‰   )ÚmockerrQ   r    c                 C   s   | j ddd„ d� t|ƒ}t|ƒdks*J ‚|d jjdks>J ‚t|d jƒdksTJ ‚|d jjd	kshJ ‚t|d jƒdks~J ‚|d
 jjdks’J ‚t|d
 jƒd
ks¨J ‚|d
 jd jjdksÂJ ‚|d
 jd jjdksÜJ ‚|d jjdksðJ ‚t|d jƒdk�sJ ‚|d jd jjdk�s$J ‚t|d jd jƒdk�sBJ ‚|d jd jd jjdk�sdJ ‚|d jd jjdk�s€J ‚|d jd
 jjdk�sœJ ‚d S )Nz;unstructured.partition.html.convert._group_element_childrenc                 S   s   | S r!   r#   )rs   r#   r#   r$   Ú<lambda>L  rV   z6test_elements_to_html_tags_by_parent.<locals>.<lambda>©Úside_effecté   r   rB   r@   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   )Úpatchr   ri   rR   r   rs   )rŠ   rQ   Úresultr#   r#   r$   Ú$test_elements_to_html_tags_by_parentG  s*    þ"r‘   )rŠ   rW   r    c                 C   sŽ   t tdœdd„}| jddd„ d� | jd|d	d
� t|ƒ}t|ƒdksJJ ‚tdd„ |D ƒƒs`J ‚t|dd�D ]\}}|jd|› �kslJ ‚qld S )N©r"   Úkwargsc                 [   s   t d| jj› d�tƒ ¡ S )Nú<div>ú</div>)r   rR   r   r   Úfindr’   r#   r#   r$   Ú_mock_get_html_elementb  s    z:test_elements_to_html_tags.<locals>._mock_get_html_elementzDunstructured.partition.html.convert._elements_to_html_tags_by_parentc                 S   s   | S r!   r#   )rW   r#   r#   r$   r‹   g  rV   z,test_elements_to_html_tags.<locals>.<lambda>rŒ   z@unstructured.partition.html.convert.ElementHtml.get_html_elementT)r�   Zautospecé
   c                 s   s   | ]}t |tƒV  qd S r!   ©ra   r   ©rS   r^   r#   r#   r$   Ú	<genexpr>p  rV   z-test_elements_to_html_tags.<locals>.<genexpr>r@   ©Ústartútest-element-)r   r   r�   r   ri   ÚallÚ	enumerater\   )rŠ   rW   r—   r�   ÚirT   r#   r#   r$   Útest_elements_to_html_tagsa  s    þýr¢   c                 C   s¦  t t tdœdd„}t t t t t  dœdd„}| jd|d� | jd	|d� t|ƒ}t|ƒd
ksdJ ‚tdd„ |D ƒƒszJ ‚|d jdksŒJ ‚|d d dks J ‚t|d jƒdks¶J ‚|d jd j	dksÎJ ‚|d jdksàJ ‚|d d dksôJ ‚t|d jƒd
k�sJ ‚t
|d jd
d�D ] \}}|j	d|› �k�sJ ‚�q|d d d
k�sVJ ‚t|d jƒdk�snJ ‚t
|d jdd�D ] \}}|j	d|› �k�s€J ‚�q€d S )N©rW   Ú_c                 S   s   dd„ | D ƒS )Nc                 S   s$   g | ]}t d |j› d�tƒ ¡ ‘qS ©r”   r•   ©r   r   r   r–   ©rS   rR   r#   r#   r$   rU   w  s   z[test_elements_to_html_tags_by_page.<locals>._mock_elements_to_html_tags.<locals>.<listcomp>r#   r£   r#   r#   r$   Ú_mock_elements_to_html_tagsv  s    ÿzGtest_elements_to_html_tags_by_page.<locals>._mock_elements_to_html_tags)rW   r    c                 S   s<   t tƒ}| D ]"}|jjd ur||jj  |¡ qt| ¡ ƒS r!   )r   Úlistr   r1   ÚappendÚvalues)rW   Z
pages_dictrR   r#   r#   r$   Ú_mock_group_elements_by_page{  s
    zHtest_elements_to_html_tags_by_page.<locals>._mock_group_elements_by_pageú:unstructured.partition.html.convert._elements_to_html_tagsrŒ   z:unstructured.partition.html.convert.group_elements_by_pagerH   c                 s   s   | ]}t |tƒV  qd S r!   r™   rš   r#   r#   r$   r›   Œ  rV   z5test_elements_to_html_tags_by_page.<locals>.<genexpr>r   rY   údata-page_numberr@   rB   rD   rœ   rž   rˆ   é   )r©   r   Úboolr�   r   ri   rŸ   rb   rj   r\   r    )rŠ   rW   r¨   r¬   r�   r¡   rT   r#   r#   r$   Ú"test_elements_to_html_tags_by_pageu  s4    þþr±   )ÚcaplogrW   r    c                 C   sØ   t |ƒ}t|ƒdksJ ‚t|d ƒdks,J ‚t|d ƒdks@J ‚t|d ƒdksTJ ‚|d d jdksjJ ‚t|d dd�D ]\}}|jd|› �kszJ ‚qzt|d d	d�D ]\}}|jd|› �ks¨J ‚q¨d
| jv sÔJ ‚d S )NrH   r   r@   rD   rˆ   rB   rœ   rž   r¯   z?Page number is not set for an element test-element-2. Skipping.)r   ri   r   r    r   )r²   rW   r�   r¡   rT   r#   r#   r$   Útest_group_elements_by_pageœ  s    r³   )rŠ   rX   r    c                 C   sv   t t tdœdd„}| jd|d�}|  d¡}t|ddd�}d	|v sFJ ‚d
|v sRJ ‚d|vs^J ‚| |d¡ | ¡  d S )Nr£   c                 S   s   dd„ | D ƒS )Nc                 S   s$   g | ]}t d |j› d�tƒ ¡ ‘qS r¥   r¦   r§   r#   r#   r$   rU   ®  s   z_test_elements_to_html_no_group_by_page.<locals>._mock_elements_to_html_tags.<locals>.<listcomp>r#   r£   r#   r#   r$   r¨   ­  s    ÿzKtest_elements_to_html_no_group_by_page.<locals>._mock_elements_to_html_tagsr­   rŒ   úBunstructured.partition.html.convert._elements_to_html_tags_by_pageT©Zexclude_binary_image_dataZno_group_by_pagez <div>
   test-element-1
  </div>z <div>
   test-element-2
  </div>r®   )r©   r   r°   r�   r   Úassert_called_once_withÚassert_not_called)rŠ   rX   r¨   Úmock_elements_to_html_tagsÚ"mock_elements_to_html_tags_by_pager�   r#   r#   r$   Ú&test_elements_to_html_no_group_by_pageª  s    þÿrº   c                 C   s˜   t t tdœdd„}| jd|d�}|  d¡}t|ddd	�}t|tƒ}| d
ddi¡j 	¡ dksbJ ‚| d
ddi¡j 	¡ dks€J ‚| 
|d¡ | ¡  d S )Nr£   c                 S   s   dd„ | D ƒS )Nc                 S   s.   g | ]&}t d |jj› d|j› d�tƒ ¡ ‘qS )z<div data-page_number='z'>r•   )r   r   r1   r   r   r–   r§   r#   r#   r$   rU   Æ  s
   üþzdtest_elements_to_html_group_by_page.<locals>._mock_elements_to_html_tags_by_page.<locals>.<listcomp>r#   r£   r#   r#   r$   Ú#_mock_elements_to_html_tags_by_pageÅ  s    ûzPtest_elements_to_html_group_by_page.<locals>._mock_elements_to_html_tags_by_pager´   rŒ   r­   TFrµ   rY   r®   Ú1rB   Ú2rC   )r©   r   r°   r�   r   r   r   r–   r\   Ústripr¶   r·   )rŠ   rX   r»   r¹   r¸   r�   r]   r#   r#   r$   Ú#test_elements_to_html_group_by_pageÂ  s     	þÿÿ
r¿   c                 C   sF   |   dd¡ tjtdd�� t|ƒ W d   ƒ n1 s80    Y  d S )Nz1unstructured.partition.html.convert.HTML_TEMPLATEz-<html><head><title>Test</title></head></html>z'Body tag not found in the HTML template)Úmatch)r�   ÚpytestZraisesÚ
ValueErrorr   )rŠ   rW   r#   r#   r$   Ú+test_elements_to_html_invalid_html_templateà  s    þrÃ   )BÚcollectionsr   Útypingr   r   rÁ   Zbs4r   r   Zpytest_mockr   Zunstructured.documents.elementsr   r	   r
   Z#unstructured.partition.html.convertr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r*   Zfixturer8   r:   r;   r<   r=   r>   r?   r©   rQ   rW   rX   r_   rd   rn   rq   rr   rt   rv   rz   r|   r€   rƒ   r†   r‰   r‘   r¢   r±   ZLogCaptureFixturer³   rº   r¿   rÃ   r#   r#   r#   r$   Ú<module>   sp   DU	

þ'þþþ