a
    ¡˜ëjÍ  ã                	   @  s   d Z ddlmZ ddlZddlZddlZddlZddlmZm	Z	 ddl
Z
ddlmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZmZmZmZmZmZm Z m!Z! dd
l"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddœdd„Z)ddœdd„Z*dd„ Z+dd„ Z,ddœdd„Z-dd„ Z.e
j/ 0dddg¡d d!œd"d#„ƒZ1e
j/ 0dg d$¢¡d d!œd%d&„ƒZ2e
j/ 0dddg¡d d!œd'd(„ƒZ3e
j/ 0dg d$¢¡d d!œd)d*„ƒZ4e
j/ 0dddg¡d d!œd+d,„ƒZ5e
j/ 0dg d$¢¡d d!œd-d.„ƒZ6d/d0„ Z7d1d2„ Z8ddœd3d4„Z9ddœd5d6„Z:ddœd7d8„Z;d9d:„ Z<d;d<„ Z=d=d>„ Z>e
j/ 0d?d@dAgd@fd@g dBfdCdBdAgdBfg¡dDdEdFdEdGœdHdI„ƒZ?dJdK„ Z@dLdM„ ZAdNdO„ ZBdPdQ„ ZCdRdS„ ZDe
j/ 0dTdUdVg¡d d dTœdWdX„ƒZEdYdZ„ ZFd[d\„ ZGd]d^„ ZHd_d`„ ZIdadb„ ZJdcdd„ ZKdedf„ ZLdgdh„ ZMdidj„ ZNdkdl„ ZOdmdn„ ZPdodp„ ZQdqdr„ ZRdsdt„ ZSdudv„ ZTdwdx„ ZUdydz„ ZVd{d|„ ZWe
j/ 0d}g d~¢¡d dœd€d�„ƒZXd‚dƒ„ ZYd„d…„ ZZd†d‡„ Z[dˆd‰„ Z\dŠd‹„ Z]dŒd�„ Z^dŽd�„ Z_d�d‘„ Z`d’d“„ Zad”d•„ Zbd–d—„ Zcd˜d™œdšd›„Zddœd�„ ZedždŸ„ Zfd d¡„ Zge
j/ 0d¢d£d¤g¡d d d¢œd¥d¦„ƒZhddœd§d¨„Zid©dª„ Zje!d«d¬�gZkG d­d®„ d®ƒZle
jmdDd¯œd°d±„ƒZne
jmd²d™œd³d´„ƒZoG dµd¶„ d¶ƒZpG d·d¸„ d¸ƒZqe
j/ 0d¹g dº¢¡d»d¼„ ƒZrdS )½z>Test suite for `unstructured.partition.html.partition` module.é    )ÚannotationsN)ÚAnyÚOptional)Úetree)ÚFixtureRequestÚMockÚassert_round_trips_through_JSONÚexample_doc_pathÚexample_doc_textÚfunction_mock)Úchunk_by_title)Úclean_extra_whitespace)	ÚAddressÚCompositeElementÚElementTypeÚListItemÚNarrativeTextÚTableÚ
TableChunkÚTextÚTitle)Úread_txt_file©Úpartition_html)ÚHtmlPartitionerOptionsÚ_HtmlPartitionerzpathlib.Path)Útmp_pathc              	   C  s¨   t | d ƒ}t|dƒ�}| d¡ W d   ƒ n1 s60    Y  t|ƒ}t|ƒdksXJ ‚|tdƒtdƒtdƒtdƒtd	ƒtd
ƒtdƒgksŽJ ‚t	dd„ |D ƒƒs¤J ‚d S )Núsample-doc.htmlÚwaâ  <html>
  <body>
    <h1>A Great and Glorious Section</h1>
    <p>Dear Leader is the best. He is such a wonderful engineer!</p>
    <p></p>
    <p>Another Magnificent paragraph</p>
    <p><b>The prior element is a title based on its capitalization patterns!</b></p>
    <table>
      <tbody>
        <tr>
          <td><p>I'm in a table</p></td>
        </tr>
      </tbody>
    </table>
    <h2>A New Beginning</h2>
    <div>Here is the start of a new page.</div>
  </body>
</html>
é   zA Great and Glorious Sectionz9Dear Leader is the best. He is such a wonderful engineer!zAnother Magnificent paragraphzBThe prior element is a title based on its capitalization patterns!zI'm in a tablezA New Beginningz Here is the start of a new page.c                 s  s   | ]}|j jd kV  qdS )r   N©ÚmetadataÚfilename©Ú.0Úe© r&   ún/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/html/test_partition.pyÚ	<genexpr>T   ó    z:test_partition_html_accepts_a_file_path.<locals>.<genexpr>)
ÚstrÚopenÚwriter   Úlenr   r   r   r   Úall)r   Ú	file_pathÚfÚelementsr&   r&   r'   Ú'test_partition_html_accepts_a_file_path0   s"    ÿ"ù
	r2   c                 C  sF   | d }|  tdƒ¡ | d¡ tt| ¡ ƒd�}t|ƒdksBJ ‚d S )Nzexample-10k-readonly.htmlúexample-10k-1p.htmli$  ©r"   r   )Ú
write_textr
   Úchmodr   r*   Úresolver-   )r   Zread_only_file_pathr1   r&   r&   r'   Ú:test_user_without_file_write_permission_can_partition_htmlW   s
    
r8   c                  C  sb   t tdƒdƒ�} t| d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚tdd„ |D ƒƒs^J ‚d S )Nr3   Úrb©Úfiler   c                 s  s   | ]}|j jd u V  qd S ©Nr    r#   r&   r&   r'   r(   f   r)   zAtest_partition_html_accepts_a_file_like_object.<locals>.<genexpr>©r+   r	   r   r-   r.   ©r0   r1   r&   r&   r'   Ú.test_partition_html_accepts_a_file_like_objecta   s    (r?   c                  C  s"   t tdƒd�} t| ƒdksJ ‚d S )Nr3   ©Útextr   )r   r
   r-   ©r1   r&   r&   r'   Ú'test_partition_html_accepts_an_html_stri   s    rC   r   ©Úrequests_get_c                 C  sF   t tdƒdddid�| _tdd�}| jdi dd	� t|ƒd
ksBJ ‚d S )Nr3   éÈ   úContent-Typeú	text/html©rA   Ústatus_codeÚheadersúhttps://fake.url©ÚurlT©rK   Úverifyr   )ÚFakeResponser
   Úreturn_valuer   Úassert_called_once_withr-   ©rE   r1   r&   r&   r'   Ú5test_partition_html_accepts_a_url_to_an_HTML_documentn   s    ý
rU   c                   C  s8   t jtdd�� tƒ  W d   ƒ n1 s*0    Y  d S )Nz6Exactly one of filename, file, text, or url must be sp©Úmatch)ÚpytestÚraisesÚ
ValueErrorr   r&   r&   r&   r'   ÚKtest_partition_html_raises_when_no_path_or_file_or_text_or_url_is_specified{   s    r[   r"   úexample-10k-utf-16.htmlú&example-steelJIS-datasheet-utf-16.htmlr*   r4   c              	   C  sh   t  t¡�J tt| ƒdƒ�}t|dd� W d   ƒ n1 s<0    Y  W d   ƒ n1 sZ0    Y  d S )Nr9   úutf-8©r;   Úencoding©rX   rY   ÚUnicodeDecodeErrorr+   r	   r   ©r"   r0   r&   r&   r'   ÚHtest_partition_html_from_filename_raises_when_explicit_encoding_is_wrongƒ   s    rd   )r\   r]   úfake-html-lang-de.htmlc                   sN   t tˆ ƒƒ}t|ƒdksJ ‚t‡ fdd„|D ƒƒs6J ‚ˆ dkrJ|tksJJ ‚d S )Nr   c                 3  s   | ]}|j jˆ kV  qd S r<   r    r#   r4   r&   r'   r(   ”   r)   zEtest_partition_html_from_filename_default_encoding.<locals>.<genexpr>re   )r   r	   r-   r.   ÚEXPECTED_OUTPUT_LANGUAGE_DE)r"   r1   r&   r4   r'   Ú2test_partition_html_from_filename_default_encodingŒ   s
    rg   c                 C  sz   t t| ƒdƒ�}t | ¡ ¡}W d   ƒ n1 s20    Y  tjtdd�� t|dd� W d   ƒ n1 sl0    Y  d S )Nr9   ú,'utf-8' codec can't decode byte 0xff in posirV   r^   r_   )	r+   r	   ÚioÚBytesIOÚreadrX   rY   rb   r   )r"   r0   r;   r&   r&   r'   Ú3test_partition_html_from_file_raises_encoding_error™   s    ,rl   c                 C  s`   t t| ƒdƒ�}t|d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚| dkr\|tks\J ‚d S ©Nr9   r:   r   re   ©r+   r	   r   r-   rf   ©r"   r0   r1   r&   r&   r'   Ú.test_partition_html_from_file_default_encoding¤   s
    (rp   c              	   C  sl   t jtdd��J tt| ƒdƒ�}t|dd� W d   ƒ n1 s@0    Y  W d   ƒ n1 s^0    Y  d S )Nrh   rV   r9   r^   r_   ra   rc   r&   r&   r'   Ú6test_partition_html_from_file_rb_raises_encoding_error±   s    rq   c                 C  s`   t t| ƒdƒ�}t|d�}W d   ƒ n1 s.0    Y  t|ƒdksHJ ‚| dkr\|tks\J ‚d S rm   rn   ro   r&   r&   r'   Ú1test_partition_html_from_file_rb_default_encodingº   s
    (rr   c                  C  s$   d} t | d�}|d jdks J ‚d S )Nu+   <html><div><p>æ¯�æ—¥æ–°é—»</p></div></html>r@   r   u   æ¯�æ—¥æ–°é—»)r   rA   ©Ú	html_textr1   r&   r&   r'   Ú/test_partition_html_processes_chinese_chractersÇ   s    
ru   c                   C  s   t dd�tdƒgksJ ‚d S )Nz3<html charset="utf-8"><p>Hello &#128512;</p></html>r@   u
   Hello ðŸ˜€©r   r   r&   r&   r&   r'   Ú'test_emoji_appears_with_emoji_utf8_codeÍ   s    ÿrw   c                 C  sT   t tdƒdddid�| _tjtdd�� tdd	� W d   ƒ n1 sF0    Y  d S )
Nr3   iô  rG   rH   rI   z-Error status code on GET of provided URL: 500rV   rL   rM   ©rQ   r
   rR   rX   rY   rZ   r   rD   r&   r&   r'   ÚCtest_partition_html_from_url_raises_on_failure_response_status_codeÖ   s    ýry   c                 C  sT   t tdƒdddid�| _tjtdd�� tdd	� W d   ƒ n1 sF0    Y  d S )
Nr3   rF   rG   zapplication/jsonrI   z6Expected content type text/html. Got application/json.rV   rL   rM   rx   rD   r&   r&   r'   ÚEtest_partition_html_from_url_raises_on_response_of_wrong_content_typeá   s    ýrz   c                 C  s<   t ddddid�| _tdddid	� | jdddid
d� d S )NzS<html><head></head><body><p>What do I know? Who needs to know it?</p></body></html>rF   rG   rH   rI   zhttps://example.comz
User-AgentÚtest)rN   rK   TrO   )rQ   rR   r   rS   rD   r&   r&   r'   Ú<test_partition_from_url_includes_provided_headers_in_requestì   s    ý
ÿr|   c                  C  sh   t tdƒƒ} t| ƒdksJ ‚| d }|tdƒks4J ‚|jjd u sDJ ‚|jjd u sTJ ‚|jjd usdJ ‚d S )Nzideas-page.htmlé   r   a  January 2023 ( Someone fed my essays into GPT to make something that could answer
questions based on them, then asked it where good ideas come from.  The
answer was ok, but not what I would have said. This is what I would have said.) The way to get new ideas is to notice anomalies: what seems strange,
or missing, or broken? You can see anomalies in everyday life (much
of standup comedy is based on this), but the best place to look for
them is at the frontiers of knowledge. Knowledge grows fractally.
From a distance its edges look smooth, but when you learn enough
to get close to one, you'll notice it's full of gaps. These gaps
will seem obvious; it will seem inexplicable that no one has tried
x or wondered about y. In the best case, exploring such gaps yields
whole new fractal buds.)r   r	   r-   r   r!   Úemphasized_text_contentsÚ	link_urlsÚtext_as_html©r1   r%   r&   r&   r'   Ú!test_partition_html_on_ideas_pageÿ   s    ÿ
r‚   c                  C  s&   t tdƒƒ} tdd„ | D ƒƒs"J ‚d S )Nzexample-with-scripts.htmlc                 s  s   | ]}d |j vV  qdS )z
function (Nr@   r#   r&   r&   r'   r(     r)   z?test_it_does_not_extract_text_in_script_tags.<locals>.<genexpr>)r   r	   r.   rB   r&   r&   r'   Ú,test_it_does_not_extract_text_in_script_tags  s    rƒ   c                  C  s0   d} t | d�\}t|tƒsJ ‚|jdks,J ‚d S )Nzc<html>
<body>
  <p><style> p { margin:0; padding:0; } </style>Lorem ipsum dolor</p>
</body>
</html>r@   zLorem ipsum dolor)r   Ú
isinstancer   rA   ©rt   Úelementr&   r&   r'   Ú+test_it_does_not_extract_text_in_style_tags  s
    ÿr‡   )Úextract_to_payloadÚextract_typesÚexpect_base64TZImageF)TNFúdict[str, Any]ÚboolzOptional[list[str]])Ú	opts_argsrˆ   r‰   rŠ   c           
      C  sÀ   d}d| }d}d|› d|› d�}|| d< || d< || d	< t f i | ¤Ž}tt |¡ƒ\}	|	jtjkshJ ‚|	j|ksvJ ‚|rœ|	jj	|ksŠJ ‚|	jj
d
ks¼J ‚n |	jj	d u s¬J ‚|	jj
d u s¼J ‚d S )NztiVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==zdata:image/png;base64,zBase64 Imageú*
    <div class="Page">
        <img src="ú" alt="ú">
    </div>
    rA   Zextract_image_block_to_payloadZextract_image_block_typesz	image/png)r   Úlistr   Úiter_elementsÚcategoryr   ÚIMAGErA   r!   Zimage_base64Zimage_mime_type)
r�   rˆ   r‰   rŠ   Úbase64ÚsrcÚalt_textÚhtmlÚoptsr†   r&   r&   r'   Ú%test_partition_html_base64_for_images1  s*    ÿþþrš   c                  C  sX   d} d}d| › d|› d�}t |d�\}|jtjks6J ‚|j|ksDJ ‚|jj| ksTJ ‚d S )Nzhttps://example.com/image.pngz	URL ImagerŽ   r�   r�   r@   )r   r“   r   r”   rA   r!   Ú	image_url)r›   r—   r˜   Úimager&   r&   r'   Ú+test_partition_html_includes_url_for_images\  s    þþÿr�   c                  C  s@   d} t | d�\}t|tƒsJ ‚|jdks,J ‚|jjdks<J ‚dS )z@Bare-bones means no `<thead>`, `<tbody>`, or `<tfoot>` elements.z›<html>
<body>
  <table>
    <tr><td>Lorem</td><td>Ipsum</td></tr>
    <tr><td>Ut enim non</td><td>ad minim
veniam quis</td></tr>
  </table>
</body>
</html>r@   z,Lorem Ipsum Ut enim non ad minim
veniam quiszr<table><tr><td>Lorem</td><td>Ipsum</td></tr><tr><td>Ut enim non</td><td>ad minim<br/>veniam quis</td></tr></table>N)r   r„   r   rA   r!   r€   r…   r&   r&   r'   Ú7test_it_can_parse_a_bare_bones_table_to_a_Table_elementp  s    ÿÿrž   c                  C  s2   d} t | d�\}t|tƒsJ ‚|jjdks.J ‚dS )a	  Cells within a `table/thead` element are included in the text and html.

    The presence of a `<thead>` element in the original also determines whether a `<thead>` element
    appears in `.text_as_html` or whether the first row of cells is simply in the body.
    áW  <html>
<body>
  <table>
    <thead>
      <tr><th>Lorem</th><th>Ipsum</th></tr>
    </thead>
    <tbody>
      <tr><th>Lorem ipsum</th><td>dolor sit amet nulla</td></tr>
      <tr><th>Ut enim non</th><td>ad minim
veniam quis</td></tr>
    </tbody>
    <tfoot>
      <tr><th>Dolor</th><td>Equis</td></tr>
    </tfoot>
  </table>
</body>
</html>r@   zÑ<table><tr><td>Lorem</td><td>Ipsum</td></tr><tr><td>Lorem ipsum</td><td>dolor sit amet nulla</td></tr><tr><td>Ut enim non</td><td>ad minim<br/>veniam quis</td></tr><tr><td>Dolor</td><td>Equis</td></tr></table>N)r   r„   r   r!   r€   r…   r&   r&   r'   ÚTtest_it_accommodates_column_heading_cells_enclosed_in_thead_tbody_and_tfoot_elements‹  s    ÿÿr    c                  C  s   d} t | d�g ksJ ‚d S )Nzv<html>
<body>
  <table>
    <tr><td> </td><td> </td></tr>
    <tr><td> </td><td> </td></tr>
  </table>
</body>
</html>r@   r   )rt   r&   r&   r'   Ú>test_it_does_not_emit_a_Table_element_for_a_table_with_no_text±  s    ÿr¡   c                  C  sZ   d} t | d�\}|jj}|d us$J ‚t |t ¡ ¡}|d us@J ‚tj|td�dksVJ ‚d S )NrŸ   r@   )r`   zë<html><body><table><tr><td>Lorem</td><td>Ipsum</td></tr><tr><td>Lorem ipsum</td><td>dolor sit amet nulla</td></tr><tr><td>Ut enim non</td><td>ad minim<br/>veniam quis</td></tr><tr><td>Dolor</td><td>Equis</td></tr></table></body></html>)r   r!   r€   r   Ú
fromstringÚ
HTMLParserÚtostringr*   )rt   r†   r€   r˜   r&   r&   r'   Ú/test_it_provides_parseable_HTML_in_text_as_htmlÀ  s    ÿÿr¥   )ÚtagÚexpected_text_as_html)Útheadú:<table><tr><td>Header 1</td><td>Header 2</td></tr></table>)Útfootr©   c                 C  s0   t d| › d| › d�d�}|d jj|ks,J ‚d S )Nz<table>
  <z6>
    <tr><th>Header 1</th><th>Header 2</th></tr>
  </z
>
</table>r@   r   )r   r!   r€   )r¦   r§   r1   r&   r&   r'   Ú.test_partition_html_parses_table_without_tbodyé  s    ÿýþ	r«   c                  C  s4   d} t | d�\}|tdƒks J ‚|jjdks0J ‚d S )Nzà<table>
 <tr>
  <td>
   <table>
     <tr><td>foo</td><td>bar</td></tr>
     <tr><td>baz</td><td>bng</td></tr>
   </table>
  </td>
  <td>
   <table>
     <tr><td>fizz</td><td>bang</td></tr>
   </table>
  </td>
 </tr>
</table>r@   zfoo bar baz bng fizz bangzB<table><tr><td>foo bar baz bng</td><td>fizz bang</td></tr></table>©r   r   r!   r€   r…   r&   r&   r'   ÚZtest_partition_html_reduces_a_nested_table_to_its_text_placed_in_the_cell_that_contains_itý  s    ÿÿr­   c                  C  s4   d} t | d�\}|tdƒks J ‚|jjdks0J ‚dS )z)Like this example from an SEC 10k filing.aà  <table>
 <tr>
  <td></td>
  <td></td>
 </tr>
 <tr>
  <td>
   <p>
    <span>
     <ix:nonNumeric id="F_be4cc145-372a-4689-be60-d8a70b0c8b9a" contextRef="C_1de69f73-df01-4830-8af0-0f11b469bc4a" name="dei:DocumentAnnualReport" format="ixt-sec:boolballotbox">
     <span>&#9746;</span>
     </ix:nonNumeric>
    </span>
   </p>
  </td>
  <td>
   <p>
    <span>ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934</span>
   </p>
  </td>
 </tr>
</table>
r@   uX   â˜’ ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934u”   <table><tr><td/><td/></tr><tr><td>â˜’</td><td>ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934</td></tr></table>Nr¬   r…   r&   r&   r'   Ú8test_partition_html_accommodates_tds_with_child_elements  s    ÿÿ
ÿr®   c                  C  sl  d} t | d�}t|ƒdksJ ‚|d }|tdƒks6J ‚|jjd u sFJ ‚|d }|tdƒks^J ‚|jjdksnJ ‚|d }|td	ƒks†J ‚|jjdks–J ‚|d
 }|tdƒks®J ‚|jjdks¾J ‚|d }|tdƒksÖJ ‚|jjd
ksæJ ‚|d }|tdƒksþJ ‚|jjdk�sJ ‚|d }|tdƒk�s*J ‚|jjdk�s<J ‚|d }|tdƒk�sVJ ‚|jjdk�shJ ‚d S )Naa  <p>This is narrative text, it's long, flows and has meaning</p>
<h1>This heading is a title, even though it's long, flows and has meaning</h1>
<h2>A heading that is at the second level</h2>
<h3>Finally, the third heading</h3>
<h4>December 1-17, 2017</h4>
<h5>email@example.com</h5>
<h6>* bullet point</h6>
<h3><li>- invalidly nested list item</li></h3>
r@   é   r   z8This is narrative text, it's long, flows and has meaningr}   zEThis heading is a title, even though it's long, flows and has meaningé   z%A heading that is at the second levelé   zFinally, the third headingé   zDecember 1-17, 2017é   zemail@example.comé   z* bullet pointr   z- invalidly nested list item)r   r-   r   r!   Úcategory_depthr   r   ©rt   r1   r%   r&   r&   r'   ÚDtest_partition_html_recognizes_h1_to_h6_as_Title_with_category_depthG  s8    ÿ
r·   c                  C  s„   t tdƒƒ} tdt| ƒ›�ƒ t| ƒdks.J ‚t| d jƒ d¡sFJ ‚t| d tƒsXJ ‚| d j	j
dkslJ ‚| d j	jdks€J ‚d S )Nzfake-html-pre.htmzlen(elements)=r   z[107th Congress Public Law 56]rH   )r   r	   Úprintr-   r   rA   Ú
startswithr„   r   r!   Zfiletyper"   rB   r&   r&   r'   Ú4test_partition_html_with_widely_encompassing_pre_tagp  s    rº   c                   C  s.   t dd�tdƒtdƒtdƒtdƒgks*J ‚d S )Nz‘<pre>The Big Brown Bear</pre>
<div>The big brown bear is growling.</div>
<pre>The big brown bear is sleeping.</pre>
<div>The Big Blue Bear</div>
r@   zThe Big Brown BearzThe big brown bear is growling.zThe big brown bear is sleeping.zThe Big Blue Bear)r   r   r   r&   r&   r&   r'   Ú#test_pre_tag_parsing_respects_order{  s    þüùr»   c                  C  sf   d} t | d�}|tdƒtdƒtdƒtdƒgks2J ‚|d }|jjg d¢ksNJ ‚|jjg d	¢ksbJ ‚d S )
Na*  <!DOCTYPE html>
<html>
<body>
<div>
  <h1>Header 1</h1>
  <p>Text</p>
  <h2>Header 2</h2>
  <pre>
    <b>Param1</b> = Y<br><b>Param2</b> = 1<br><b>Param3</b> = 2<br><b>Param4</b> = A
    <br><b>Param5</b> = A,B,C,D,E<br><b>Param6</b> = 7<br><b>Param7</b> = Five<br>
  </pre>
</div>
</body>
</html>
r@   zHeader 1r   zHeader 2zd    Param1 = Y
Param2 = 1
Param3 = 2
Param4 = A
    
Param5 = A,B,C,D,E
Param6 = 7
Param7 = Five

  r±   )ZParam1ZParam2ZParam3ZParam4ZParam5ZParam6ZParam7)Úbr¼   r¼   r¼   r¼   r¼   r¼   )r   r   r   r!   r~   Úemphasized_text_tagsr¶   r&   r&   r'   Ú"test_partition_html_br_tag_parsing‹  s    ÿ
ÿü

	r¾   c                  C  s0   d} t | d�}|tdƒtdƒtdƒgks,J ‚d S )Nz`<html>
<body>
<div>
    Head
    <div><span>Nested</span></div>
    Tail
</div>
</body>
</html>
r@   ZHeadZNestedZTailrv   rs   r&   r&   r'   Ú$test_partition_html_tag_tail_parsing¶  s    ÿ
r¿   c                   C  s   t dd�g ksJ ‚d S )NÚ r@   r   r&   r&   r&   r'   Ú5test_partition_html_from_text_works_with_empty_stringË  s    rÁ   c                  C  s\   d} t | d�}|tdƒtdƒtdƒgks,J ‚|d jjdgksBJ ‚|d jjdgksXJ ‚d S )	Na�  
    <div>
      We start out normally
      <cite>
        and then add a citation
        <p>But whoa, this is a paragraph inside a phrasing element.</p>
        so we close the first element at the start of the block element and emit it, then we
        <b>emit</b> the block element,
      </cite>
      and then start a new element for the tail and whatever phrasing follows it.
    </div>
    r@   z-We start out normally and then add a citationz8But whoa, this is a paragraph inside a phrasing element.z¸so we close the first element at the start of the block element and emit it, then we emit the block element, and then start a new element for the tail and whatever phrasing follows it.r°   Úemitr¼   )r   r   r!   r~   r½   rs   r&   r&   r'   ÚJtest_partition_html_accommodates_block_item_nested_inside_phrasing_elementÏ  s    
ÿý
	rÃ   c                  C  s”   d} t | d�}dd„ |D ƒg d¢ks(J ‚|d }|jjdgksBJ ‚|jjdgksTJ ‚td	d
„ |dd … D ƒƒsrJ ‚tdd
„ |dd … D ƒƒs�J ‚d S )Nzé
    <div>
      O Deep Thought
      <a href="http://eie.io">
        computer, he said,
        <p>The task we have designed you to perform is this.</p>
        We want you to tell us....
      </a>
      he paused,
    </div>
    r@   c                 S  s   g | ]
}|j ‘qS r&   r@   r#   r&   r&   r'   Ú
<listcomp>û  r)   zMtest_partition_html_handles_anchor_with_nested_block_item.<locals>.<listcomp>)z!O Deep Thought computer, he said,z1The task we have designed you to perform is this.z%We want you to tell us.... he paused,r   zcomputer, he said,zhttp://eie.ioc                 s  s   | ]}|j jd u V  qd S r<   )r!   Ú
link_textsr#   r&   r&   r'   r(     r)   zLtest_partition_html_handles_anchor_with_nested_block_item.<locals>.<genexpr>r}   c                 s  s   | ]}|j jd u V  qd S r<   )r!   r   r#   r&   r&   r'   r(     r)   )r   r!   rÅ   r   r.   )rt   r1   Zlink_annotated_elementr&   r&   r'   Ú9test_partition_html_handles_anchor_with_nested_block_itemì  s    
rÆ   c                  C  sT   d} t | d�}|tdƒtdƒtdƒtdƒtdƒtdƒtd	ƒtd
ƒtdƒg	ksPJ ‚d S )Na’  <div dir=3D"ltr">Hi All,
  <div><br></div>
  <div>Get excited for our first annual family day!</div>
  <div>Best.<br clear="all">
    <div><br></div>
    -- <br>
    <div dir=3D"ltr">
      <div dir=3D"ltr">Dino the Datasaur<div>
      Unstructured Technologies<br>
      <div>Data Scientist</div>
      <div>Doylestown, PA 18901</div>
      <div><br></div>
    </div>
  </div>
  See you there!
</div>
r@   zHi All,z,Get excited for our first annual family day!zBest.z--zDino the DatasaurzUnstructured TechnologieszData ScientistzDoylestown, PA 18901zSee you there!)r   r   r   r   rs   r&   r&   r'   Ú'test_containers_with_text_are_processed  s    ÿ
÷rÇ   c                  C  s*   d} t | d�}|tdƒtdƒgks&J ‚d S )Nz’<html>
  <body>
    <ol>
      <li>Happy Groundhog's day!</li>
      <li>Looks like six more weeks of winter ...</li>
    </ol>
  </body>
</html>
r@   úHappy Groundhog's day!ú'Looks like six more weeks of winter ...©r   r   rs   r&   r&   r'   Ú%test_html_grabs_bulleted_text_in_tags*  s    ÿ
þrË   c                  C  s*   d} t | d�}|tdƒtdƒgks&J ‚d S )Nz¹<html>
  <body>
    <p>
      <span>&#8226; Happy Groundhog's day!</span>
    </p>
    <p>
      <span>&#8226; Looks like six more weeks of winter ...</span>
    </p>
  </body>
</html>
r@   rÈ   rÉ   rÊ   rs   r&   r&   r'   Ú&test_html_grabs_bulleted_text_in_paras>  s    ÿ
þrÌ   c                  C  s    t dd�} | tdƒgksJ ‚d S )Nz&<p>Hello again peet mag<i>ic</i>al</p>r@   zHello again peet magicalrv   rB   r&   r&   r'   Útest_joins_tag_text_correctlyU  s    
rÍ   c                  C  s    t dd�} | tdƒgksJ ‚d S )Nu8   <html charset="unicode">
<p>Hello again ðŸ˜€</p>
</html>r@   u   Hello again ðŸ˜€)r   r   rB   r&   r&   r'   Útest_sample_doc_with_emojiZ  s    
rÎ   c                  C  s    t dd�} | tdƒgksJ ‚d S )Nz<body>Hello</body>r@   ÚHellorv   rB   r&   r&   r'   Ú&test_only_text_and_no_elements_in_body_  s    
rÐ   c                  C  s&   t dd�} | tdƒtdƒgks"J ‚d S )Nz<body>Hello<p>World</p></body>r@   rÏ   ZWorldrv   rB   r&   r&   r'   Ú!test_text_before_elements_in_bodyd  s    
rÑ   c                  C  s    t dd�} | tdƒgksJ ‚d S )Nz<div>Hello<br/>World</div>r@   zHello Worldrv   rB   r&   r&   r'   Útest_line_break_in_containeri  s    
rÒ   r¦   )ÚdelÚformÚnoscript)r¦   c                 C  s,   d| › d| › d�}t |d�}|g ks(J ‚d S )Nz
<body>
  <z#>
    There is some text here.
  </z
>
</body>
r@   r   )r¦   rt   r1   r&   r&   r'   Útest_exclude_tag_typesn  s    
rÖ   c                  C  sF   t dƒ} t| dd�}tt| ƒƒ}tdd„ |D ƒƒs6J ‚||ksBJ ‚d S )Nr3   Zby_title)Zchunking_strategyc                 s  s   | ]}t |tttfƒV  qd S r<   )r„   r   r   r   )r$   Úcr&   r&   r'   r(   ‚  r)   zCtest_partition_html_can_chunk_while_partitioning.<locals>.<genexpr>)r	   r   r   r.   )r/   ÚchunksZchunks_2r&   r&   r'   Ú0test_partition_html_can_chunk_while_partitioning~  s
    rÙ   c                   C  s$   t ddd�tdƒtdƒgks J ‚d S )NzÐ<html>
  <header>
    <p>Header</p>
  </header>
  <body>
    <h1>My First Heading</h1>
    <p>It was a dark and stormy night. No one was around.</p>
  </body>
  <footer>
    <p>Footer</p>
  </footer>
</html>
T)rA   Úskip_headers_and_footerszMy First Headingz2It was a dark and stormy night. No one was around.)r   r   r   r&   r&   r&   r'   Ú0test_partition_html_can_skip_headers_and_footers‰  s    ñþðrÛ   c                  C  s2   dd„ t tdƒƒD ƒ} t| ƒtt| ƒƒks.J ‚d S )Nc                 S  s   g | ]
}|j ‘qS r&   ©Úidr#   r&   r&   r'   rÄ   ¤  r)   z3test_all_element_ids_are_unique.<locals>.<listcomp>z&fake-html-with-duplicate-elements.html)r   r	   r-   Úset)Úidsr&   r&   r'   Útest_all_element_ids_are_unique£  s    rà   c                  C  s4   dd„ t dƒD ƒ} dd„ t dƒD ƒ}| |ks0J ‚d S )Nc                 S  s   g | ]
}|j ‘qS r&   rÜ   r#   r&   r&   r'   rÄ   ©  r)   z6test_element_ids_are_deterministic.<locals>.<listcomp>z3example-docs/fake-html-with-duplicate-elements.htmlc                 S  s   g | ]
}|j ‘qS r&   rÜ   r#   r&   r&   r'   rÄ   ª  r)   r   )rß   Zids_2r&   r&   r'   Ú"test_element_ids_are_deterministic¨  s    rá   c                  C  sæ  t dd�} t| ƒdksJ ‚| d }t|tƒs0J ‚|jdks>J ‚|jjd u sNJ ‚|jjd u s^J ‚| d }t|tƒstJ ‚|jdks‚J ‚|jjdks’J ‚|jjd u s¢J ‚| d }t|tƒs¸J ‚|jd	ksÆJ ‚|jjd u sÖJ ‚|jj| d j	ksìJ ‚| d
 }t|t
ƒ�sJ ‚|jdk�sJ ‚|jjdk�s&J ‚|jj| d j	k�s>J ‚| d }t|t
ƒ�sVJ ‚|jdk�sfJ ‚|jjdk�sxJ ‚|jj| d j	k�s�J ‚| d }t|tƒ�s¨J ‚|jdk�s¸J ‚|jjd u �sÊJ ‚|jj| d j	k�sâJ ‚d S )NaŠ  <html>
  <p>Preamble gets no category_depth or parent_id</p>
  <h1>Heading gets category_depth but no parent_id</h1>
  <p>Body paragraph gets parent_id but no category_depth</p>
  <ul>
    <li>List item gets category_depth and parent_id</li>
    <li>Second list item gets category_depth and parent_id</li>
  </ul>
  <p>Body paragraph after list gets parent_id but no category_depth</p>
</html>
r@   r´   r   z,Preamble gets no category_depth or parent_idr}   z,Heading gets category_depth but no parent_idr°   z3Body paragraph gets parent_id but no category_depthr±   z+List item gets category_depth and parent_idr²   z2Second list item gets category_depth and parent_idr³   z>Body paragraph after list gets parent_id but no category_depth)r   r-   r„   r   rA   r!   rµ   Z	parent_idr   rÝ   r   r�   r&   r&   r'   Ú.test_partition_html_records_hierarchy_metadataµ  sD    þrâ   c                  C  s<  t dd�} | d }|tdƒks"J ‚|jjdgks4J ‚|jjdgksFJ ‚| d }|tdƒks^J ‚|jjg d	¢ksrJ ‚|jjg d
¢ks†J ‚| d }|tdƒksžJ ‚|jjdgks°J ‚|jjdgksÂJ ‚| d }|tdƒksÚJ ‚|jjd u sêJ ‚|jjd u súJ ‚| d }|tdƒk�sJ ‚|jjd u �s&J ‚|jjd u �s8J ‚d S )Nzú<html>
  <p>Hello there I am a very <strong>important</strong> text!</p>
  <p>Here is a <span>list</span> of <b>my <i>favorite</i> things</b></p>
  <ul>
    <li><em>Parrots</em></li>
    <li>Dogs</li>
  </ul>
  <span>A lone span text!</span>
</html>
r@   r   z'Hello there I am a very important text!Z	importantr¼   r}   ú$Here is a list of my favorite things)ZmyZfavoriteZthings)r¼   Zbir¼   r°   ÚParrotsÚir±   ÚDogsr²   zA lone span text!)r   r   r!   r~   r½   r   r   r�   r&   r&   r'   Ú*test_partition_html_grabs_emphasized_textsé  s.    þrç   c                  C  sL   t tdƒƒ} t| ƒdksJ ‚tdd„ | D ƒƒs2J ‚tdd„ | D ƒƒsHJ ‚d S )Nr3   r   c                 s  s   | ]}|j jd kV  qdS )r3   Nr    r#   r&   r&   r'   r(     r)   zatest_partition_html_from_filename_uses_source_filename_for_metadata_by_default.<locals>.<genexpr>c                 s  s   | ]}|j jtd ƒkV  qdS )rÀ   N)r!   Zfile_directoryr	   r#   r&   r&   r'   r(     r)   ©r   r	   r-   r.   rB   r&   r&   r'   ÚNtest_partition_html_from_filename_uses_source_filename_for_metadata_by_default  s    ré   c                  C  s:   t tdƒdd�} t| ƒdks J ‚tdd„ | D ƒƒs6J ‚d S )Nr3   r{   )Úmetadata_filenamer   c                 s  s   | ]}|j jd kV  qdS ©r{   Nr    )r$   r†   r&   r&   r'   r(     r)   zNtest_partition_html_from_filename_prefers_metadata_filename.<locals>.<genexpr>rè   rB   r&   r&   r'   Ú;test_partition_html_from_filename_prefers_metadata_filename  s    rì   c                  C  sd   t tdƒdƒ�} t| dd�}W d   ƒ n1 s00    Y  t|ƒdksJJ ‚tdd„ |D ƒƒs`J ‚d S )Nr3   r9   r{   )r;   rê   r   c                 s  s   | ]}|j jd kV  qdS rë   r    r#   r&   r&   r'   r(   $  r)   zJtest_partition_html_from_file_prefers_metadata_filename.<locals>.<genexpr>r=   r>   r&   r&   r'   Ú7test_partition_html_from_file_prefers_metadata_filename  s    *rí   c                  C  s&   t tdƒƒ} | d jjdgks"J ‚d S )Nr3   r   Úeng)r   r	   r!   Ú	languagesrB   r&   r&   r'   Ú2test_partition_html_element_metadata_has_languages*  s    rð   c                  C  s@   t tdƒdd�} dd„ | D ƒdgddgdgdgdggks<J ‚d S )Nzlanguage-docs/eng_spa_mult.htmlT)Zdetect_language_per_elementc                 S  s   g | ]}|j j‘qS r&   )r!   rï   r#   r&   r&   r'   rÄ   4  r)   zLtest_partition_html_respects_detect_language_per_element.<locals>.<listcomp>rî   Zspa)r   r	   rB   r&   r&   r'   Ú8test_partition_html_respects_detect_language_per_element/  s    ÿûrñ   r   ©Úrequestc                 C  sJ   t | ddd�}tdƒ}t|ƒ}| |¡ |s0J ‚tdd„ |D ƒƒsFJ ‚d S )Nú<unstructured.partition.html.partition.get_last_modified_dateú2024-06-17T22:22:20)rR   úfake-html.htmlc                 s  s   | ]}|j jd kV  qdS )rõ   N©r!   Úlast_modifiedr#   r&   r&   r'   r(   L  r)   zXtest_partition_html_from_filename_pulls_last_modified_from_filesystem.<locals>.<genexpr>)r   r	   r   rS   r.   )ró   Úget_last_modified_date_r/   r1   r&   r&   r'   ÚEtest_partition_html_from_filename_pulls_last_modified_from_filesystem@  s    ý
rú   c                  C  s<   t tdƒdd�} t| d tƒs"J ‚tdd„ | D ƒƒs8J ‚d S )Nrö   ú2023-07-05T09:24:28)Zmetadata_last_modifiedr   c                 s  s   | ]}|j jd kV  qdS )rû   Nr÷   r#   r&   r&   r'   r(   U  r)   zStest_partition_html_from_filename_prefers_metadata_last_modified.<locals>.<genexpr>)r   r	   r„   r   r.   rB   r&   r&   r'   Ú@test_partition_html_from_filename_prefers_metadata_last_modifiedO  s
    ÿrü   c                  C  s<  d} t | d�}|d }|tdƒks&J ‚|jjdgks8J ‚|jjdgksJJ ‚|d }|tdƒksbJ ‚|jjd u srJ ‚|jjd u s‚J ‚|d	 }|td
ƒksšJ ‚|jjdgks¬J ‚|jjd
gks¾J ‚|d }|tdƒksÖJ ‚|jjd u sæJ ‚|jjd u söJ ‚|d }|tdƒk�sJ ‚|jjdgk�s$J ‚|jjdgk�s8J ‚d S )Na  <html>
  <p>Hello there I am a <a href="/link">very important link!</a></p>
  <p>Here is a list of my favorite things</p>
  <ul>
    <li><a href="https://en.wikipedia.org/wiki/Parrot">Parrots</a></li>
    <li>Dogs</li>
  </ul>
  <a href="/loner">A lone link!</a>
</html>
r@   r   z'Hello there I am a very important link!ú/linkzvery important link!r}   rã   r°   rä   z$https://en.wikipedia.org/wiki/Parrotr±   ræ   r²   úA lone link!ú/loner)r   r   r!   r   rÅ   r   r   r¶   r&   r&   r'   Útest_partition_html_grabs_links[  s.    ÿ
r   c                  C  sÆ   d} t | d�}|d }|jjdgks(J ‚|jjdgks:J ‚|d }|jjdgksTJ ‚|jjdgksfJ ‚|d	 }|jjdgks€J ‚|jjdgks’J ‚|d
 }|jjddgks®J ‚|jjddgksÂJ ‚d S )NzØ<html>
  <a href="/loner">A lone link!</a>
  <p>Hello <a href="/link">link!</a></p>
  <p>
   Hello <a href="/link">link!</a></p>
  <p><a href="/wiki/parrots">Parrots</a> and <a href="/wiki/dogs">Dogs</a></p>
</html>
r@   r   rþ   rÿ   r}   zlink!rý   r°   r±   rä   ræ   z/wiki/parrotsz
/wiki/dogs)r   r!   rÅ   r   r¶   r&   r&   r'   Útest_partition_html_links€  s    ÿ	
r  )rt   Úexpected_value)z:<table><tr><th>Header 1</th><th>Header 2</th></tr></table>r©   )zd<table><tr><td>Dimensions</td><td>Weight</td></tr><tr><td>4'-6" x 1'</td><td>18 kg</td></tr></table>zs<table><tr><td>Dimensions</td><td>Weight</td></tr><tr><td>4&#x27;-6&quot; x 1&#x27;</td><td>18 kg</td></tr></table>c                 C  s2   t | d�}t|ƒdksJ ‚|d jj|ks.J ‚d S )Nr@   r}   r   )r   r-   r!   r€   )rt   r  r1   r&   r&   r'   Ú<test_partition_html_applies_text_as_html_metadata_for_tables�  s    
r  c                 C  s\   t tdƒdddid�| _tdd�}| jdi dd	� t|ƒd
ksBJ ‚tdd„ |D ƒƒsXJ ‚d S )Nr3   rF   rG   rH   rI   úhttps://trusttheforceluke.comrM   TrO   r   c                 s  s   | ]}|j jd kV  qdS )r  N)r!   rN   r#   r&   r&   r'   r(   È  r)   zDtest_partition_html_from_url_adds_url_to_metadata.<locals>.<genexpr>)rQ   r
   rR   r   rS   r-   r.   rT   r&   r&   r'   Ú1test_partition_html_from_url_adds_url_to_metadata½  s    ý
r  c                  C  s   t tdƒƒ} t| ƒ d S )Nr3   )r   r	   r   rB   r&   r&   r'   Ú,test_partition_html_round_trips_through_jsonÐ  s    r  uD   Jahresabschluss zum GeschÃ¤ftsjahr vom 01.01.2020 bis zum 31.12.2020r@   c                   @  s"   e Zd Zi fddddœdd„ZdS )rQ   r*   Úintzdict[str, str]rI   c                 C  s    || _ || _|dk | _|| _d S )Ni,  )rA   rJ   ÚokrK   )ÚselfrA   rJ   rK   r&   r&   r'   Ú__init__ß  s    
zFakeResponse.__init__N)Ú__name__Ú
__module__Ú__qualname__r
  r&   r&   r&   r'   rQ   Þ  s   rQ   )Úreturnc                
   C  s   dddddi ddddœ	S )z¼All default arguments for `HtmlPartitionerOptions`.

    Individual argument values can be changed to suit each test. Makes construction of opts more
    compact for testing purposes.
    NTF)	r;   r/   rA   r`   rN   rK   Z
ssl_verifyrÚ   Údetection_originr&   r&   r&   r&   r'   r�   æ  s    ÷r�   zpytest.FixtureRequestc                 C  s
   t | dƒS )Nz2unstructured.partition.html.partition.requests.get©r   rò   r&   r&   r'   rE   ú  s    rE   c                   @  sÒ   e Zd ZdZej dddg¡dddœdd	„ƒZdd
œdd„Zdd
œdd„Z	dd
œdd„Z
dddœdd„Zdd
œdd„Zdddœdd„Zdd
œdd„Zej dddg¡ddd œd!d"„ƒZe ¡ d#dd$œd%d&„ƒZdS )'ÚDescribeHtmlPartitionerOptionszSUnit-test suite for `unstructured.partition.html.partition.HtmlPartitionerOptions`.r  r˜   Nz
str | Noner‹   )r  r�   c                 C  s(   ||d< t f i |¤Ž}|j|ks$J ‚d S )Nr  )r   r  )r	  r  r�   r™   r&   r&   r'   Ú-it_knows_the_caller_provided_detection_origin  s    zLDescribeHtmlPartitionerOptions.it_knows_the_caller_provided_detection_origin©r�   c                 C  sJ   t dƒ}||d< tf i |¤Ž}|j}t|tƒs2J ‚|t|ƒd ksFJ ‚d S )Nr3   r/   r}   )r	   r   rt   r„   r*   r   )r	  r�   r/   r™   rt   r&   r&   r'   Ú8it_gets_the_HTML_from_the_file_path_when_one_is_provided  s    zWDescribeHtmlPartitionerOptions.it_gets_the_HTML_from_the_file_path_when_one_is_providedc                 C  s‚   t dƒ}t|dƒ�}t | ¡ ¡}W d   ƒ n1 s60    Y  ||d< tf i |¤Ž}|j}t|tƒsjJ ‚|t	|ƒd ks~J ‚d S )Nr3   r9   r;   r}   )
r	   r+   ri   rj   rk   r   rt   r„   r*   r   )r	  r�   r/   r0   r;   r™   rt   r&   r&   r'   ÚCand_it_gets_the_HTML_from_the_file_like_object_when_one_is_provided!  s    ,zbDescribeHtmlPartitionerOptions.and_it_gets_the_HTML_from_the_file_like_object_when_one_is_providedc                 C  s(   d|d< t f i |¤Ž}|jdks$J ‚d S )Nz-<html><body><p>Hello World!</p></body></html>rA   )r   rt   ©r	  r�   r™   r&   r&   r'   Ú?and_it_uses_the_HTML_in_the_text_argument_when_that_is_provided/  s    z^DescribeHtmlPartitionerOptions.and_it_uses_the_HTML_in_the_text_argument_when_that_is_providedr   )rE   r�   c                 C  s<   t ddddid�|_d|d< tf i |¤Ž}|jdks8J ‚d S )Nz?<html><body><p>I just flew over the internet!</p></body></html>rF   rG   rH   rI   zhttps://insta.tweet.face.orgrN   )rQ   rR   r   rt   )r	  rE   r�   r™   r&   r&   r'   Ú6and_it_gets_the_HTML_from_the_url_when_one_is_provided7  s    ýzUDescribeHtmlPartitionerOptions.and_it_gets_the_HTML_from_the_url_when_one_is_providedc                 C  sF   t f i |¤Ž}tjtdd�� |j W d   ƒ n1 s80    Y  d S )Nz3Exactly one of filename, file, text, or url must berV   )r   rX   rY   rZ   rt   r  r&   r&   r'   Ú>but_it_raises_when_no_path_or_file_or_text_or_url_was_providedD  s    z]DescribeHtmlPartitionerOptions.but_it_raises_when_no_path_or_file_or_text_or_url_was_provided)r�   rù   c                 C  s<   d|d< d|_ tf i |¤Ž}|j}| d¡ |dks8J ‚d S )Nza/b/document.htmlr/   z2024-04-02T20:32:35)rR   r   rø   rS   )r	  r�   rù   r™   rø   r&   r&   r'   ÚDit_gets_last_modified_from_the_filesystem_when_file_path_is_providedN  s    
zcDescribeHtmlPartitionerOptions.it_gets_last_modified_from_the_filesystem_when_file_path_is_providedc                 C  s6   t  d¡}||d< tf i |¤Ž}|j}|d u s2J ‚d S )Ns   abcdefgr;   )ri   rj   r   rø   )r	  r�   r;   r™   rø   r&   r&   r'   ÚRbut_it_falls_back_to_None_for_the_last_modified_date_when_no_file_path_is_providedZ  s
    
zqDescribeHtmlPartitionerOptions.but_it_falls_back_to_None_for_the_last_modified_date_when_no_file_path_is_providedrÚ   TFrŒ   )rÚ   r�   c                 C  s(   ||d< t f i |¤Ž}|j|u s$J ‚d S )NrÚ   )r   rÚ   )r	  rÚ   r�   r™   r&   r&   r'   Ú=it_knows_the_caller_provided_skip_headers_and_footers_settingg  s    z\DescribeHtmlPartitionerOptions.it_knows_the_caller_provided_skip_headers_and_footers_settingr   )ró   r  c                 C  s
   t |dƒS )Nrô   r  )r	  ró   r&   r&   r'   rù   r  s    ÿz6DescribeHtmlPartitionerOptions.get_last_modified_date_)r  r  r  Ú__doc__rX   ÚmarkÚparametrizer  r  r  r  r  r  r  r  r  Úfixturerù   r&   r&   r&   r'   r    s   



r  c                   @  sr   e Zd ZdZddœdd„Zddœdd„Zddœdd	„Zddœd
d„Zddœdd„Zddœdd„Z	ddœdd„Z
dS )ÚDescribe_HtmlPartitionerzMUnit-test suite for `unstructured.partition.html.partition._HtmlPartitioner`.r‹   r  c                 C  s2   d|d< t f i |¤Ž}t|ƒ}|jjdks.J ‚d S )Nz­<body>
  <header></header>
  <p>Lots preamble stuff yada yada yada</p>
  <main>
    <h2>A Wonderful Section!</h2>
    <p>Look at this amazing section!</p>
  </main>
</body>
rA   Úmain©r   r   Ú_mainr¦   ©r	  r�   r™   Zpartitionerr&   r&   r'   Ú,it_can_find_the_main_element_in_the_document~  s
    ÿ
zEDescribe_HtmlPartitioner.it_can_find_the_main_element_in_the_documentc                 C  s2   d|d< t f i |¤Ž}t|ƒ}|jjdks.J ‚dS )zQAnd there is always a <body>, the parser adds one if there's not one in the HTML.z–<body>
  <header></header>
  <p>Lots preamble stuff yada yada yada</p>
  <h2>A Wonderful Section!</h2>
  <p>Look at this amazing section!</p>
</body>
rA   ÚbodyNr#  r%  r&   r&   r'   Ú3and_it_falls_back_to_the_body_when_there_is_no_main�  s
    ÿzLDescribe_HtmlPartitioner.and_it_falls_back_to_the_body_when_there_is_no_mainc                 C  s:   d|d< t f i |¤Ž}tt |¡ƒ\}|tdƒks6J ‚d S )Nz<p>NO PARTICULAR TYPE.</p>rA   zNO PARTICULAR TYPE.)r   r‘   r   r’   r   ©r	  r�   r™   r†   r&   r&   r'   ÚMit_produces_a_Text_element_when_the_tag_contents_are_not_narrative_or_a_title¡  s    zfDescribe_HtmlPartitioner.it_produces_a_Text_element_when_the_tag_contents_are_not_narrative_or_a_titlec                 C  s:   d|d< t f i |¤Ž}tt |¡ƒ\}|tdƒks6J ‚d S )Nu   <p>â—� An excellent point!</p>rA   zAn excellent point!)r   r‘   r   r’   r   r)  r&   r&   r'   ÚWit_produces_a_ListItem_element_when_the_tag_contains_are_preceded_by_a_bullet_character«  s    zpDescribe_HtmlPartitioner.it_produces_a_ListItem_element_when_the_tag_contains_are_preceded_by_a_bullet_characterc                 C  s0   d|d< t f i |¤Ž}tt |¡ƒg ks,J ‚d S )Nu
   <p>â—�</p>rA   ©r   r‘   r   r’   r  r&   r&   r'   ÚAbut_not_when_the_tag_contains_only_a_bullet_character_and_no_textµ  s    zZDescribe_HtmlPartitioner.but_not_when_the_tag_contains_only_a_bullet_character_and_no_textc                 C  s0   d|d< t f i |¤Ž}tt |¡ƒg ks,J ‚d S )Nz<p></p>rA   r,  r  r&   r&   r'   Ú2it_produces_no_element_when_the_tag_has_no_content½  s    zKDescribe_HtmlPartitioner.it_produces_no_element_when_the_tag_has_no_contentc                 C  s0   d|d< t f i |¤Ž}tt |¡ƒg ks,J ‚d S )Nz<p>$</p>rA   r,  r  r&   r&   r'   Ú<and_it_produces_no_element_when_the_tag_contains_only_a_stubÃ  s    zUDescribe_HtmlPartitioner.and_it_produces_no_element_when_the_tag_contains_only_a_stubN)r  r  r  r  r&  r(  r*  r+  r-  r.  r/  r&   r&   r&   r'   r!  y  s   

r!  )Ú	test_caseÚcontent))Ú
empty_filerÀ   )Úempty_bytesr)   )Zwhitespace_onlyz
   
	  
  c              	   C  s    | dkrt  |¡}t|d� n~tjdddd��(}| |¡ | ¡  |j}W d  ƒ n1 s\0    Y  z(t|d�}t|ƒd	ks‚J ‚W t	 
|¡ nt	 
|¡ 0 dS )
a³  Test that partitioning empty/whitespace-only HTML content won't
    raise TypeError: Invalid input object: NoneType.

    This reproduces the production error where lxml.etree.strip_elements is called with None
    when the HTML content is empty, causing etree.fromstring to return None.

    Args:
        test_case: Description of the test scenario
        content: The content to test (empty string, empty bytes, or whitespace)
    r3  r:   r   z.htmlF)ÚmodeÚsuffixÚdeleteNr4   r   )ri   rj   r   ÚtempfileÚNamedTemporaryFiler,   ÚflushÚnamer-   ÚosÚunlink)r0  r1  r2  r0   Ztemp_filenamer1   r&   r&   r'   Ú3test_partition_html_with_empty_content_raises_errorÌ  s    

$
r=  )sr  Ú
__future__r   ri   r;  Úpathlibr7  Útypingr   r   rX   Úlxmlr   Ztest_unstructured.unit_utilsr   r   r   r	   r
   r   Zunstructured.chunking.titler   Zunstructured.cleaners.corer   Zunstructured.documents.elementsr   r   r   r   r   r   r   r   r   Z unstructured.file_utils.encodingr   Zunstructured.partition.htmlr   Z%unstructured.partition.html.partitionr   r   r2   r8   r?   rC   rU   r[   r  r  rd   rg   rl   rp   rq   rr   ru   rw   ry   rz   r|   r‚   rƒ   r‡   rš   r�   rž   r    r¡   r¥   r«   r­   r®   r·   rº   r»   r¾   r¿   rÁ   rÃ   rÆ   rÇ   rË   rÌ   rÍ   rÎ   rÐ   rÑ   rÒ   rÖ   rÙ   rÛ   rà   rá   râ   rç   ré   rì   rí   rð   rñ   rú   rü   r   r  r  r  r  rf   rQ   r   r�   rE   r  r!  r=  r&   r&   r&   r'   Ú<module>   s   ,	'
ÿþ	ÿþ	ÿþ		

üþ	"&)þþ/)+#4'%ûþ
ÿrSþ