a
    ¡˜ëj€*  ã                   @  s2  d Z ddlmZ ddlmZmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z" eej#ƒed#ddddi ddddddddœddddddddddddddddœdd„ƒƒZ$G dd „ d ƒZ%G d!d"„ d"ƒZ&dS )$zProvides `partition_html().é    )Úannotations)ÚIOÚAnyÚIteratorÚListÚLiteralÚOptionalÚcastN)Úetree)Úadd_chunking_strategy)ÚElementÚElementType)Úread_txt_file)ÚFileType)Úapply_metadataÚget_last_modified_date)ÚFlowÚhtml_parser)Ú!ontology_to_unstructured_elementsÚparse_html_to_ontology)Úis_temp_file_pathÚlazypropertyTFÚv1Úto_text)ÚfileÚtextÚencodingÚurlÚheadersÚ
ssl_verifyÚskip_headers_and_footersÚdetection_originÚhtml_parser_versionÚimage_alt_modeÚextract_image_block_to_payloadÚextract_image_block_typeszOptional[str]zOptional[IO[bytes]]údict[str, str]ÚboolúLiteral['v1', 'v2']úOptional[Literal['to_text']]úOptional[list[str]]r   zlist[Element])Úfilenamer   r   r   r   r   r   r    r!   r"   r#   r$   r%   ÚkwargsÚreturnc                K  sT   |dur$|  ¡ dkr$|s$| s$|s$g S t| |||||||||	|
||d�}tt |¡ƒS )aó  Partitions an HTML document into its constituent elements.

    HTML source parameters
    ----------------------
    The HTML to be partitioned can be specified four different ways:

    filename
        A string defining the target filename path.
    file
        A file-like object using "r" mode --> open(filename, "r").
    text
        The string representation of the HTML document.
    url
        The URL of a webpage to parse. Only for URLs that return an HTML document.
    headers
        The HTTP headers to be used in the HTTP request when `url` is specified.
    ssl_verify
        If the URL parameter is set, determines whether or not SSL verification is performed
        on the HTTP request.
    encoding
        The encoding method used to decode the text input. If None, utf-8 will be used.
    skip_headers_and_footers
        If True, ignores any content that is within <header> or <footer> tags

    html_parser_version (Literal['v1', 'v2']):
        The version of the HTML parser to use. The default is 'v1'. For 'v2' the parser will
        use the ontology schema to parse the HTML document.

    image_alt_mode (Literal['to_text']):
        When set 'to_text', the v2 parser will include the alternative text of images in the output.
    NÚ ©Ú	file_pathr   r   r   r   r   r   r    r!   r"   r#   r%   r$   )ÚstripÚHtmlPartitionerOptionsÚlistÚ_HtmlPartitionerÚiter_elements)r+   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r,   Úopts© r7   úd/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/html/partition.pyÚpartition_html   s$    3 ór9   c                   @  s®   e Zd ZdZdddddœdddddd	d
d
ddddd
dœdd„Zeddœdd„ƒZeddœdd„ƒZeddœdd„ƒZed
dœdd„ƒZ	eddœdd„ƒZ
ed
dœdd„ƒZdS )r2   zVEncapsulates partitioning option validation, computation, and application of defaults.r   r   NF)r"   r#   r%   r$   z
str | NonezIO[bytes] | Noner&   r'   r(   r)   r*   r/   c                C  sR   || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|| _
|| _|| _d S ©N)Ú
_file_pathÚ_fileÚ_textÚ	_encodingÚ_urlÚ_headersÚ_ssl_verifyÚ_skip_headers_and_footersÚ_detection_originÚ_html_parser_versionÚ_image_alt_modeÚ_extract_image_block_typesÚ_extract_image_block_to_payload)Úselfr0   r   r   r   r   r   r   r    r!   r"   r#   r%   r$   r7   r7   r8   Ú__init__e   s    zHtmlPartitionerOptions.__init__©r-   c                 C  s   | j S )zOTrace of initial partitioner to be included in metadata for debugging purposes.)rC   ©rH   r7   r7   r8   r!   „   s    z'HtmlPartitionerOptions.detection_originÚstrc                 C  s°   | j rt| j | jd�d S | jr4t| j| jd�d S | jrDt| jƒS | jr¤tj| j| j	| j
d�}|jsvtd|j› �ƒ‚|j dd¡}| d¡sžtd	|› d
�ƒ‚|jS tdƒ‚dS )zIThe HTML document as a string, loaded from wherever the caller specified.)r+   r   é   )r   r   )r   Úverifyz*Error status code on GET of provided URL: zContent-Typer.   z	text/htmlz%Expected content type text/html. Got Ú.z>Exactly one of filename, file, text, or url must be specified.N)r;   r   r>   r<   r=   rL   r?   ÚrequestsÚgetr@   rA   ÚokÚ
ValueErrorÚstatus_coder   Ú
startswithr   )rH   ÚresponseÚcontent_typer7   r7   r8   Ú	html_text‰   s"    

ÿ
z HtmlPartitionerOptions.html_textc                 C  s   | j rt| j ƒrdS t| j ƒS )zHThe best last-modified date available, None if no sources are available.N)r;   r   r   rK   r7   r7   r8   Úlast_modified£   s    ÿÿýz$HtmlPartitionerOptions.last_modifiedc                 C  s   | j S )zAWhen True, elements located within a header or footer are pruned.)rB   rK   r7   r7   r8   r    ¬   s    z/HtmlPartitionerOptions.skip_headers_and_footersc                 C  s   | j S )zEWhen html_parser_version=='v2', HTML elements follow ontology schema.)rD   rK   r7   r7   r8   r"   ±   s    z*HtmlPartitionerOptions.html_parser_versionc                 C  s
   | j dkS )zDWhen True, the alternative text of images is included in the output.r   )rE   rK   r7   r7   r8   Úadd_img_alt_text¶   s    z'HtmlPartitionerOptions.add_img_alt_text)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rI   r   r!   rX   rY   r    r"   rZ   r7   r7   r7   r8   r2   b   s$   ñ*r2   c                   @  st   e Zd ZdZddœdd„Zdddœd	d
„Zedddœdd„ƒZddœdd„Ze	ddœdd„ƒZ
e	ddœdd„ƒZdS )r4   z/Partition HTML document into document-elements.r2   )r6   c                 C  s
   || _ d S r:   )Ú_opts)rH   r6   r7   r7   r8   rI   ¿   s    z_HtmlPartitioner.__init__r   r'   )Úelementr-   c                 C  s,   |j tjko*| jjo*| jjduo*d| jjv S )zGDetermines if an image_base64 element should be included in the output.NZImage)Úcategoryr   ZIMAGEr_   rG   rF   )rH   r`   r7   r7   r8   Ú_should_include_image_base64Â   s    ÿ
þ
üz-_HtmlPartitioner._should_include_image_base64zIterator[Element])r6   r-   c                 c  s   | |ƒ  ¡ E dH  dS )zBPartition HTML document provided by `opts` into document-elements.N)Ú_iter_elements)Úclsr6   r7   r7   r8   r5   Ë   s    z_HtmlPartitioner.iter_elementsrJ   c                 c  s~   | j j}|r| ¡ dkrdS | j jdkr2| j ¡ n| j}|D ]<}| j j|j_| j j	|j_	|  
|¡srd|j_d|j_|V  q<dS )zˆGenerated document-elements (e.g. Title, NarrativeText, etc.) parsed from document.

        Elements appear in document order.
        r.   Nr   )r_   rX   r1   r"   Ú_mainr5   Ú_from_ontologyrY   Úmetadatar!   rb   Zimage_base64Zimage_mime_type)rH   rX   Zelements_iterÚer7   r7   r8   rc   Ð   s    
ÿý
z_HtmlPartitioner._iter_elementsr   c                 C  s°   | j j}zt |t¡}W n$ ty<   t | d¡t¡}Y n0 tj|g d¢dd� | j jrntj|ddgdd� | 	d¡ }durŠt
t|ƒS | 	d	¡ }dur¦t
t|ƒS t
t|ƒS )
zThe root HTML element.zutf-8)ÚdelÚlinkÚmetaÚnoscriptÚscriptÚstyleF)Z	with_tailÚheaderÚfooterz.//mainNz.//body)r_   rX   r
   Ú
fromstringr   rS   ÚencodeZstrip_elementsr    Úfindr	   r   )rH   rX   ÚrootÚmainÚbodyr7   r7   r8   re   ê   s    
ÿ

z_HtmlPartitioner._mainzList[Element]c                 C  s8   | j j}|r| ¡ dkrg S t|ƒ}t|| j jd�}|S )zHConvert an ontology elements represented in HTML to an ontology element.r.   )rZ   )r_   rX   r1   r   r   rZ   )rH   rX   ZontologyZunstructured_elementsr7   r7   r8   rf     s    ÿz_HtmlPartitioner._from_ontologyN)r[   r\   r]   r^   rI   rb   Úclassmethodr5   rc   r   re   rf   r7   r7   r7   r8   r4   ¼   s   	!r4   )N)'r^   Ú
__future__r   Útypingr   r   r   r   r   r   r	   rP   Úlxmlr
   Zunstructured.chunkingr   Zunstructured.documents.elementsr   r   Z unstructured.file_utils.encodingr   Zunstructured.file_utils.modelr   Z&unstructured.partition.common.metadatar   r   Z"unstructured.partition.html.parserr   r   Z+unstructured.partition.html.transformationsr   r   Zunstructured.utilsr   r   ZHTMLr9   r2   r4   r7   r7   r7   r8   Ú<module>   s>   $ ÿò2GZ