a
    ¡˜ëjµC  ã                   @  s   U d Z ddlmZ ddlZddlmZ ddlmZmZm	Z	m
Z
 ddlZddlZddlZddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z- dZ.de/d< dZ0de/d< e%e#j1ƒed2ddddddœdddddddddœd d!„ƒƒZ2G d"d#„ d#ƒZ3G d$d%„ d%ƒZ4G d&d'„ d'ƒZ5G d(d)„ d)ƒZ6dd*d+œd,d-„Z7ddd#d.d/œd0d1„Z8dS )3z0Partitioner for Excel 2007+ (XLSX) spreadsheets.é    )ÚannotationsN)ÚSpooledTemporaryFile)ÚIOÚAnyÚIteratorÚOptional)ÚSelfÚ	TypeAlias)Úadd_chunking_strategy)Úclean_bullets)Ú	HtmlTable)ÚElementÚElementMetadataÚListItemÚNarrativeTextÚTableÚTextÚTitle)ÚFileType)Úapply_metadataÚget_last_modified_date)Úis_bulleted_textÚis_possible_narrative_textÚis_possible_numbered_listÚis_possible_title)Úlazypropertyztuple[int, int]r	   Ú_CellCoordinateZxlsxÚstrÚDETECTION_ORIGINTFé   )ÚfileÚfind_subtableÚinclude_headerÚinfer_table_structureÚstarting_page_numberúOptional[str]úOptional[IO[bytes]]ÚboolÚintr   zlist[Element])Úfilenamer    r!   r"   r#   r$   ÚkwargsÚreturnc             	   K  sx  t | ||||d�}g }t|j ¡ |d�D �]H\}	\}
}|js�t |jd|jdd�¡}t	|r`|j
nd|
|	|j|jd�}t|_| t|j|d�¡ q(t |¡D ]Ö}t|jƒ}| ¡ D ](}tt|ƒƒ}t|
|	|ƒ|_| |¡ q°|j}|du�r<t |jd|jdd�¡}t|jd	�}t|
|	|ƒ|_|j�r*|j
nd|j_| |¡ | ¡ D ]*}tt|ƒƒ}t|
|	|ƒ|_| |¡ �qDqšq(|S )
a#  Partitions Microsoft Excel Documents in .xlsx format into its document elements.

    Parameters
    ----------
    filename
        A string defining the target filename path.
    file
        A file-like object using "rb" mode --> open(filename, "rb").
    find_subtable
        Detect "subtables" on each worksheet and partition each of those as a separate `Table`
        element. When `False`, each worksheet is partitioned as a single `Table` element. A
        subtable is a contiguous block of cells with more than two cells in each row.
    infer_table_structure
        If True, any Table elements that are extracted will also have a metadata field
        named "text_as_html" where the table's text content is rendered into an html string.
        I.e., rows and cells are preserved.
        Whether True or False, the "text" field is always present in any Table element
        and is the text content of the table (no structure).
    include_header
        Determines whether or not header info is included in text and medatada.text_as_html
    ©Ú	file_pathr    r!   r"   r#   )ÚstartFÚ )ÚindexÚheaderZna_repN)Útext_as_htmlÚ	page_nameÚpage_numberr)   Úlast_modified)ÚtextÚmetadata©r6   )Ú_XlsxPartitionerOptionsÚ	enumerateÚsheetsÚitemsr!   r   Zfrom_html_textZto_htmlr"   r   ÚhtmlÚmetadata_file_pathr5   r   Zdetection_originÚappendr   r6   Ú_ConnectedComponentsÚfrom_worksheet_dfÚ_SubtableParserÚsubtableÚ#iter_leading_single_cell_rows_textsÚ_create_elementr   Ú_get_metadatar7   Ú
core_tabler#   r2   Ú$iter_trailing_single_cell_rows_texts)r)   r    r!   r"   r#   r$   r*   ÚoptsÚelementsr4   Ú
sheet_nameZsheetZ
html_tabler7   Ú	componentZsubtable_parserÚcontentÚelementrG   © rO   úZ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/partition/xlsx.pyÚpartition_xlsx)   sZ    !û
ÿÿû

ÿÿ
rQ   c                   @  s¤   e Zd ZdZddddddœdd„Zeddœd	d
„ƒZeddœdd„ƒZeddœdd„ƒZeddœdd„ƒZ	eddœdd„ƒZ
eddœdd„ƒZeddœdd„ƒZdS )r9   zVEncapsulates partitioning option validation, computation, and application of defaults.r%   r&   r'   r,   c                C  s"   || _ || _|| _|| _|| _d S ©N)Ú
_file_pathÚ_fileÚ_find_subtableÚ_include_headerÚ_infer_table_structure)Úselfr-   r    r!   r"   r#   rO   rO   rP   Ú__init__‹   s
    	z _XlsxPartitionerOptions.__init__©r+   c                 C  s   | j S )zÀTrue when partitioner should detect and emit separate `Table` elements for subtables.

        A subtable is (roughly) a contiguous rectangle of populated cells bounded by empty rows.
        )rU   ©rX   rO   rO   rP   r!   š   s    z%_XlsxPartitionerOptions.find_subtablez
int | Nonec                 C  s   | j r
dS dS )zNThe index of the row Pandas should treat as column-headings. Either 0 or None.r   N©rV   r[   rO   rO   rP   Úheader_row_idx¢   s    z&_XlsxPartitionerOptions.header_row_idxc                 C  s   | j S )z6True when column headers should be included in tables.r\   r[   rO   rO   rP   r"   §   s    z&_XlsxPartitionerOptions.include_headerc                 C  s   | j S )zGTrue when partitioner should compute and apply `text_as_html` metadata.)rW   r[   rO   rO   rP   r#   ¬   s    z-_XlsxPartitionerOptions.infer_table_structurec                 C  s   | j rt| j ƒS dS )zHThe best last-modified date available, None if no sources are available.N)rS   r   r[   rO   rO   rP   r5   ±   s    z%_XlsxPartitionerOptions.last_modifiedz
str | Nonec                 C  s   | j S )zHThe best available file-path for this document or `None` if unavailable.)rS   r[   rO   rO   rP   r>   ¶   s    z*_XlsxPartitionerOptions.metadata_file_pathzdict[str, pd.DataFrame]c                 C  sf   | j  }rtj|d| jd�S | j }rZt|tƒrH| d¡ t 	| 
¡ ¡}tj|d| jd�S tdƒ‚dS )zFThe spreadsheet worksheets, each as a data-frame mapped by sheet-name.N)rK   r1   r   z7Either 'filename' or 'file' argument must be specified.)rS   ÚpdZ
read_excelr]   rT   Ú
isinstancer   ÚseekÚioÚBytesIOÚreadÚ
ValueError)rX   r-   ÚfrO   rO   rP   r;   »   s    



z_XlsxPartitionerOptions.sheetsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rY   r   r!   r]   r"   r#   r5   r>   r;   rO   rO   rO   rP   r9   ˆ   s    r9   c                   @  sx   e Zd ZdZdddœdd„Zeddœd	d
„ƒZd d dœdd„Zeddœdd„ƒZeddœdd„ƒZ	eddœdd„ƒZ
dS )Ú_ConnectedComponentzÊA collection of cells that are "2d-connected" in a worksheet.

    2d-connected means there is a path from each cell to every other cell by traversing up, down,
    left, or right (not diagonally).
    úpd.DataFramezset[_CellCoordinate])Ú	worksheetÚcell_coordinate_setc                 C  s   || _ || _d S rR   )Ú
_worksheetÚ_cell_coordinate_set)rX   rl   rm   rO   rO   rP   rY   Ñ   s    z_ConnectedComponent.__init__r(   rZ   c                 C  s
   | j d S )z7The right-most column index of the connected component.é   ©Ú_extentsr[   rO   rO   rP   Úmax_xÕ   s    z_ConnectedComponent.max_x)Úotherr+   c                 C  s   t | j| j |j¡ƒS )z¼Produce new instance with union of cells in `self` and `other`.

        Used to combine regions of workshet that are "overlapping" row-wise but not actually
        2D-connected.
        )rj   rn   ro   Úunion)rX   rt   rO   rO   rP   ÚmergeÚ   s    ÿz_ConnectedComponent.mergec                 C  s
   | j d S )z6The left-most column index of the connected component.r   rq   r[   rO   rO   rP   Úmin_xä   s    z_ConnectedComponent.min_xc                 C  s.   | j \}}}}| jj||d …||d …f S )z÷The connected region of the worksheet as a `DataFrame`.

        The subtable is the rectangular region of the worksheet inside the connected-component
        bounding-box. Row-indices and column labels are preserved, not restarted at 0.
        r   )rr   rn   Úiloc)rX   rw   Úmin_yrs   Úmax_yrO   rO   rP   rC   é   s    z_ConnectedComponent.subtableztuple[int, int, int, int]c                 C  s€   t dƒt dƒt dƒt dƒf\}}}}| jD ]8\}}||k r>|}||krJ|}||k rV|}||kr*|}q*t|ƒt|ƒt|ƒt|ƒfS )z1Compute bounding box of this connected component.Úinfz-inf)Úfloatro   r(   )rX   rw   ry   rs   rz   ÚxÚyrO   rO   rP   rr   ó   s    $z_ConnectedComponent._extentsN)rf   rg   rh   ri   rY   r   rs   rv   rw   rC   rr   rO   rO   rO   rP   rj   Ê   s   
	rj   c                   @  sb   e Zd ZdZddœdd„Zddœdd	„Zedd
dœdd„ƒZeddœdd„ƒZ	dddœdd„Z
dS )r@   zÊThe collection of connected-components for a single worksheet.

    "Connected-components" refers to the graph algorithm we use to detect contiguous groups of
    non-empty cells in an excel sheet.
    rk   )Úworksheet_dfc                 C  s
   || _ d S rR   )Ú_worksheet_df)rX   r   rO   rO   rP   rY   
  s    z_ConnectedComponents.__init__zIterator[_ConnectedComponent]rZ   c                 C  s
   t | jƒS rR   )ÚiterÚ_connected_componentsr[   rO   rO   rP   Ú__iter__  s    z_ConnectedComponents.__iter__r   )r   r+   c                 C  s   | |ƒS )zKConstruct from a worksheet dataframe produced by reading Excel with pandas.rO   )Úclsr   rO   rO   rP   rA     s    z&_ConnectedComponents.from_worksheet_dfzlist[_ConnectedComponent]c                   sv   ˆ j j\}}t ||f¡j}ˆ j  ¡ j}dd„ || D ƒ}t ||¡}| |¡ t 	|¡}t
ˆ  ‡ fdd„|D ƒ¡ƒS )z=The `_ConnectedComponent` objects comprising this collection.c                 S  s   g | ]}t |ƒ‘qS rO   ©Útuple)Ú.0ÚpairrO   rO   rP   Ú
<listcomp>  ó    z>_ConnectedComponents._connected_components.<locals>.<listcomp>c                   s   g | ]}t ˆ j|ƒ‘qS rO   )rj   r€   )r‡   Zcomponent_node_setr[   rO   rP   r‰   *  s   ÿ)r€   ÚshapeÚnpÚindicesÚTZisnaÚnxZgrid_2d_graphZremove_nodes_fromÚconnected_componentsÚlistÚ_merge_overlapping_tables)rX   Zmax_rowZmax_colZ
node_arrayZempty_cellsZnodes_to_removeÚgraphZconnected_node_setsrO   r[   rP   r‚     s     
ÿ
þÿÿz*_ConnectedComponents._connected_components)r�   r+   c                 c  s`   t |dd„ d�}d}|D ]4}|du r*|}q|j|jkrB| |¡}q|V  |}q|dur\|V  dS )a  Merge connected-components that overlap row-wise.

        A pair of overlapping components might look like one of these:

            x x x        x x
                x        x x   x x
            x   x   OR         x x
            x
            x x x
        c                 S  s   | j S rR   )rw   )r}   rO   rO   rP   Ú<lambda>?  rŠ   z@_ConnectedComponents._merge_overlapping_tables.<locals>.<lambda>)ÚkeyN)Úsortedrw   rs   rv   )rX   r�   Zsorted_componentsZcurrent_componentrL   rO   rO   rP   r’   1  s    z._ConnectedComponents._merge_overlapping_tablesN)rf   rg   rh   ri   rY   rƒ   ÚclassmethodrA   r   r‚   r’   rO   rO   rO   rP   r@     s   r@   c                   @  s‚   e Zd ZdZddœdd„Zeddœdd	„ƒZd
dœdd„Zd
dœdd„Zeddœdd„ƒZ	eddœdd„ƒZ
eddœdd„ƒZdS )rB   a‚  Distinguishes core-table from leading and trailing title rows in a subtable.

    A *subtable* is a contiguous block of populated cells in the spreadsheet. Leading or trailing
    rows of that block containing only one populated cell are called "single-cell rows" and are
    not considered part of the core table. These are each emitted separately as a `Text`-subtype
    element.
    rk   )rC   c                 C  s
   || _ d S rR   )Ú	_subtable)rX   rC   rO   rO   rP   rY   a  s    z_SubtableParser.__init__zpd.DataFrame | NonerZ   c                 C  s>   t | jƒ}|t | jƒkrdS t | jƒt | jƒ }| j||… S )zCThe part between the leading and trailing single-cell rows, if any.N)ÚlenÚ _leading_single_cell_row_indicesr˜   Ú!_trailing_single_cell_row_indices)rX   Zcore_table_startZcore_table_stoprO   rO   rP   rG   d  s
    
z_SubtableParser.core_tablezIterator[str]c                 c  s(   | j D ]}| jj|  ¡ jd V  qdS )z8Generate the cell-text for each leading single-cell row.r   N)rš   r˜   rx   Údropna©rX   Zrow_idxrO   rO   rP   rD   v  s    
z3_SubtableParser.iter_leading_single_cell_rows_textsc                 c  s(   | j D ]}| jj|  ¡ jd V  qdS )z9Generate the cell-text for each trailing single-cell row.r   N)r›   r˜   rx   rœ   r�   rO   rO   rP   rH   {  s    
z4_SubtableParser.iter_trailing_single_cell_rows_textsztuple[int, ...]c                   s   ddœ‡ fdd„}t |ƒ ƒS )zEIndex of each leading single-cell row in subtable, in top-down order.úIterator[int]rZ   c                  3  s,   t ˆ jƒD ]\} }|| kr  d S | V  q
d S rR   )r:   Ú_single_cell_row_indices©Znext_row_idxÚidxr[   rO   rP   Ú$iter_leading_single_cell_row_indices„  s    z^_SubtableParser._leading_single_cell_row_indices.<locals>.iter_leading_single_cell_row_indicesr…   )rX   r¢   rO   r[   rP   rš   €  s    z0_SubtableParser._leading_single_cell_row_indicesc                   s   ddœ‡ fdd„}t |ƒ ƒS )z=Index of each single-cell row in subtable, in top-down order.rž   rZ   c                  3  s4   t ˆ j ¡ ƒD ] \} \}}| ¡ dkr(q| V  qd S )Nr   )r:   r˜   ZiterrowsÚcount)r¡   Ú_Úrowr[   rO   rP   Úiter_single_cell_row_idxs�  s    zK_SubtableParser._single_cell_row_indices.<locals>.iter_single_cell_row_idxsr…   )rX   r¦   rO   r[   rP   rŸ   Œ  s    z(_SubtableParser._single_cell_row_indicesc                   s<   t ˆ jƒt ˆ jƒkrdS ddœ‡ fdd„}ttt|ƒ ƒƒƒS )zFIndex of each trailing single-cell row in subtable, in top-down order.rO   rž   rZ   c                  3  sD   t ˆ jƒd } ˆ jddd… D ] }|| kr0 dS | V  | d8 } qdS )z... moving from end upward ...r   Néÿÿÿÿ)r™   r˜   rŸ   r    r[   rO   rP   Ú%iter_trailing_single_cell_row_indicesŸ  s    z`_SubtableParser._trailing_single_cell_row_indices.<locals>.iter_trailing_single_cell_row_indices)r™   rš   r˜   r†   Úreversedr‘   )rX   r¨   rO   r[   rP   r›   ˜  s    	z1_SubtableParser._trailing_single_cell_row_indicesN)rf   rg   rh   ri   rY   r   rG   rD   rH   rš   rŸ   r›   rO   rO   rO   rP   rB   X  s   rB   r   )r6   r+   c                 C  sZ   t | ƒrtt| ƒd�S t| ƒr(t| d�S t| ƒr:t| d�S t| ƒrLt| d�S t| d�S dS )z=Create `Text`-subtype document element appropriate to `text`.r8   N)	r   r   r   r   r   r   r   r   r   r8   rO   rO   rP   rE   «  s    


rE   r   )rK   r4   rI   r+   c                 C  s   t | ||j|jd�S )N)r3   r4   r)   r5   )r   r>   r5   )rK   r4   rI   rO   rO   rP   rF   ¹  s    ürF   )N)9ri   Ú
__future__r   ra   Útempfiler   Útypingr   r   r   r   Znetworkxr�   ÚnumpyrŒ   Zpandasr^   Ztyping_extensionsr   r	   Zunstructured.chunkingr
   Zunstructured.cleaners.corer   Zunstructured.common.html_tabler   Zunstructured.documents.elementsr   r   r   r   r   r   r   Zunstructured.file_utils.modelr   Z&unstructured.partition.common.metadatar   r   Z unstructured.partition.text_typer   r   r   r   Zunstructured.utilsr   r   Ú__annotations__r   ZXLSXrQ   r9   rj   r@   rB   rE   rF   rO   rO   rO   rP   Ú<module>   sD   $	 ÿù$]B9US