a
    ¡˜ëjD  ã                   @   sº   d dl mZmZmZmZ d dlZd dlmZ dZg d¢Z	e
e
dœdd„Ze
edœd	d
„Zde
eeeee f  ee
e
f dœdd„Zde
eeeee f  ee
 ee
e
f dœdd„ZdS )é    )ÚIOÚOptionalÚTupleÚUnionN)Úconvert_to_bytesgš™™™™™é?)Úutf_8Ú
iso_8859_1Ú
iso_8859_6Ú
iso_8859_8ÚasciiÚbig5Úutf_16Ú	utf_16_beÚ	utf_16_leÚutf_32Ú	utf_32_beÚ	utf_32_leÚeuc_jis_2004Úeuc_jisx0213Úeuc_jpÚeuc_krÚgb18030Ú	shift_jisÚshift_jis_2004Úshift_jisx0213)ÚencodingÚreturnc                 C   s0   |   ¡  dd¡}g d¢}||v r,|dd… }|S )zßFormat input encoding string (e.g., `utf-8`, `iso-8859-1`, etc).
    Parameters
    ----------
    encoding
        The encoding string to be formatted (e.g., `UTF-8`, `utf_8`, `ISO-8859-1`, `iso_8859_1`,
        etc).
    Ú_ú-)ziso-8859-6-iziso-8859-6-eziso-8859-8-iziso-8859-8-eNéþÿÿÿ)ÚlowerÚreplace)r   Úformatted_encodingZannotated_encodings© r#   ú_/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/file_utils/encoding.pyÚformat_encoding_str"   s
    r%   c                 C   s$   t D ]}t|ƒt| ƒkr dS qdS )z‰Checks if an encoding string is valid. Helps to avoid errors in cases where
    invalid encodings are extracted from malformed documents.TF)ÚCOMMON_ENCODINGSr%   )r   Zcommon_encodingr#   r#   r$   Úvalidate_encoding4   s    r'   Ú )ÚfilenameÚfiler   c           
   
   C   s$  | r8t | dƒ�}| ¡ }W d   ƒ qN1 s,0    Y  n|rFt|ƒ}ntdƒ‚t |¡}|d }|d }|d u sz|tk �r
tD ]t}zT| r¾t | |d��}| ¡ }W d   ƒ qÈ1 s²0    Y  n
| |¡}|}W  �qW q~ t	t
fyð   Y q~Y q~0 q~t	d|dt|ƒdƒ‚n
| |¡}t|ƒ}	|	|fS )	NÚrbz#No filename nor file were specifiedr   Ú
confidence©r   z]Unable to determine the encoding of the file or match it with any of the specified encodings.r   zInvalid encoding)ÚopenÚreadr   ÚFileNotFoundErrorÚchardetÚdetectÚENCODE_REC_THRESHOLDr&   ÚdecodeÚUnicodeDecodeErrorÚUnicodeErrorÚlenr%   )
r)   r*   ÚfZ	byte_dataÚresultr   r,   ÚencÚ	file_textr"   r#   r#   r$   Údetect_file_encoding=   s<    (

(
ú

r<   )r)   r*   r   r   c                 C   s  | r†|rxt |ƒ}t| |d��J}z| ¡ }W n, ttfyV } z|‚W Y d}~n
d}~0 0 W d  ƒ q„1 sl0    Y  nt| ƒ\}}nŽ|�r|rüt |ƒ}z4t|tƒr¨|n| ¡ }t|tƒrÆ| |¡}n|}W n, ttfyø } z|‚W Y d}~n
d}~0 0 nt|d�\}}nt	dƒ‚||fS )z6Extracts document metadata from a plain text document.r-   N)r*   zNo filename was specified)
r%   r.   r/   r5   r6   r<   Ú
isinstanceÚbytesr4   r0   )r)   r*   r   r"   r8   r;   ÚerrorZfile_contentr#   r#   r$   Úread_txt_filel   s,    :
r@   )r(   N)r(   NN)Útypingr   r   r   r   r1   Z$unstructured.partition.common.commonr   r3   r&   Ústrr%   Úboolr'   r>   r<   r@   r#   r#   r#   r$   Ú<module>   s,   
  þ
ý0   ý
ü