a
    ~˜ëj*7  ã                   @  sþ  U d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d	d
dœdd„Z&d	ddœdd„Z'd	d	dœdd„Z(e)dd„ e *¡ D ƒƒZ+de,d< dd„ e+D ƒZ-de,d< d	ddœdd„Z.d	ddœdd„Z/d	ddœd d!„Z0d	ddœd"d#„Z1d	ddœd$d%„Z2d	ddœd&d'„Z3d	ddœd(d)„Z4d	ddœd*d+„Z5d	ddœd,d-„Z6d	ddœd.d/„Z7d	ddœd0d1„Z8d	ddœd2d3„Z9d	ddœd4d5„Z:d	ddœd6d7„Z;d	ddœd8d9„Z<d	dd:œd;d<„Z=d	ddœd=d>„Z>dfd@d
ddAœdBdC„Z?e
ddD�d	ddEœdFdG„ƒZ@d@dHdIœdJdK„ZAd	ddLœdMdN„ZBdgd	dd	dPœdQdR„ZCd	d	dSdTœdUdV„ZDd	d	ddTœdWdX„ZEdYejFdZfd	d
d	d[d\œd]d^„ZGdhd@d	d`d
dddaddddbdcœddde„ZHdS )ié    )ÚannotationsN)Úbisect_right)ÚIncrementalDecoder)Ú	lru_cache)Ú	Generatoré   )ÚENCODING_MARKSÚIANA_SUPPORTED_SIMILARÚRE_POSSIBLE_ENCODING_INDICATIONÚUNICODE_RANGES_COMBINEDÚ_SECONDARY_RANGE_NAMESÚCOMMON_CJK_CHARACTERSÚ_LATINÚ_CJKÚ_HANGULÚ	_KATAKANAÚ	_HIRAGANAÚ_HALFWIDTH_KATAKANAÚ_THAIÚ_ARABICÚ_ARABIC_ISOLATED_FORMÚ	_LIGATUREÚ_SUPERSCRIPTÚ_SENTENCE_OPEN_PUNCTUATIONÚ_ACCENT_KEYWORDSÚ_ACCENTUATEDÚ_KNOWN_MB_DECODERSÚ_KNOWN_MB_CLASSESÚ_IANA_NAMESÚ_MULTIBYTE_SEARCH_RADIUSÚstrÚint)Ú	characterÚreturnc                 C  s  zt  | ¡}W n ty"   Y dS 0 d}d|v r8|tO }d|v rH|tO }d|v rX|tO }d|v rx|tO }d|v rx|tO }d|v rˆ|tO }d|v r˜|t	O }d	|v r¸|t
O }d
|v r¸|tO }d|v sÊ| d¡rÒ|tO }d|v râ|tO }|dv rò|tO }tD ]}||v rö|tO } �qqö|S )zRCompute all name-based classification flags with a single unicodedata.name() call.r   ZLATINÚCJKZHANGULZKATAKANAZ	HALFWIDTHZHIRAGANAZTHAIZARABICzISOLATED FORMZLIGATUREz	LETTER AEZSUPERSCRIPT>   zINVERTED EXCLAMATION MARKzINVERTED QUESTION MARK)ÚunicodedataÚnameÚ
ValueErrorr   r   r   r   r   r   r   r   r   Úendswithr   r   r   r   r   )r"   ÚdescÚflagsÚkw© r,   úW/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/charset_normalizer/utils.pyÚ_character_flags'   sD    r.   Úboolc                 C  s   t t| ƒt@ ƒS ©N)r/   r.   r   ©r"   r,   r,   r-   Úis_accentuatedQ   s    r2   c                 C  s.   t  | ¡}|s| S | d¡}tt|d dƒƒS )Nú r   é   )r%   ÚdecompositionÚsplitÚchrr!   )r"   Z
decomposedÚcodesr,   r,   r-   Úremove_accentU   s
    

r9   c                 c  s    | ]\}}|j |j|fV  qd S r0   )ÚstartÚstop)Ú.0r&   Z	ord_ranger,   r,   r-   Ú	<genexpr>a   s   ÿr=   zlist[tuple[int, int, str]]Ú_UNICODE_RANGES_SORTEDc                 C  s   g | ]}|d  ‘qS )r   r,   )r<   Úer,   r,   r-   Ú
<listcomp>e   ó    r@   z	list[int]Ú_UNICODE_RANGE_STARTSz
str | Nonec                 C  sT   t | ƒ}|dk rdS |dk r dS tt|ƒd }|dkrPt| \}}}||k rP|S dS )zK
    Retrieve the Unicode range official name from a single character.
    é    zControl characteré€   zBasic Latinr   r   N)Úordr   rB   r>   )r"   Zcharacter_ordÚidxr:   r;   r&   r,   r,   r-   Úunicode_rangeh   s    rG   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Úis_latin}   s    rH   c                 C  s2   t  | ¡}d|v rdS t| ƒ}|d u r*dS d|v S )NÚPTFZPunctuation©r%   ÚcategoryrG   ©r"   Úcharacter_categoryÚcharacter_ranger,   r,   r-   Úis_punctuation�   s    
rO   c                 C  sB   t  | ¡}d|v sd|v rdS t| ƒ}|d u r2dS d|v o@|dkS )NÚSÚNTFZFormsZLorJ   rL   r,   r,   r-   Ú	is_symbol�   s    
rR   c                 C  s$   t | ƒ}|d u rdS d|v p"d|v S )NFZ	EmoticonsZPictographs)rG   )r"   rN   r,   r,   r-   Úis_emoticon�   s    rS   c                 C  s.   |   ¡ s| dv rdS t | ¡}d|v p,|dv S )N>   u   ï½œú+ú>ú<TÚZ>   ZPoZPcZPd)Úisspacer%   rK   )r"   rM   r,   r,   r-   Úis_separator¦   s    
rY   c                 C  s   |   ¡ |  ¡ kS r0   )ÚislowerÚisupperr1   r,   r,   r-   Úis_case_variable¯   s    r\   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Úis_cjk³   s    r]   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Úis_hiragana·   s    r^   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Úis_katakana»   s    r_   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Ú	is_hangul¿   s    r`   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Úis_thaiÃ   s    ra   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Ú	is_arabicÇ   s    rb   c                 C  s   t t| ƒt@ ƒS r0   )r/   r.   r   r1   r,   r,   r-   Úis_arabic_isolated_formË   s    rc   c                 C  s   | t vS r0   )r   r1   r,   r,   r-   Úis_cjk_uncommonÏ   s    rd   )Ú
range_namer#   c                 C  s   | t v S r0   )r   )re   r,   r,   r-   Úis_unicode_range_secondaryÓ   s    rf   c                 C  s$   |   ¡  o"|  ¡  o"| dko"| dkS )Núu   ï»¿)rX   Úisprintabler1   r,   r,   r-   Úis_unprintable×   s    
ÿþüri   é    zbytes | bytearray)ÚsequenceÚsearch_zoner#   c           	      C  sš   t | ttfƒst‚t| ƒ}| dt||ƒ… }| ¡ }d|vrHd|vrHdS |jddd�}t 	|¡D ]4}| 
d¡ ¡  dd	¡}t |¡}|dur`|  S q`dS )
zW
    Extract using ASCII-only decoder any specified encoding in the first n-bytes.
    Ns   codings   charsetÚasciiÚignore©Úerrorsr   ú-Ú_)Ú
isinstanceÚbytesÚ	bytearrayÚ	TypeErrorÚlenÚminÚlowerÚdecoder
   ÚfinditerÚgroupÚreplacer   Úget)	rk   rl   Zseq_lenZsearch_bytesZlowered_bytesZdecoded_zoneÚmatchZspecified_encodingÚencoding_ianar,   r,   r-   Úany_specified_encodingá   s    

r�   )Úmaxsize)r&   r#   c                 C  sP   | t v rdS tD ]:}zt |¡ | ¡ W n tttfyD   Y qY n0  dS dS )zQ
    Verify is a specific encoding is a multi byte one based on it IANA name
    TF)r   r   Ú	importlibÚimport_moduleZgetcodecÚImportErrorÚAttributeErrorÚLookupError)r&   Zproviderr,   r,   r-   Úis_multi_byte_encodingý   s    	
rˆ   ztuple[str | None, bytes])rk   r#   c                 C  sJ   t D ]@}t | }t|tƒr |g}|D ]}|  |¡r$||f    S q$qdS )z9
    Identify and extract SIG/BOM in given sequence.
    )NrA   )r   rs   rt   Ú
startswith)rk   Úiana_encodingZmarksÚmarkr,   r,   r-   Úidentify_sig_or_bom  s    

rŒ   )rŠ   r#   c                 C  s   | dvS )N>   Úutf_32Úutf_16r,   )rŠ   r,   r,   r-   Úshould_strip_sig_or_bom(  s    r�   T)Úcp_nameÚstrictr#   c                 C  s>   |   ¡  dd¡} t | ¡}|dur&|S |r:td| › d�ƒ‚| S )zIReturns the Python normalized encoding name (Not the IANA official name).rq   rr   NzUnable to retrieve IANA for 'ú')ry   r}   r   r~   r'   )r�   r‘   r€   r,   r,   r-   Ú	iana_name,  s    
r“   Úfloat)Úiana_name_aÚiana_name_br#   c           	      C  sŒ   t | ƒst |ƒrdS t d| › �¡j}t d|› �¡j}|dd�}|dd�}d}tdƒD ]*}t|gƒ}| |¡| |¡krX|d7 }qX|d S )Ng        z
encodings.rn   ro   r   é   r   )rˆ   rƒ   r„   r   Úrangert   rz   )	r•   r–   Z	decoder_aZ	decoder_bZid_aZid_bZcharacter_match_countÚiZto_be_decodedr,   r,   r-   Úcp_similarity:  s    



rš   c                 C  s   | t v o|t |  v S )zŽ
    Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
    the function cp_similarity.
    )r	   )r•   r–   r,   r,   r-   Úis_cp_similarN  s    
þr›   Zcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sÚNone)r&   ÚlevelÚformat_stringr#   c                 C  s:   t  | ¡}| |¡ t  ¡ }| t  |¡¡ | |¡ d S r0   )ÚloggingÚ	getLoggerÚsetLevelÚStreamHandlerÚsetFormatterÚ	FormatterÚ
addHandler)r&   r�   rž   ÚloggerÚhandlerr,   r,   r-   Úset_logging_handlerY  s
    

r¨   Fr˜   rt   zGenerator[str, None, None])Ú	sequencesr€   ÚoffsetsÚ
chunk_sizeÚbom_or_sig_availableÚstrip_sig_or_bomÚsig_payloadÚis_multi_byte_decoderÚdecoded_payloadÚdeferred_decodingr#   c
                 c  s.  |rV|  d¡rVt|ƒ}
t| ƒ}|D ].}||
 | }|||| … }|sJ qR|V  q"�nÔ|rŠ|sŠ|D ]"}|||| … }|s~ q†|V  qb�n |	rØ|s–| n| t|ƒd … }|D ](}|||| … }|sÆ qÔt||ƒV  qª�nR|D �]J}|| }|t| ƒd krüqÜ| ||| … }|�r |�s || }|j||�r0dndd�}|�r"|dk�r"t|dƒ}|d |… }d}|�r®t|ƒ}
||
 t| ƒ }td|t ƒ}t|
|t ƒ}| |||¡dk}|�r"|�s"||v�r"t||d	 d
ƒD ]L}| ||… }|�rø|�sø|| }|j|dd�}|d |… |v �rÔ �q"�qÔ|V  qÜd S )NZiso2022_é   rn   r‘   ro   r   r4   Fé   éÿÿÿÿ)	r‰   rw   r    rz   rx   Úmaxr   Úfindr˜   )r©   r€   rª   r«   r¬   r­   r®   r¯   r°   r±   Zdecoded_lengthZsequence_lengthr™   Zdecoded_offsetÚchunkZ
base_bytesZcut_sequenceZ	chunk_endZchunk_partial_size_chkZchunk_prefixZfound_nearbyZexpected_offsetZsearch_startZ
search_endÚjr,   r,   r-   Úcut_sequence_chunksf  s~    ÿ
þ
ÿÿÿÿþý
r¹   )rj   )T)NF)IÚ
__future__r   rƒ   rŸ   r%   Úbisectr   Úcodecsr   Ú	functoolsr   Útypingr   Zconstantr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r.   r2   r9   ÚsortedÚitemsr>   Ú__annotations__rB   rG   rH   rO   rR   rS   rY   r\   r]   r^   r_   r`   ra   rb   rc   rd   rf   ri   r�   rˆ   rŒ   r�   r“   rš   r›   ÚINFOr¨   r¹   r,   r,   r,   r-   Ú<module>   s`   h*þ		 ÿý  ö