a
    WSW¿(  ã                   @   st   d dl Z d dlZddlmZ ddlmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ G dd„ deƒZe ¡  dS )é    Né   )Úmessages)Úunicode_blockÚUNICODE_BASIC_LATINÚUNICODE_LATIN_1_SUPPLEMENTÚUNICODE_LATIN_EXTENDED_BÚUNICODE_GENERAL_PUNCTUATIONÚUNICODE_ARABICÚ!UNICODE_LATIN_EXTENDED_ADDITIONALÚUNICODE_HIRAGANAÚUNICODE_KATAKANAÚUNICODE_BOPOMOFOÚUNICODE_BOPOMOFO_EXTENDEDÚUNICODE_CJK_UNIFIED_IDEOGRAPHSÚUNICODE_HANGUL_SYLLABLESc                €   @   s¬  e Zd Ze d¡ZdZdd„ Zdd„ Zdd„ Z	e
d	d
„ ƒZe
dd„ ƒZe d¡e d¡e d¡e d¡e d¡gZe d¡Ze d¡Ze de d e d ej¡Ze d¡e d¡e d¡e d¡e d¡e d¡e d¡e d¡e d¡e d ¡e d!¡e d"¡e d#¡e d$¡e d%¡e d&¡e d'¡e d(¡e d)¡e d*¡e d+¡e d,¡e d-¡e d.¡e d/¡e d0¡e d1¡e d2¡e d3¡e d4¡e d5¡e d6¡e d7¡e d8¡e d9¡e d:¡e d;¡e d<¡e d=¡e d>¡e d?¡e d@¡e dA¡e dB¡e dC¡e dD¡e dE¡e dF¡e dG¡e dH¡e dI¡e dJ¡e dK¡e dL¡e dM¡e dN¡e dO¡e dP¡e dQ¡e dR¡e dS¡e dT¡e dU¡e dV¡e dW¡e dX¡e dY¡e dZ¡e d[¡e d\¡e d]¡e d^¡e d_¡e d`¡e da¡e db¡e dc¡e dd¡e de¡e df¡e dg¡e dh¡e di¡e dj¡e dk¡e dl¡e dm¡e dn¡e do¡e dp¡e dq¡e dr¡e ds¡e dt¡e du¡e dv¡e dw¡e dx¡e dy¡e dz¡e d{¡e d|¡e d}¡e d~¡e d¡e d€¡e d�¡e d‚¡e dƒ¡e d„¡e d…¡e d†¡e d‡¡e dˆ¡e d‰¡e dŠ¡e d‹¡e dŒ¡e d�¡e dŽ¡e d�¡e d�¡e d‘¡e d’¡e d“¡e d”¡g~Zi Ze
d•d–„ ƒZd—S )˜ÚNGramzNGram.LATIN1_EXCLUDEé   c                 C   s   d| _ d| _d S )Nú F)ÚgramsÚcapitalword)Úself© r   úU/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langdetect/utils/ngram.pyÚ__init__   s    zNGram.__init__c                 C   s†   |   |¡}| jd }|dkr6d| _d| _|dkrVdS n t| jƒ| jkrV| jdd… | _|  j|7  _| ¡ r|| ¡ r‚d| _nd| _dS )z%Append a character into ngram buffer.éÿÿÿÿr   FNr   T)Ú	normalizer   r   ÚlenÚN_GRAMÚisupper)r   ÚchZ	last_charr   r   r   Úadd_char   s    

zNGram.add_charc                 C   sd   | j r
dS |dk s*|| jks*t| jƒ|k r.dS |dkrP| jd }|dkrLdS |S | j| d… S dS )zGet n-gram.Nr   r   r   )r   r   r   r   )r   Únr   r   r   r   Úget2   s     
z	NGram.getc                 C   s^  t |ƒ}|tkr>|dk s6d|  k r,dk s6n d|k r:d}�n|tkr\| j |¡dkrZd}nþ|tkr–|t d¡kr|t d¡}|t d	¡kr”t d
¡}nÄ|tkr¤d}n¶|t	krÆ|t d¡krÄt d¡}n”|t
krè|t d¡kræt d¡}nr|tkrüt d¡}n^|tk�rt d¡}nH|ttfv �r,t d¡}n.|tk�rF| j ||¡}n|tk�rZt d¡}|S )NÚAÚZÚaÚzr   r   u   È™u   ÅŸu   È›u   Å£u   ÛŒu   ÙŠu   áº u   á»ƒu   ã�‚u   ã‚¢u   ã„…u   ê°€)r   r   r   ÚLATIN1_EXCLUDEDÚfindr   ÚsixÚur   r	   r
   r   r   r   r   r   ÚCJK_MAPr"   r   )Úclsr   Úblockr   r   r   r   @   s>    &




zNGram.normalizec                    s   ‡ fdd„}ˆ j  ||¡S )zfNormalizer for Vietnamese.
        Normalize Alphabet + Diacritical Mark(U+03xx) into U+1Exx.
        c                    s2   ˆ j  |  d¡¡}ˆ j |  d¡¡}ˆ j| | S )Nr   é   )ÚTO_NORMALIZE_VI_CHARSr(   ÚgroupÚDMARK_CLASSÚNORMALIZED_VI_CHARS)ÚmZalphabetZdmark©r,   r   r   Úreplh   s    z NGram.normalize_vi.<locals>.repl)ÚALPHABET_WITH_DMARKÚsub)r,   Útextr5   r   r4   r   Únormalize_vic   s    zNGram.normalize_viZNORMALIZED_VI_CHARS_0300ZNORMALIZED_VI_CHARS_0301ZNORMALIZED_VI_CHARS_0303ZNORMALIZED_VI_CHARS_0309ZNORMALIZED_VI_CHARS_0323r/   r1   z([z])([z])zNGram.KANJI_1_0zNGram.KANJI_1_2zNGram.KANJI_1_4zNGram.KANJI_1_8zNGram.KANJI_1_11zNGram.KANJI_1_12zNGram.KANJI_1_13zNGram.KANJI_1_14zNGram.KANJI_1_16zNGram.KANJI_1_18zNGram.KANJI_1_22zNGram.KANJI_1_27zNGram.KANJI_1_29zNGram.KANJI_1_31zNGram.KANJI_1_35zNGram.KANJI_2_0zNGram.KANJI_2_1zNGram.KANJI_2_4zNGram.KANJI_2_9zNGram.KANJI_2_10zNGram.KANJI_2_11zNGram.KANJI_2_12zNGram.KANJI_2_13zNGram.KANJI_2_15zNGram.KANJI_2_16zNGram.KANJI_2_18zNGram.KANJI_2_21zNGram.KANJI_2_22zNGram.KANJI_2_23zNGram.KANJI_2_28zNGram.KANJI_2_29zNGram.KANJI_2_30zNGram.KANJI_2_31zNGram.KANJI_2_32zNGram.KANJI_2_35zNGram.KANJI_2_36zNGram.KANJI_2_37zNGram.KANJI_2_38zNGram.KANJI_3_1zNGram.KANJI_3_2zNGram.KANJI_3_3zNGram.KANJI_3_4zNGram.KANJI_3_5zNGram.KANJI_3_8zNGram.KANJI_3_9zNGram.KANJI_3_11zNGram.KANJI_3_12zNGram.KANJI_3_13zNGram.KANJI_3_15zNGram.KANJI_3_16zNGram.KANJI_3_18zNGram.KANJI_3_19zNGram.KANJI_3_22zNGram.KANJI_3_23zNGram.KANJI_3_27zNGram.KANJI_3_29zNGram.KANJI_3_30zNGram.KANJI_3_31zNGram.KANJI_3_32zNGram.KANJI_3_35zNGram.KANJI_3_36zNGram.KANJI_3_37zNGram.KANJI_3_38zNGram.KANJI_4_0zNGram.KANJI_4_9zNGram.KANJI_4_10zNGram.KANJI_4_16zNGram.KANJI_4_17zNGram.KANJI_4_18zNGram.KANJI_4_22zNGram.KANJI_4_24zNGram.KANJI_4_28zNGram.KANJI_4_34zNGram.KANJI_4_39zNGram.KANJI_5_10zNGram.KANJI_5_11zNGram.KANJI_5_12zNGram.KANJI_5_13zNGram.KANJI_5_14zNGram.KANJI_5_18zNGram.KANJI_5_26zNGram.KANJI_5_29zNGram.KANJI_5_34zNGram.KANJI_5_39zNGram.KANJI_6_0zNGram.KANJI_6_3zNGram.KANJI_6_9zNGram.KANJI_6_10zNGram.KANJI_6_11zNGram.KANJI_6_12zNGram.KANJI_6_16zNGram.KANJI_6_18zNGram.KANJI_6_20zNGram.KANJI_6_21zNGram.KANJI_6_22zNGram.KANJI_6_23zNGram.KANJI_6_25zNGram.KANJI_6_28zNGram.KANJI_6_29zNGram.KANJI_6_30zNGram.KANJI_6_32zNGram.KANJI_6_34zNGram.KANJI_6_35zNGram.KANJI_6_37zNGram.KANJI_6_39zNGram.KANJI_7_0zNGram.KANJI_7_3zNGram.KANJI_7_6zNGram.KANJI_7_7zNGram.KANJI_7_9zNGram.KANJI_7_11zNGram.KANJI_7_12zNGram.KANJI_7_13zNGram.KANJI_7_16zNGram.KANJI_7_18zNGram.KANJI_7_19zNGram.KANJI_7_20zNGram.KANJI_7_21zNGram.KANJI_7_23zNGram.KANJI_7_25zNGram.KANJI_7_28zNGram.KANJI_7_29zNGram.KANJI_7_32zNGram.KANJI_7_33zNGram.KANJI_7_35zNGram.KANJI_7_37c                 C   s,   | j D ] }|d }|D ]}|| j|< qqd S )Nr   )Ú	CJK_CLASSr+   )r,   Zcjk_listZrepresentativer   r   r   r   Ú_init_cjk_mapý   s    
zNGram._init_cjk_mapN)Ú__name__Ú
__module__Ú__qualname__r   Z
get_stringr'   r   r   r    r"   Úclassmethodr   r9   r2   r/   r1   ÚreÚcompileÚUNICODEr6   r:   r+   r;   r   r   r   r   r      s0  

"
û

þ‚ r   )r@   r)   Ú r   r   r   r   r   r   r	   r
   r   r   r   r   r   r   Úobjectr   r;   r   r   r   r   Ú<module>   s   < n