a
    ~¦wSF  ã                   @   sH   d dl mZ d dlZd dlZd dlmZ ddlmZ G dd„ deƒZ	dS )é    )ÚdefaultdictN)Úxrangeé   )ÚNGramc                   @   sJ   e Zd ZdZdZe d¡Ze d¡Zddd„Z	dd	„ Z
d
d„ Zdd„ ZdS )ÚLangProfileé   i † z
^[A-Za-z]$z.*[A-Za-z].*Nc                 C   sB   t tƒ| _|d ur| j |¡ |d u r2dgtj }|| _|| _d S )Nr   )r   ÚintÚfreqÚupdater   ÚN_GRAMÚnameÚn_words)Úselfr   r	   r   © r   ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langdetect/utils/lang_profile.pyÚ__init__   s    
zLangProfile.__init__c                 C   s`   | j du s|du rdS t|ƒ}|dk s0|tjkr4dS | j|d   d7  < | j|  d7  < dS )zAdd n-gram to profile.Nr   )r   Úlenr   r   r   r	   )r   ÚgramÚlengthr   r   r   Úadd   s    zLangProfile.addc                 C   sâ   | j du rdS t| jd | j | jƒ}d}tt | j¡ƒD ]H\}}||krn| jt	|ƒd   |8  < | j|= q:| j
 |¡r:||7 }q:|| jd d k rÞtt | j¡ƒD ]6\}}| j |¡r¦| jt	|ƒd   |8  < | j|= q¦dS )zAEliminate below less frequency n-grams and noise Latin alphabets.Nr   r   é   )r   Úmaxr   ÚLESS_FREQ_RATIOÚMINIMUM_FREQÚlistÚsixÚ	iteritemsr	   r   ÚROMAN_CHAR_REÚmatchÚROMAN_SUBSTR_RE)r   Ú	thresholdZromanÚkeyÚcountr   r   r   Úomit_less_freq&   s    


zLangProfile.omit_less_freqc                 C   sZ   |du rdS t  |¡}t ƒ }|D ]4}| |¡ tdt jd ƒD ]}|  | |¡¡ q>q dS )zˆUpdate the language profile with (fragmented) text.
        Extract n-grams from text and add their frequency into the profile.
        Nr   )r   Znormalize_viZadd_charr   r   r   Úget)r   Útextr   ÚchÚnr   r   r   r
   ;   s    

zLangProfile.update)NNN)Ú__name__Ú
__module__Ú__qualname__r   r   ÚreÚcompiler   r   r   r   r#   r
   r   r   r   r   r   
   s   



r   )
Úcollectionsr   r+   r   Z	six.movesr   Zngramr   Úobjectr   r   r   r   r   Ú<module>   s
   