a
    òWÒ!  ã                   @   sp   d dl Z d dlZd dlZd dlmZmZ ddlmZmZ ddl	m
Z
 ddlmZ ddlmZ G dd	„ d	eƒZdS )
é    N)ÚzipÚxrangeé   )Ú	ErrorCodeÚLangDetectException)ÚLanguage)ÚNGram)Úunicode_blockc                   @   sÈ   e Zd ZdZdZdZdZdZdZdZ	dZ
e d	¡Ze d
¡Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ Zd)d*„ Zd+d,„ Zd-S ).ÚDetectoraj  
    Detector class is to detect language from specified text.
    Its instance is able to be constructed via the factory class DetectorFactory.

    After appending a target text to the Detector instance with .append(string),
    the detector provides the language detection results for target text via .detect() or .get_probabilities().

    .detect() method returns a single language name which has the highest probability.
    .get_probabilities() methods returns a list of multiple languages and their probabilities.

    The detector has some parameters for language detection.
    See set_alpha(double), .set_max_text_length(int) .set_prior_map(dict).

    Example:

        from langdetect.detector_factory import DetectorFactory
        factory = DetectorFactory()
        factory.load_profile('/path/to/profile/directory')

        def detect(text):
            detector = factory.create()
            detector.append(text)
            return detector.detect()

        def detect_langs(text):
            detector = factory.create()
            detector.append(text)
            return detector.get_probabilities()
    g      à?gš™™™™™©?iè  gš™™™™™¹?gwJëÿï?é'  Úunknownz'https?://[-_.?&~;+=/#0-9A-Za-z]{1,2076}z>[-_.0-9A-Za-z]{1,64}@[-_0-9A-Za-z]{1,255}[-_.0-9A-Za-z]{1,255}c                 C   sR   |j | _ |j| _|j| _t ¡ | _d| _d | _| j| _d| _	d| _
d | _d| _d S )NÚ é   r   F)Úword_lang_prob_mapÚlanglistÚseedÚrandomÚRandomÚtextÚlangprobÚALPHA_DEFAULTÚalphaÚn_trialÚmax_text_lengthÚ	prior_mapÚverbose)ÚselfÚfactory© r   úR/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langdetect/detector.pyÚ__init__8   s    
zDetector.__init__c                 C   s
   d| _ d S )NT)r   ©r   r   r   r   Úset_verboseF   s    zDetector.set_verbosec                 C   s
   || _ d S ©N)r   )r   r   r   r   r   Ú	set_alphaI   s    zDetector.set_alphac                 C   s¨   dgt | jƒ | _d}tt | jƒƒD ]D}| j| }||v r$|| }|dk rVttjdƒ‚|| j|< ||7 }q$|dkr~ttjdƒ‚tt | jƒƒD ]}| j|  |  < qŒdS )z3Set prior information about language probabilities.ç        r   z'Prior probability must be non-negative.z/More one of prior probability must be non-zero.N)Úlenr   r   r   r   r   ZInitParamError)r   r   ÚsumpÚiÚlangÚpr   r   r   Úset_prior_mapL   s    


zDetector.set_prior_mapc                 C   s
   || _ dS )zqSpecify max size of target text to use for language detection.
        The default value is 10000(10KB).
        N)r   )r   r   r   r   r   Úset_max_text_length]   s    zDetector.set_max_text_lengthc                 C   sr   | j  d|¡}| j d|¡}t |¡}d}ttt|ƒ| jƒƒD ].}|| }|dksZ|dkrh|  j	|7  _	|}q>dS )zÅAppend the target text for language detection.
        If the total size of target text exceeds the limit size specified by
        Detector.set_max_text_length(int), the rest is cut down.
        ú r   N)
ÚURL_REÚsubÚMAIL_REr   Znormalize_vir   Úminr&   r   r   )r   r   Úprer(   Úchr   r   r   Úappendc   s    
zDetector.appendc                 C   s–   d\}}| j D ]H}d|  kr&dkr4n n
|d7 }q|t d¡krt|ƒdkr|d7 }q|d |k r’d}| j D ]}|dk s‚d|k rn||7 }qn|| _ d	S )
zƒCleaning text to detect
        (eliminate URL, e-mail address and Latin sentence if it is not written in Latin alphabet).
        )r   r   ÚAÚzr   u   Ì€zLatin Extended Additionalé   r   N)r   ÚsixÚur	   )r   Zlatin_countZnon_latin_countr3   Ztext_without_latinr   r   r   Úcleaning_textr   s    




zDetector.cleaning_textc                 C   s   |   ¡ }|r|d jS | jS )zsDetect language of the target text and return the language name
        which has the highest probability.
        r   )Úget_probabilitiesr)   ÚUNKNOWN_LANG)r   Zprobabilitiesr   r   r   Údetect„   s    
zDetector.detectc                 C   s   | j d u r|  ¡  |  | j ¡S r#   )r   Ú_detect_blockÚ_sort_probabilityr!   r   r   r   r;   �   s    
zDetector.get_probabilitiesc                 C   s"  |   ¡  |  ¡ }|s ttjdƒ‚dgt| jƒ | _| j 	| j	¡ t
| jƒD ]Ò}|  ¡ }| j| j dd¡| j  }d}|  || j |¡|¡ |d dkrÊ|  |¡| jksÔ|| jkr²qÔ| jrÊt d|  |¡¡ |d7 }qtt
t| jƒƒD ] }| j|  || | j 7  < qâ| jrJt d|  |¡¡ qJd S )	NzNo features in text.r%   ç      ð?r   é   ú>r   z==>)r:   Ú_extract_ngramsr   r   ZCantDetectErrorr&   r   r   r   r   r   r   Ú_init_probabilityr   ÚgaussÚALPHA_WIDTHÚ_update_lang_probÚchoiceÚ_normalize_probÚCONV_THRESHOLDÚITERATION_LIMITr   r8   Úprint_r?   )r   ZngramsÚtÚprobr   r(   Újr   r   r   r>   ’   s*    
zDetector._detect_blockc                 C   s2   | j durt| j ƒS dt| jƒ gt| jƒ S dS )zzInitialize the map of language probabilities.
        If there is the specified prior map, use it as initial map.
        Nr@   )r   Úlistr&   r   r!   r   r   r   rD   ­   s    

zDetector._init_probabilityc                 C   sŒ   t tdtjd ƒƒ}g }tƒ }| jD ]b}| |¡ |jr:q$|D ]F}t|jƒ|k rT q$|j| d… }|r>|dkr>|| j	v r>| 
|¡ q>q$|S )z!Extract n-grams from target text.r   Nr-   )rP   r   r   ZN_GRAMr   Zadd_charZcapitalwordr&   Zgramsr   r4   )r   ÚRANGEÚresultZngramr3   ÚnÚwr   r   r   rC   ¶   s    

zDetector._extract_ngramsc                 C   s~   |du s|| j vrdS | j | }| jrFt d||  |¡|  |¡f ¡ || j }tt|ƒƒD ]}||  |||  9  < q\dS )z:Update language probabilities with N-gram string(N=1,2,3).NFz
%s(%s): %sT)	r   r   r8   rL   Ú_unicode_encodeÚ_word_prob_to_stringÚ	BASE_FREQr   r&   )r   rN   Úwordr   Zlang_prob_mapZweightr(   r   r   r   rG   É   s    
 
zDetector._update_lang_probc                 C   s@   d}t t|ƒƒD ]*}|| }|dkr|d| j| |f 7 }q|S )Nr   gñhãˆµøä>z %s:%.5f)r   r&   r   )r   rN   rR   rO   r*   r   r   r   rV   ×   s    zDetector._word_prob_to_stringc                 C   sD   dt |ƒ }}tt|ƒƒD ]$}|| | }||k r6|}|||< q|S )zRNormalize probabilities and check convergence by the maximun probability.
        r%   )Úsumr   r&   )r   rN   Zmaxpr'   r(   r*   r   r   r   rI   ß   s    
zDetector._normalize_probc                    s*   ‡ fdd„t ˆ j|ƒD ƒ}|jdd� |S )Nc                    s$   g | ]\}}|ˆ j krt||ƒ‘qS r   )ÚPROB_THRESHOLDr   )Ú.0r)   r*   r!   r   r   Ú
<listcomp>ë   ó    z.Detector._sort_probability.<locals>.<listcomp>T)Úreverse)r   r   Úsort)r   rN   rR   r   r!   r   r?   ê   s    zDetector._sort_probabilityc                 C   sl   d}|D ]^}|t  d¡kr^tdt|ƒ ƒdd … }t|ƒdk rHd| }q2|d|dd	…  7 }q||7 }q|S )
Nr   õ   Â€i   r7   é   Ú0z\ur   rA   )r8   r9   ÚhexÚordr&   )r   rX   Úbufr3   Ústr   r   r   rU   ï   s    

zDetector._unicode_encodeN) Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   rF   rK   rZ   rJ   rW   r<   ÚreÚcompiler.   r0   r    r"   r$   r+   r,   r4   r:   r=   r;   r>   rD   rC   rG   rV   rI   r?   rU   r   r   r   r   r
      s6   

		r
   )r   rk   r8   Z	six.movesr   r   Zlang_detect_exceptionr   r   Úlanguager   Zutils.ngramr   Zutils.unicode_blockr	   Úobjectr
   r   r   r   r   Ú<module>   s   