a
     ˜ëj¬  ã                   @   sh   d Z ddlmZ ddlmZ zddlZW n ey>   dZY n0 G dd„ dƒZdd„ Z	e
d	krde	ƒ  dS )
a˜  
A module for language identification using the TextCat algorithm.
An implementation of the text categorization algorithm
presented in Cavnar, W. B. and J. M. Trenkle,
"N-Gram-Based Text Categorization".

The algorithm takes advantage of Zipf's law and uses
n-gram frequencies to profile languages and text-yet to
be identified-then compares using a distance measure.

Language n-grams are provided by the "An Crubadan"
project. A corpus reader was created separately to read
those files.

For details regarding the algorithm, see:
https://www.let.rug.nl/~vannoord/TextCat/textcat.pdf

For details about An Crubadan, see:
https://borel.slu.edu/crubadan/index.html
é    )Úmaxsize)ÚtrigramsNc                   @   sP   e Zd ZdZi ZdZdZi Zdd„ Zdd„ Z	dd	„ Z
d
d„ Zdd„ Zdd„ ZdS )ÚTextCatNú<ú>c                 C   s>   t stdƒ‚ddlm} || _| j ¡ D ]}| j |¡ q(d S )Nz›classify.textcat requires the regex module that supports unicode. Try '$ pip install regex' and see https://pypi.python.org/pypi/regex for further details.r   )Úcrubadan)ÚreÚOSErrorÚnltk.corpusr   Ú_corpusÚlangsÚ	lang_freq)Úselfr   Úlang© r   úT/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/classify/textcat.pyÚ__init__7   s    ÿzTextCat.__init__c                 C   s   t  dd|¡S )z)Get rid of punctuation except apostrophesz[^\P{P}\']+Ú )r   Úsub©r   Útextr   r   r   Úremove_punctuationG   s    zTextCat.remove_punctuationc                 C   s„   ddl m}m} |  |¡}||ƒ}|ƒ }|D ]R}t| j| | j ƒ}dd„ |D ƒ}	|	D ]&}
|
|v rt||
  d7  < qVd||
< qVq,|S )z'Create FreqDist of trigrams within textr   )ÚFreqDistÚword_tokenizec                 S   s   g | ]}d   |¡‘qS )r   )Újoin)Ú.0Ztrir   r   r   Ú
<listcomp>U   ó    z#TextCat.profile.<locals>.<listcomp>é   )Znltkr   r   r   r   Ú_START_CHARÚ	_END_CHAR)r   r   r   r   Z
clean_textÚtokensÚfingerprintÚtZtoken_trigram_tuplesZtoken_trigramsZcur_trigramr   r   r   ÚprofileK   s    
zTextCat.profilec                 C   sR   | j  |¡}d}||v rJt| ¡ ƒ |¡}t| ¡ ƒ |¡}t|| ƒ}nt}|S )zgCalculate the "out-of-place" measure between the
        text and language profile for a single trigramr   )r   r   ÚlistÚkeysÚindexÚabsr   )r   r   ÚtrigramZtext_profileZlang_fdÚdistZidx_lang_profileZidx_textr   r   r   Ú	calc_dist_   s    zTextCat.calc_distc                 C   sL   i }|   |¡}| jj ¡ D ],}d}|D ]}||  |||¡7 }q&|||< q|S )zOCalculate the "out-of-place" measure between
        the text and all languagesr   )r$   r   Z_all_lang_freqr&   r+   )r   r   Z	distancesr$   r   Z	lang_distr)   r   r   r   Ú
lang_distst   s    

zTextCat.lang_distsc                 C   s   |   |¡| _t| j| jjd�S )zYFind the language with the min distance
        to the text and return its ISO 639-3 code)Úkey)r,   Úlast_distancesÚminÚgetr   r   r   r   Úguess_language†   s    zTextCat.guess_language)Ú__name__Ú
__module__Ú__qualname__r   Zfingerprintsr   r    r.   r   r   r$   r+   r,   r1   r   r   r   r   r   /   s   r   c            
   
      sì   ddl m}  g d¢}dddddd	d
dddœ	}tƒ }|D ]°}|  |¡‰tˆƒd }tttˆƒƒ}d}td|ƒD ]4‰ dd ‡ ‡fdd„td|ˆ  ƒD ƒ¡ }||7 }qlt	d|dd…  d ƒ | 
|¡}	t	d|	› d||	 › d�ƒ t	dƒ q6d S )Nr   )Úudhr)	zKurdish-UTF8zAbkhaz-UTF8zFarsi_Persian-UTF8z
Hindi-UTF8zHawaiian-UTF8zRussian-UTF8zVietnamese-UTF8zSerbian_Srpski-UTF8zEsperanto-UTF8zNorthern KurdishZ	AbkhazianzIranian PersianZHindiZHawaiianÚRussianZ
VietnameseZSerbianZ	Esperanto)	ZkmrZabkZpesZhinZhawZrusZvieZsrpZepor   r   ú c                    s   g | ]}ˆˆ  | ‘qS r   r   )r   Új©ÚiZraw_sentencesr   r   r   ¶   r   zdemo.<locals>.<listcomp>zLanguage snippet: éŒ   z...zLanguage detection: z (ú)zŒ############################################################################################################################################)r
   r5   r   ZsentsÚlenr%   ÚmapÚranger   Úprintr1   )
r5   r   ZfriendlyZtcZcur_langÚrowsÚcolsÚsampleZcur_sentÚguessr   r9   r   Údemo�   s2    ÷
(

rE   Ú__main__)Ú__doc__Úsysr   Z	nltk.utilr   Úregexr   ÚImportErrorr   rE   r2   r   r   r   r   Ú<module>	   s   
`1