a
     ˜ëj|   ã                   @   s@   d dl Z d dlmZ d dlmZ d dlmZ G dd„ deƒZdS )é    N)Úwarn)ÚElementTree)ÚCorpusReaderc                       sh   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Z‡  ZS )ÚBCP47CorpusReaderu~  
    Parse BCP-47 composite language tags

    Supports all the main subtags, and the 'u-sd' extension:

    >>> from nltk.corpus import bcp47
    >>> bcp47.name('oc-gascon-u-sd-fr64')
    'Occitan (post 1500): Gascon: PyrÃ©nÃ©es-Atlantiques'

    Can load a conversion table to Wikidata Q-codes:
    >>> bcp47.load_wiki_q()
    >>> bcp47.wiki_q['en-GI-spanglis']
    'Q79388'

    c                    s¢   t ƒ  ||¡ i | _|  d¡�&}|  | ¡  d¡¡| _W d  ƒ n1 sJ0    Y  |  d¡�(}|  t	 
|¡ d¡¡| _W d  ƒ n1 sŒ0    Y  |  ¡  dS )zRead the BCP-47 databasez!iana/language-subtag-registry.txtz%%
Nzcldr/common-subdivisions-en.xmlz+localeDisplayNames/subdivisions/subdivision)ÚsuperÚ__init__ÚlangcodeÚopenÚ	data_dictÚreadÚsplitÚdbÚsubdiv_dictÚetÚparseÚiterfindÚsubdivÚ
morphology)ÚselfÚrootZfileidsÚfp©Ú	__class__© úW/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/corpus/reader/bcp47.pyr       s    4ÿ$zBCP47CorpusReader.__init__c                 C   sP   |   d¡�2}|  | ¡  ¡  d¡dd… ¡| _W d  ƒ n1 sB0    Y  dS )z:Load conversion table to Wikidata Q-codes (only if needed)z-cldr/tools-cldr-rdf-external-entityToCode.tsvÚ
é   N)r	   Ú	wiki_dictr   Ústripr   Zwiki_q)r   r   r   r   r   Úload_wiki_q,   s    zBCP47CorpusReader.load_wiki_qc                 C   s   dd„ dd„ |D ƒD ƒS )z7Convert Wikidata list of Q-codes to a BCP-47 dictionaryc                 S   s$   i | ]}|d  |d   d¡d “qS )r   r   ú/éÿÿÿÿ©r   )Ú.0Úpairr   r   r   Ú
<dictcomp>3   s   ÿz/BCP47CorpusReader.wiki_dict.<locals>.<dictcomp>c                 S   s   g | ]}|  ¡  d ¡‘qS )ú	)r   r   )r#   Úliner   r   r   Ú
<listcomp>5   ó    z/BCP47CorpusReader.wiki_dict.<locals>.<listcomp>r   )r   Úlinesr   r   r   r   1   s    þzBCP47CorpusReader.wiki_dictc                 C   s   dd„ |D ƒS )z2Convert the CLDR subdivisions list to a dictionaryc                 S   s   i | ]}|j d  |j“qS )Útype)ÚattribÚtext)r#   Úsubr   r   r   r%   :   r)   z1BCP47CorpusReader.subdiv_dict.<locals>.<dictcomp>r   )r   Zsubdivsr   r   r   r   8   s    zBCP47CorpusReader.subdiv_dictc              
   C   s°   t jt jt jt jt jdœ| _d}d}d}d}t |d › d�¡t |d › ¡t |› |d › �¡t d|d	 › d
|d › d�¡t |d › |d d › �¡t |› ¡dœ| _d S )N)ÚlanguageÚextlangÚscriptÚregionÚvariantz[0-9]z[a-z]z[A-Z]z[a-zA-Z0-9]é   ú?ú(é   z)|(ú)é   )r/   r0   r1   r2   r3   Ú	singleton)ÚstrÚlowerÚtitleÚupperÚcasingÚreÚcompileÚformat)r   ÚdigÚlowZupZalnumr   r   r   r   <   s"    û
úzBCP47CorpusReader.morphologyc                 C   s~  |d   dd¡ ¡ | _i }i |d< dD ]}i |d |< q&|dd… D �]2}dd	„ | ¡  d
¡D ƒ}|d d }|d d }||vrŠi ||< i }|dd… D ]†}	t|	ƒdkrÖ|	\}
}|
|vrÆ|g||
< qö||
  |¡ n ||
 d  d|	d  ¡  7  < d|vrš|dkrš|
dkrš|| j||
 d < qš|D ](}
t||
 ƒdk�r&||
 d ||
< �q&d|v �rl||d | |< qD||| |< qD|S )z;Convert the BCP-47 language subtag registry to a dictionaryr   z
File-Date:Ú Ú
deprecated)r/   r0   r1   r2   r3   Ú	redundantÚgrandfatheredr   Nc                 S   s   g | ]}|  d ¡‘qS )ú: r"   )r#   Úfieldr   r   r   r(   a   r)   z/BCP47CorpusReader.data_dict.<locals>.<listcomp>r   r7   r!   ú Z
Deprecatedr/   ÚDescription)Úreplacer   Úversionr   ÚlenÚappendr   )r   ÚrecordsZdicÚlabelÚrecordÚfieldsÚtypÚtagZ	subfieldsrJ   ÚkeyÚvalr   r   r   r
   Q   sB    	 ÿþý
zBCP47CorpusReader.data_dictc                 C   s   t |ƒtkr|d }|S )zReturn only first valuer   )r+   Úlist)r   rX   r   r   r   Úval2str   s    zBCP47CorpusReader.val2strc                 C   s2   |d › }dD ]}||v r|d|| › �7 }q|S )zConcatenate subtag valuesr/   )r0   r1   r2   r3   Ú	extensionrI   r   )r   Z	lg_recordÚnamerR   r   r   r   Úlang2str†   s
    
zBCP47CorpusReader.lang2strc                 C   sø  |  d¡}i }g d¢}|�rô|�rô| d¡}d}|�rV| d¡}| j| |ƒ}| j|  |¡r0|| j| v r¼d}|  | j| | d ¡}|dkr®||v r®||  d| 7  < n|||< �qVq0|| jd	 | v r0d}d
|›d|› d�}	d| jd	 | | v �r(| jd	 | | d }
|	d|  |
¡› d�7 }	|  | jd	 | | d ¡||< t|	ƒ �qVq0|s|dk�r |d dk�r |d }|| jv �r’| j| }nd|› d�}nF|› d 	dd„ |D ƒ¡› � 
¡ }| jd  |¡�sæd|› d�}t|ƒ ||d< g }q|S )z8Convert a BCP-47 tag to a dictionary of labelled subtagsú-)r/   r0   r1   r2   r3   r3   r   FTrL   r3   rI   rF   úThe rK   z code is deprecatedúPreferred-Valuez', prefer 'ú'ÚuÚsdr   z<Unknown subdivision: ú>rE   c                 S   s   g | ]}d | ‘qS )r^   r   )r#   Úextr   r   r   r(   ·   r)   z/BCP47CorpusReader.parse_tag.<locals>.<listcomp>r:   z<Invalid extension: r[   )r   Úpopr?   rB   Ú	fullmatchr   rZ   r   r   Újoinr<   )r   rV   ZsubtagsÚlangÚlabelsZsubtagÚfoundrR   ZvalstrÚnoteÚpreferrc   re   r   r   r   Ú	parse_tagŽ   sV    


ÿÿ zBCP47CorpusReader.parse_tagc                 C   s  dD ]Ì}d}|| j | v r@| j | | d › }d|›d|› �}n||| j d | v r¼| j d | | d › }d|›d|› d�}d| j d | | v r¼| j d | | d }|d	|  |¡›�7 }|rt|ƒ |  S qz|  |  |¡¡W S    td
|›d�ƒ Y dS 0 dS )z×
        Convert a BCP-47 tag to a colon-separated string of subtag names

        >>> from nltk.corpus import bcp47
        >>> bcp47.name('ca-Latn-ES-valencia')
        'Catalan: Latin: Spain: Valencian'

        )rG   rH   NrL   r_   z	 code is rF   z and deprecatedr`   z	, prefer zTag z was not recognized)r   rZ   r   r]   rn   )r   rV   rR   rX   rl   rm   r   r   r   r\   ¿   s&    	
zBCP47CorpusReader.name)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r
   rZ   r]   rn   r\   Ú__classcell__r   r   r   r   r      s   .1r   )	r@   Úwarningsr   Z	xml.etreer   r   Znltk.corpus.readerr   r   r   r   r   r   Ú<module>   s   