a
     ˜ëj~0  ã                   @   s  d Z ddlZddlZddlmZ ddlmZmZ zddl	m
Z
 W n eyR   Y n0 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ G dd„ deƒZG dd„ deƒZdd„ Zdd„ Zdd„ Zd#dd„Zdd„ Zdd„ ZG dd„ deƒZd$d d!„Ze d"k�redƒ edƒ dS )%z
Named entity chunker
é    N)ÚElementTree)ÚClassifierBasedTaggerÚpos_tag)ÚMaxentClassifier)ÚChunkParserI)Ú
ChunkScore©Úfind)Úword_tokenize)ÚTreec                   @   s2   e Zd ZdZddd„Zdd„ Zdd„ Zd	d
„ ZdS )ÚNEChunkParserTaggerz2
    The IOB tagger used by the chunk parser.
    Nc                 C   s   t j| || j|d� d S )N)ÚtrainZclassifier_builderÚ
classifier)r   Ú__init__Ú_classifier_builder)Úselfr   r   © r   úV/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/chunk/named_entity.pyr   $   s    üzNEChunkParserTagger.__init__c                 C   s   t j|dddd�S )NZiisé   é   )Ú	algorithmZgaussian_prior_sigmaÚtrace)r   r   ©r   r   r   r   r   r   ,   s    ûz'NEChunkParserTagger._classifier_builderc                 C   sD   z
| j }W n4 ty>   ddlm} t| d¡ƒ| _ | j }Y n0 |S )Nr   )Úwordszen-basic)Z_en_wordlistÚAttributeErrorZnltk.corpusr   Úset)r   Zwlr   r   r   r   Ú_english_wordlist5   s    
z%NEChunkParserTagger._english_wordlistc                 C   s0  || d }t || d ƒ}|dkrBd  }}d  }}	d  }
 }}nÂ|dkr”||d  d  ¡ }d }t ||d  d ƒ}d }	||d  d }d  }
}np||d  d  ¡ }||d  d  ¡ }t ||d  d ƒ}t ||d  d ƒ}	||d  }||d  }t|ƒ}
|t|ƒd k�r(d  }}d  }}n”|t|ƒd k�rl||d  d  ¡ }||d  d  ¡ }d }d }nP||d  d  ¡ }||d  d  ¡ }||d  d  ¡ }||d  d  ¡ }dt|ƒt|ƒ|d d…  ¡ |dd …  ¡ ||||  ¡ v |||||| ¡ › d|› �|› d|› �|
› d|› �dœ}|S )	Nr   r   r   Té   éýÿÿÿú+)ZbiasÚshapeZwordlenZprefix3Zsuffix3ÚposÚwordzen-wordlistÚprevtagÚprevposÚnextposÚprevwordÚnextwordzword+nextposzpos+prevtagzshape+prevtag)Úsimplify_posÚlowerr    Úlenr   )r   ÚtokensÚindexÚhistoryr"   r!   r&   Zprevprevwordr$   ZprevprevposZ	prevshaper#   Zprevprevtagr'   Znextnextwordr%   ZnextnextposÚfeaturesr   r   r   Ú_feature_detector?   sd    


ðz%NEChunkParserTagger._feature_detector)NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r/   r   r   r   r   r      s
   
	
r   c                   @   s<   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zed
d„ ƒZ	dS )ÚNEChunkParserú2
    Expected input: list of pos-tagged words
    c                 C   s   |   |¡ d S ©N)Ú_trainr   r   r   r   r      s    zNEChunkParser.__init__c                 C   s   | j  |¡}|  |¡}|S )z8
        Each token should be a pos-tagged word
        )Ú_taggerÚtagÚ_tagged_to_parse)r   r+   ZtaggedÚtreer   r   r   Úparse‚   s    
zNEChunkParser.parsec                    s"   ‡ fdd„|D ƒ}t |d�ˆ _d S )Nc                    s   g | ]}ˆ   |¡‘qS r   )Ú_parse_to_tagged)Ú.0Ús©r   r   r   Ú
<listcomp>Œ   ó    z(NEChunkParser._train.<locals>.<listcomp>)r   )r   r8   )r   Zcorpusr   r@   r   r7   Š   s    zNEChunkParser._trainc                 C   s´   t dg ƒ}|D ] \}}|dkr*| |¡ q| d¡rP| t |dd… |gƒ¡ q| d¡r|r”t|d t ƒr”|d  ¡ |dd… kr”|d  |¡ q| t |dd… |gƒ¡ q|S )zH
        Convert a list of tagged tokens to a chunk-parse tree.
        ÚSÚOúB-r   NúI-éÿÿÿÿ)r   ÚappendÚ
startswithÚ
isinstanceÚlabel)r   Ztagged_tokensÚsentÚtokr9   r   r   r   r:   �   s    


*zNEChunkParser._tagged_to_parsec                 C   sˆ   g }| D ]z}t |tƒrtt|ƒdkr,tdƒ q| |d d| ¡ › �f¡ |dd… D ]}| |d| ¡ › �f¡ qTq| |df¡ q|S )zH
        Convert a chunk-parse tree to a list of tagged tokens.
        r   z"Warning -- empty chunk in sentencerE   r   NrF   rD   )rJ   r   r*   ÚprintrH   rK   )rL   ÚtoksÚchildrM   r   r   r   r=   ¢   s    
zNEChunkParser._parse_to_taggedN)
r0   r1   r2   r3   r   r<   r7   r:   Ústaticmethodr=   r   r   r   r   r4   z   s   r4   c                 C   s^   t  d| t j¡rdS t  d| t j¡r(dS t  d| t j¡rV|  ¡ rDdS |  ¡ rPdS dS nd	S d S )
Nz![0-9]+(\.[0-9]*)?|[0-9]*\.[0-9]+$Únumberz\W+$Úpunctz\w+$ZupcaseZdowncaseZ	mixedcaseÚother)ÚreÚmatchÚUNICODEÚistitleÚislower)r"   r   r   r   r    µ   s    r    c                 C   s    |   d¡rdS |  d¡d S d S )NÚVú-r   )rI   Úsplit)r?   r   r   r   r(   Å   s    
r(   c                 C   s„   |   ¡ }dd„ t|ƒD ƒ}tdg ƒ}| D ]V}t|tƒrl| t| ¡ g ƒ¡ |D ]}|d  |t|ƒf¡ qNq(| |t|ƒf¡ q(|S )Nc                 s   s   | ]\}}|V  qd S r6   r   )r>   r"   r!   r   r   r   Ú	<genexpr>Ï   rB   zpostag_tree.<locals>.<genexpr>rC   rG   )Zleavesr   r   rJ   rH   rK   Únext)r;   r   Ztag_iterZnewtreerP   Zsubchildr   r   r   Úpostag_treeÌ   s    

r_   ÚbinaryTc                 c   sb   | D ]X}t  |¡D ]H\}}}| d¡r,|r,q|D ](}| d¡r0tt j ||¡|ƒE d H  q0qqd S )NZbnewsz.sgm)ÚosÚwalkÚendswithÚload_ace_fileÚpathÚjoin)ÚrootsÚfmtZ
skip_bnewsÚrootÚdirsÚfilesÚfr   r   r   Úload_ace_dataÛ   s    
rm   c                 c   s   t dtj | ¡d › �ƒ | d }g }t|ƒ�}t |¡ ¡ }W d   ƒ n1 sR0    Y  | d¡D ]d}| 	d¡j
}| d¡D ]H}| d¡dkr”q€t| 	d	¡j
ƒ}	t| 	d
¡j
ƒd }
| |	|
|f¡ q€qft| ƒ�}| ¡ }W d   ƒ n1 sò0    Y  t dd|¡}dd„ }t d||¡}t dd|¡}t dd|¡}t dd|¡}dd„ |D ƒ}|dk�röd}tdg ƒ}t|ƒD ]^\}	}
}|	|k �r�|}	|
|	k�rž�qx| t|||	… ƒ¡ | td||	|
…  ¡ ƒ¡ |
}�qx| t||d … ƒ¡ |V  n¦|dk�r”d}tdg ƒ}t|ƒD ]^\}	}
}|	|k �r.|}	|
|	k�r<�q| t|||	… ƒ¡ | t|||	|
…  ¡ ƒ¡ |
}�q| t||d … ƒ¡ |V  ntdƒ‚d S )Nz  - r   z.tmx.rdc.xmlzdocument/entityZentity_typeZentity_mentionZTYPEÚNAMEzhead/charseq/startzhead/charseq/endz<(?!/?TEXT)[^>]+>Ú c                 S   s   d|   ¡ |  ¡  d  S )Nú é   )ÚendÚstart)Úmr   r   r   Úsubfuncþ   s    zload_ace_file.<locals>.subfuncz[\s\S]*<TEXT>z</TEXT>[\s\S]*z``z "z''z" c                 S   s   h | ]\}}}|’qS r   r   )r>   r?   ÚeÚtypr   r   r   Ú	<setcomp>  rB   z load_ace_file.<locals>.<setcomp>r`   r   rC   ZNEÚ
multiclasszbad fmt value)rN   ra   re   r\   ÚopenÚETr<   ÚgetrootÚfindallr	   ÚtextÚgetÚintrH   ÚreadrU   Úsubr   ÚsortedÚextendr
   Ú
ValueError)Ztextfilerh   ZannfileÚentitiesÚinfileÚxmlÚentityrw   Zmentionr?   rv   r~   ru   Zentity_typesÚirO   r   r   r   rd   å   sb    
,
&







rd   c                 C   s¬   t  | ¡} t  |¡}d}t| |ƒD ]„\\}}\}}||  krFdkr„n n:|s¦td|d›d|d›d|› �ƒ td ddd¡ƒ d}q"d}td|d›d|d›d|› �ƒ q"d S )	NFrD   z  Z15rp   z  {:15} {:15} {2}z...T)r4   r=   ÚziprN   Úformat)ÚcorrectZguessedZellipsisÚwÚctÚgtr   r   r   Ú
cmp_chunks.  s    

r‘   c                   @   s*   e Zd ZdZd
dd„Zdd„ Zdd„ Zd	S )ÚMaxent_NE_Chunkerr5   ry   c                 C   s0   ddl m} || _|d|› d�ƒ| _|  ¡  d S )Nr   r   z+chunkers/maxent_ne_chunker_tab/english_ace_ú/)Ú	nltk.datar	   Ú_fmtÚ_tab_dirÚload_params)r   rh   r	   r   r   r   r   E  s    zMaxent_NE_Chunker.__init__c                 C   sF   ddl m}m} || jƒ\}}}}t||||d�|ƒ}t|d�| _d S )Nr   )ÚBinaryMaxentFeatureEncodingÚload_maxent_params)Zalwayson_features)r   )Únltk.classify.maxentr˜   r™   r–   r   r   r8   )r   r˜   r™   ÚwgtÚmpgÚlabÚaonZmcr   r   r   r—   L  s    ÿzMaxent_NE_Chunker.load_paramsc           	      C   sV   ddl m} | jj}|j}|j}|j}|j}|j}| j	}|||||d|› d�d� d S )Nr   )Úsave_maxent_paramsz/tmp/english_ace_r“   )Ztab_dir)
rš   rŸ   r8   Z_classifierÚ	_encodingZ_weightsÚ_mappingZ_labelsZ	_alwaysonr•   )	r   rŸ   ZclassifZecgr›   rœ   r�   rž   rh   r   r   r   Úsave_paramsU  s    zMaxent_NE_Chunker.save_paramsN)ry   )r0   r1   r2   r3   r   r—   r¢   r   r   r   r   r’   @  s   
	r’   ry   c                 C   s   t | ƒ}| ¡  |S r6   )r’   r¢   )rh   Zchunkerr   r   r   Úbuild_modelb  s    r£   Ú__main__)r`   T)ry   )!r3   ra   rU   Z	xml.etreer   r{   Znltk.tagr   r   Znltk.classifyr   ÚImportErrorZnltk.chunk.apir   Znltk.chunk.utilr   r”   r	   Znltk.tokenizer
   Z	nltk.treer   r   r4   r    r(   r_   rm   rd   r‘   r’   r£   r0   r   r   r   r   Ú<module>	   s4   [;

I"
2
