a
     ˜ëjæ  ã                   @   sT   d Z ddlmZ ddlmZ ddlmZ G dd„ dƒZdd„ Zd	d
„ Z	ddd„Z
dS )aˆ  
Simple classifier for RTE corpus.

It calculates the overlap in words and named entities between text and
hypothesis, and also whether there are words / named entities in the
hypothesis which fail to occur in the text, since this is an indicator that
the hypothesis is more informative than (i.e not entailed by) the text.

TO DO: better Named Entity classification
TO DO: add lemmatization
é    )ÚMaxentClassifier)Úaccuracy)ÚRegexpTokenizerc                   @   sF   e Zd ZdZddd„Zddd„Zddd	„Zed
d„ ƒZedd„ ƒZ	dS )ÚRTEFeatureExtractorz™
    This builds a bag of words for both the text and the hypothesis after
    throwing away some stopwords, then calculates overlap and difference.
    TFc                    sÖ   |ˆ _ h d£ˆ _h d£ˆ _tdƒ}| |j¡ˆ _| |j¡ˆ _t	ˆ jƒˆ _
t	ˆ jƒˆ _|r†‡ fdd„ˆ jD ƒˆ _
‡ fdd„ˆ jD ƒˆ _ˆ j r¨ˆ j
ˆ j ˆ _
ˆ jˆ j ˆ _ˆ jˆ j
@ ˆ _ˆ jˆ j
 ˆ _ˆ j
ˆ j ˆ _dS )z­
        :param rtepair: a ``RTEPair`` from which features should be extracted
        :param stop: if ``True``, stopwords are thrown away.
        :type stop: bool
        >   ZhaveÚinÚtoZtheyÚwereZtheú,Ú.ZareÚitÚandZveryZofÚaÚis>   ÚneverZrejectedÚfailedZdeniedÚnoÚnotz[\w.@:/]+|\w+|\$[\d.]+c                    s   h | ]}ˆ   |¡’qS © ©Ú
_lemmatize©Ú.0Útoken©Úselfr   úY/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/classify/rte_classify.pyÚ	<setcomp>D   ó    z/RTEFeatureExtractor.__init__.<locals>.<setcomp>c                    s   h | ]}ˆ   |¡’qS r   r   r   r   r   r   r   E   r   N)ÚstopZ	stopwordsÚnegwordsr   ÚtokenizeÚtextZtext_tokensZhypZ
hyp_tokensÚsetÚ
text_wordsÚ	hyp_wordsÚ_overlapÚ
_hyp_extraZ
_txt_extra)r   Úrtepairr   Zuse_lemmatizeÚ	tokenizerr   r   r   Ú__init__   s"    

zRTEFeatureExtractor.__init__c                    sd   ‡ fdd„ˆ j D ƒ}|dkr.|r*td|ƒ |S |dkrT|rJtdˆ j | ƒ ˆ j | S td| ƒ‚dS )	z°
        Compute the overlap between text and hypothesis.

        :param toktype: distinguish Named Entities from ordinary words
        :type toktype: 'ne' or 'word'
        c                    s   h | ]}ˆ   |¡r|’qS r   ©Ú_ner   r   r   r   r   V   r   z.RTEFeatureExtractor.overlap.<locals>.<setcomp>Únez
ne overlapÚwordzword overlapzType not recognized:'%s'N)r%   ÚprintÚ
ValueError)r   ÚtoktypeÚdebugÚ
ne_overlapr   r   r   ÚoverlapO   s    

zRTEFeatureExtractor.overlapc                    sB   ‡ fdd„ˆ j D ƒ}|dkr |S |dkr2ˆ j | S td| ƒ‚dS )z²
        Compute the extraneous material in the hypothesis.

        :param toktype: distinguish Named Entities from ordinary words
        :type toktype: 'ne' or 'word'
        c                    s   h | ]}ˆ   |¡r|’qS r   r*   r   r   r   r   r   i   r   z0RTEFeatureExtractor.hyp_extra.<locals>.<setcomp>r,   r-   zType not recognized: '%s'N)r&   r/   )r   r0   r1   Zne_extrar   r   r   Ú	hyp_extrab   s    
zRTEFeatureExtractor.hyp_extrac                 C   s   |   ¡ s|  ¡ rdS dS )zz
        This just assumes that words in all caps or titles are
        named entities.

        :type token: str
        TF)ÚistitleÚisupper)r   r   r   r   r+   q   s    zRTEFeatureExtractor._nec                 C   s,   ddl m} |j| |jd�}|dur(|S | S )zI
        Use morphy from WordNet to find the base form of verbs.
        r   )Úwordnet)ÚposN)Únltk.corpusr7   ZmorphyZVERB)r-   ZwnZlemmar   r   r   r   }   s
    zRTEFeatureExtractor._lemmatizeN)TF)F)T)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r)   r3   r4   Ústaticmethodr+   r   r   r   r   r   r      s   
0


r   c                 C   sˆ   t | ƒ}i }d|d< t| d¡ƒ|d< t| d¡ƒ|d< t| d¡ƒ|d< t| d¡ƒ|d< t|j|j@ ƒ|d	< t|j|j@ ƒ|d
< |S )NTZalwaysonr-   Zword_overlapZword_hyp_extrar,   r2   Zne_hyp_extraZneg_txtZneg_hyp)r   Úlenr3   r4   r   r#   r$   )r'   Z	extractorÚfeaturesr   r   r   Úrte_featuresŠ   s    rA   c                 C   s   dd„ | D ƒS )Nc                 S   s   g | ]}t |ƒ|jf‘qS r   )rA   Úvalue)r   Úpairr   r   r   Ú
<listcomp>˜   r   z!rte_featurize.<locals>.<listcomp>r   )Z	rte_pairsr   r   r   Úrte_featurize—   s    rE   Nc           
      C   s¾   ddl m} | g d¢¡}| g d¢¡}|d urH|d |… }|d |… }t|ƒ}t|ƒ}tdƒ | dv rvt || ¡}n&| dv rŒt || ¡}ntdƒ}t|ƒ‚td	ƒ t	||ƒ}	td
|	 ƒ |S )Nr   )Úrte)zrte1_dev.xmlzrte2_dev.xmlzrte3_dev.xml)zrte1_test.xmlzrte2_test.xmlzrte3_test.xmlzTraining classifier...)Zmegam)ZGISZIISzFRTEClassifier only supports these algorithms:
 'megam', 'GIS', 'IIS'.
zTesting classifier...zAccuracy: %6.4f)
r9   rF   ÚpairsrE   r.   r   ZtrainÚstrÚ	Exceptionr   )
Ú	algorithmZsample_NZ
rte_corpusZ	train_setZtest_setZfeaturized_train_setZfeaturized_test_setZclfÚerr_msgÚaccr   r   r   Úrte_classifier›   s*    ÿ
rM   )N)r=   Znltk.classify.maxentr   Znltk.classify.utilr   Znltk.tokenizer   r   rA   rE   rM   r   r   r   r   Ú<module>   s   q