a
     ˜ëjF:  ã                   @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	 d dl
mZmZ d dlmZmZ dd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd0d$d%„Zd&d'„ Zd1d(d)„Zed*d+gƒZeg d,¢ƒZ d-d.„ Z!e"d/k�reƒ  dS )2é    N)Útreebank)ÚBrillTaggerTrainerÚRegexpTaggerÚUnigramTagger)ÚPosÚWord)ÚTemplateÚ
error_listc                   C   s
   t ƒ  dS )z„
    Run a demo with defaults. See source comments for details,
    or docstrings of any of the more specific demo_* functions.
    N©Úpostag© r   r   úL/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/tbl/demo.pyÚdemo   s    r   c                   C   s   t dd� dS )úN
    Exemplify repr(Rule) (see also str(Rule) and Rule.format("verbose"))
    Úrepr©Ú
ruleformatNr
   r   r   r   r   Údemo_repr_rule_format   s    r   c                   C   s   t dd� dS )r   Ústrr   Nr
   r   r   r   r   Údemo_str_rule_format$   s    r   c                   C   s   t dd� dS )z*
    Exemplify Rule.format("verbose")
    Úverboser   Nr
   r   r   r   r   Údemo_verbose_rule_format+   s    r   c                   C   s   t ttg d¢ƒƒgd� dS )a¾  
    The feature/s of a template takes a list of positions
    relative to the current word where the feature should be
    looked for, conceptually joined by logical OR. For instance,
    Pos([-1, 1]), given a value V, will hold whenever V is found
    one step to the left and/or one step to the right.

    For contiguous ranges, a 2-arg form giving inclusive end
    points can also be used: Pos(-3, -1) is the same as the arg
    below.
    )éýÿÿÿéþÿÿÿéÿÿÿÿ©Ú	templatesN)r   r   r   r   r   r   r   Údemo_multiposition_feature2   s    r   c                   C   s$   t ttdgƒtddgƒƒgd� dS )z8
    Templates can have more than a single feature.
    r   r   r   r   N)r   r   r   r   r   r   r   r   Údemo_multifeature_templateA   s    r   c                   C   s   t ddd� dS )ah  
    Show aggregate statistics per template. Little used templates are
    candidates for deletion, much used templates may possibly be refined.

    Deleting unused templates is mostly about saving time and/or space:
    training is basically O(T) in the number of templates T
    (also in terms of memory usage, which often will be the limiting factor).
    T)Úincremental_statsÚtemplate_statsNr
   r   r   r   r   Údemo_template_statisticsH   s    	r!   c                  C   sj   t jg d¢ddgdd�} tjg d¢ddgdd�}ttj| |gdd	�ƒ}td
 t|ƒ¡ƒ t|ddd� dS )a	  
    Template.expand and Feature.expand are class methods facilitating
    generating large amounts of templates. See their documentation for
    details.

    Note: training with 500 templates can easily fill all available
    even on relatively small corpora
    )r   r   é   r"   é   F)Zexcludezero)r   r   r   r"   T)r"   é   )Úcombinationsz8Generated {} templates for transformation-based learning)r   r   r    N)	r   Úexpandr   Úlistr   ÚprintÚformatÚlenr   )ZwordtplsZtagtplsr   r   r   r   Údemo_generated_templatesT   s    	ÿÿr+   c                   C   s   t dddd� dS )z‚
    Plot a learning curve -- the contribution on tagging accuracy of
    the individual rules.
    Note: requires matplotlib
    Tzlearningcurve.png)r   Úseparate_baseline_dataÚlearning_curve_outputNr
   r   r   r   r   Údemo_learning_curveh   s
    ýr.   c                   C   s   t dd� dS )zW
    Writes a file with context for each erroneous word after tagging testing data
    z
errors.txt)Úerror_outputNr
   r   r   r   r   Údemo_error_analysisu   s    r0   c                   C   s   t dd� dS )zm
    Serializes the learned tagger to a file in pickle format; reloads it
    and validates the process.
    z
tagger.pcl)Úserialize_outputNr
   r   r   r   r   Údemo_serialize_tagger|   s    r2   c                   C   s   t dddd� dS )z˜
    Discard rules with low accuracy. This may hurt performance a bit,
    but will often produce rules which are more interesting read to a human.
    i¸  g¸…ëQ¸î?é
   )Ú	num_sentsÚmin_accÚ	min_scoreNr
   r   r   r   r   Údemo_high_accuracy_rules„   s    r7   éè  é,  r$   çš™™™™™é?Fr   c           &      C   s‚  |pt }| du r&ddlm}m} |ƒ } t|||||ƒ\}}}}|ràtj |¡sžt||d�}t	|dƒ�}t
 ||¡ W d  ƒ n1 s†0    Y  td |¡ƒ t	|ƒ�(}t
 |¡}td|› �ƒ W d  ƒ qô1 sÔ0    Y  nt||d�}tdƒ |�rtd	 | |¡¡ƒ t ¡ }t|| ||	d
�}tdƒ | ||||¡}tdt ¡ | d›d�ƒ |�rptd| |¡ ƒ |dk�r¸tdƒ t| ¡ dƒD ]&\}}t|d›d| |	¡d›�ƒ �q�|
�r*tdƒ | ||¡\} }!tdƒ |�sìtdƒ | ¡ }"|�r| |!¡ |�rJt||!|"|d� td|› �ƒ n tdƒ | |¡} |�rJ| ¡  |du�r¼t	|dƒ�>}#|# d| ¡ |# d t|| ƒ¡ d¡d ¡ W d  ƒ n1 �s¤0    Y  td|› �ƒ |du�r~| |¡} t	|dƒ�}t
 ||¡ W d  ƒ n1 �sþ0    Y  td|› �ƒ t	|ƒ�}t
 |¡}$W d  ƒ n1 �s@0    Y  td|› �ƒ | |¡}%| |%k�rvtd ƒ ntd!ƒ dS )"a’
  
    Brill Tagger Demonstration
    :param templates: how many sentences of training and testing data to use
    :type templates: list of Template

    :param tagged_data: maximum number of rule instances to create
    :type tagged_data: C{int}

    :param num_sents: how many sentences of training and testing data to use
    :type num_sents: C{int}

    :param max_rules: maximum number of rule instances to create
    :type max_rules: C{int}

    :param min_score: the minimum score for a rule in order for it to be considered
    :type min_score: C{int}

    :param min_acc: the minimum score for a rule in order for it to be considered
    :type min_acc: C{float}

    :param train: the fraction of the the corpus to be used for training (1=all)
    :type train: C{float}

    :param trace: the level of diagnostic tracing output to produce (0-4)
    :type trace: C{int}

    :param randomize: whether the training data should be a random subset of the corpus
    :type randomize: C{bool}

    :param ruleformat: rule output format, one of "str", "repr", "verbose"
    :type ruleformat: C{str}

    :param incremental_stats: if true, will tag incrementally and collect stats for each rule (rather slow)
    :type incremental_stats: C{bool}

    :param template_stats: if true, will print per-template statistics collected in training and (optionally) testing
    :type template_stats: C{bool}

    :param error_output: the file where errors will be saved
    :type error_output: C{string}

    :param serialize_output: the file where the learned tbl tagger will be saved
    :type serialize_output: C{string}

    :param learning_curve_output: filename of plot of learning curve(s) (train and also test, if available)
    :type learning_curve_output: C{string}

    :param learning_curve_take: how many rules plotted
    :type learning_curve_take: C{int}

    :param baseline_backoff_tagger: the file where rules will be saved
    :type baseline_backoff_tagger: tagger

    :param separate_baseline_data: use a fraction of the training data exclusively for training baseline
    :type separate_baseline_data: C{bool}

    :param cache_baseline_tagger: cache baseline tagger to this file (only interesting as a temporary workaround to get
                                  deterministic output from the baseline unigram tagger between python versions)
    :type cache_baseline_tagger: C{string}


    Note on separate_baseline_data: if True, reuse training data both for baseline and rule learner. This
    is fast and fine for a demo, but is likely to generalize worse on unseen data.
    Also cannot be sensibly used for learning curves on training data (the baseline will be artificially high).
    Nr   )Úbrill24Údescribe_template_sets)ÚbackoffÚwz)Trained baseline tagger, pickled it to {}zReloaded pickled tagger from zTrained baseline taggerz!    Accuracy on test set: {:0.4f}r   zTraining tbl tagger...zTrained tbl tagger in z0.2fz secondsz    Accuracy on test set: %.4fr"   z
Learned rules: Z4dú ÚszJIncrementally tagging the test data, collecting individual rule statisticsz    Rule statistics collectedzbWARNING: train_stats asked for separate_baseline_data=True; the baseline will be artificially high)Útakez Wrote plot of learning curve to zTagging the test datazErrors for Brill Tagger %r

Ú
zutf-8z)Wrote tagger errors including context to zWrote pickled tagger to z4Reloaded tagger tried on test set, results identicalz;PROBLEM: Reloaded tagger gave different results on test set)ÚREGEXP_TAGGERÚnltk.tag.brillr;   r<   Ú_demo_prepare_dataÚosÚpathÚexistsr   ÚopenÚpickleÚdumpr(   r)   ÚloadZaccuracyÚtimer   ÚtrainÚ	enumerateÚrulesZbatch_tag_incrementalZtrain_statsZprint_template_statisticsÚ
_demo_plotZ	tag_sentsÚwriteÚjoinr	   Úencode)&r   Útagged_datar4   Z	max_rulesr6   r5   rN   ÚtraceÚ	randomizer   r   r    r/   r1   r-   Zlearning_curve_takeZbaseline_backoff_taggerr,   Zcache_baseline_taggerr;   r<   Útraining_dataÚbaseline_dataÚ	gold_dataÚtesting_dataZbaseline_taggerZprint_rulesZtbrillZtrainerZbrill_taggerZrulenoZruleZ
taggedtestÚ	teststatsÚ
trainstatsÚfZbrill_tagger_reloadedZtaggedtest_reloadedr   r   r   r   Œ   s¨    X
ÿÿ*ÿÿ

.ÿÿÿ
 ÿÿÿ
ÿ

@

,
*


r   c                 C   s2  | d u rt dƒ t ¡ } |d u s,t| ƒ|kr4t| ƒ}|rPt t| ƒ¡ t | ¡ t|| ƒ}| d |… }| ||… }dd„ |D ƒ}|sŒ|}	n&t|ƒd }
|d |
… ||
d …  }	}t|ƒ\}}t|ƒ\}}t|	ƒ\}}t d|d›d|d›d�ƒ t d	|d›d|d›d�ƒ t d
 	|||�rdnd¡ƒ ||	||fS )Nz%Loading tagged data from treebank... c                 S   s   g | ]}d d„ |D ƒ‘qS )c                 S   s   g | ]}|d  ‘qS )r   r   )Ú.0Útr   r   r   Ú
<listcomp>a  ó    z1_demo_prepare_data.<locals>.<listcomp>.<listcomp>r   )r_   Úsentr   r   r   ra   a  rb   z&_demo_prepare_data.<locals>.<listcomp>r$   zRead testing data (Údz sents/z wds)zRead training data (z-Read baseline data ({:d} sents/{:d} wds) {:s}Ú z[reused the training set])
r(   r   Ztagged_sentsr*   ÚrandomÚseedÚshuffleÚintÚcorpus_sizer)   )rU   rN   r4   rW   r,   ÚcutoffrX   rZ   r[   rY   Z	bl_cutoffZ	trainseqsZtraintokensZtestseqsZ
testtokensZbltrainseqsZbltraintokensr   r   r   rE   Q  s>    


þýÿrE   c           	         sÐ   ˆ d g}ˆ d D ]}|  |d | ¡ q‡ fdd„|d |… D ƒ}ˆd g}ˆd D ]}|  |d | ¡ qV‡fdd„|d |… D ƒ}dd lm} ttt|ƒƒƒ}| ||||¡ | g d¢¡ | | ¡ d S )	NZinitialerrorsZ
rulescoresr   c                    s   g | ]}d |ˆ d   ‘qS ©r"   Z
tokencountr   ©r_   Úx)r\   r   r   ra   }  rb   z_demo_plot.<locals>.<listcomp>c                    s   g | ]}d |ˆ d   ‘qS rl   r   rm   )r]   r   r   ra   ‚  rb   r   )NNNg      ð?)	ÚappendZmatplotlib.pyplotZpyplotr'   Úranger*   ZplotZaxisZsavefig)	r-   r\   r]   rA   Z	testcurveZ	rulescoreZ
traincurveZpltÚrr   )r\   r]   r   rQ   y  s    

rQ   ©z^-?[0-9]+(\.[0-9]+)?$ZCD©z.*ÚNN)	rr   )z(The|the|A|a|An|an)$ÚAT)z.*able$ZJJ)z.*ness$rt   )z.*ly$ZRB)z.*s$ZNNS)z.*ing$ZVBG)z.*ed$ZVBDrs   c                 C   s   t | ƒtdd„ | D ƒƒfS )Nc                 s   s   | ]}t |ƒV  qd S )N)r*   rm   r   r   r   Ú	<genexpr>ž  rb   zcorpus_size.<locals>.<genexpr>)r*   Úsum)Zseqsr   r   r   rj   �  s    rj   Ú__main__)NNr8   r9   r$   Nr:   r$   Fr   FFNNNr9   NFN)NN)#rF   rJ   rf   rM   Znltk.corpusr   Znltk.tagr   r   r   rD   r   r   Znltk.tblr   r	   r   r   r   r   r   r   r!   r+   r.   r0   r2   r7   r   rE   rQ   ZNN_CD_TAGGERrC   rj   Ú__name__r   r   r   r   Ú<module>
   sb   	                   í
 F(
ÿ
