a
     ˜ëj   ã                   @   s`   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 G dd„ de
ƒZdS )é    N)ÚZipFilePathPointer)Úfind_dir)Ú
TokenizerIc                   @   sT   e Zd ZdZddd„Zdd„ Zddd	„Zd
d„ Zedd„ ƒZ	edd„ ƒZ
dd„ ZdS )ÚReppTokenizeraé  
    A class for word tokenization using the REPP parser described in
    Rebecca Dridan and Stephan Oepen (2012) Tokenization: Returning to a
    Long Solved Problem - A Survey, Contrastive  Experiment, Recommendations,
    and Toolkit. In ACL. http://anthology.aclweb.org/P/P12/P12-2.pdf#page=406

    >>> sents = ['Tokenization is widely regarded as a solved problem due to the high accuracy that rulebased tokenizers achieve.' ,
    ... 'But rule-based tokenizers are hard to maintain and their rules language specific.' ,
    ... 'We evaluated our method on three languages and obtained error rates of 0.27% (English), 0.35% (Dutch) and 0.76% (Italian) for our best models.'
    ... ]
    >>> tokenizer = ReppTokenizer('/home/alvas/repp/') # doctest: +SKIP
    >>> for sent in sents:                             # doctest: +SKIP
    ...     tokenizer.tokenize(sent)                   # doctest: +SKIP
    ...
    (u'Tokenization', u'is', u'widely', u'regarded', u'as', u'a', u'solved', u'problem', u'due', u'to', u'the', u'high', u'accuracy', u'that', u'rulebased', u'tokenizers', u'achieve', u'.')
    (u'But', u'rule-based', u'tokenizers', u'are', u'hard', u'to', u'maintain', u'and', u'their', u'rules', u'language', u'specific', u'.')
    (u'We', u'evaluated', u'our', u'method', u'on', u'three', u'languages', u'and', u'obtained', u'error', u'rates', u'of', u'0.27', u'%', u'(', u'English', u')', u',', u'0.35', u'%', u'(', u'Dutch', u')', u'and', u'0.76', u'%', u'(', u'Italian', u')', u'for', u'our', u'best', u'models', u'.')

    >>> for sent in tokenizer.tokenize_sents(sents): # doctest: +SKIP
    ...     print(sent)                              # doctest: +SKIP
    ...
    (u'Tokenization', u'is', u'widely', u'regarded', u'as', u'a', u'solved', u'problem', u'due', u'to', u'the', u'high', u'accuracy', u'that', u'rulebased', u'tokenizers', u'achieve', u'.')
    (u'But', u'rule-based', u'tokenizers', u'are', u'hard', u'to', u'maintain', u'and', u'their', u'rules', u'language', u'specific', u'.')
    (u'We', u'evaluated', u'our', u'method', u'on', u'three', u'languages', u'and', u'obtained', u'error', u'rates', u'of', u'0.27', u'%', u'(', u'English', u')', u',', u'0.35', u'%', u'(', u'Dutch', u')', u'and', u'0.76', u'%', u'(', u'Italian', u')', u'for', u'our', u'best', u'models', u'.')
    >>> for sent in tokenizer.tokenize_sents(sents, keep_token_positions=True): # doctest: +SKIP
    ...     print(sent)                                                         # doctest: +SKIP
    ...
    [(u'Tokenization', 0, 12), (u'is', 13, 15), (u'widely', 16, 22), (u'regarded', 23, 31), (u'as', 32, 34), (u'a', 35, 36), (u'solved', 37, 43), (u'problem', 44, 51), (u'due', 52, 55), (u'to', 56, 58), (u'the', 59, 62), (u'high', 63, 67), (u'accuracy', 68, 76), (u'that', 77, 81), (u'rulebased', 82, 91), (u'tokenizers', 92, 102), (u'achieve', 103, 110), (u'.', 110, 111)]
    [(u'But', 0, 3), (u'rule-based', 4, 14), (u'tokenizers', 15, 25), (u'are', 26, 29), (u'hard', 30, 34), (u'to', 35, 37), (u'maintain', 38, 46), (u'and', 47, 50), (u'their', 51, 56), (u'rules', 57, 62), (u'language', 63, 71), (u'specific', 72, 80), (u'.', 80, 81)]
    [(u'We', 0, 2), (u'evaluated', 3, 12), (u'our', 13, 16), (u'method', 17, 23), (u'on', 24, 26), (u'three', 27, 32), (u'languages', 33, 42), (u'and', 43, 46), (u'obtained', 47, 55), (u'error', 56, 61), (u'rates', 62, 67), (u'of', 68, 70), (u'0.27', 71, 75), (u'%', 75, 76), (u'(', 77, 78), (u'English', 78, 85), (u')', 85, 86), (u',', 86, 87), (u'0.35', 88, 92), (u'%', 92, 93), (u'(', 94, 95), (u'Dutch', 95, 100), (u')', 100, 101), (u'and', 102, 105), (u'0.76', 106, 110), (u'%', 110, 111), (u'(', 112, 113), (u'Italian', 113, 120), (u')', 120, 121), (u'for', 122, 125), (u'our', 126, 129), (u'best', 130, 134), (u'models', 135, 141), (u'.', 141, 142)]
    Úutf8c                 C   s    |   |¡| _t ¡ | _|| _d S )N)Úfind_repptokenizerÚrepp_dirÚtempfileÚ
gettempdirÚworking_dirÚencoding)Úselfr   r   © r   úQ/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/tokenize/repp.pyÚ__init__6   s    
zReppTokenizer.__init__c                 C   s   t |  |g¡ƒS )zÈ
        Use Repp to tokenize a single sentence.

        :param sentence: A single sentence string.
        :type sentence: str
        :return: A tuple of tokens.
        :rtype: tuple(str)
        )ÚnextÚtokenize_sents)r   Zsentencer   r   r   Útokenize=   s    	zReppTokenizer.tokenizeFc           
      c   s¦   t jd| jddd��~}|D ]}| t|ƒd ¡ q| ¡  |  |j¡}|  |¡ 	| j
¡ ¡ }|  |¡D ]}|s|t|Ž \}}}	|V  qfW d  ƒ n1 s˜0    Y  dS )zà
        Tokenize multiple sentences using Repp.

        :param sentences: A list of sentence strings.
        :type sentences: list(str)
        :return: A list of tuples of tokens
        :rtype: iter(tuple(str))
        zrepp_input.ÚwF)ÚprefixÚdirÚmodeÚdeleteÚ
N)r	   ÚNamedTemporaryFiler   ÚwriteÚstrÚcloseÚgenerate_repp_commandÚnameÚ_executeÚdecoder   ÚstripÚparse_repp_outputsÚzip)
r   Ú	sentencesZkeep_token_positionsZ
input_fileÚsentÚcmdÚrepp_outputZtokenized_sentZstartsZendsr   r   r   r   H   s    	
ÿzReppTokenizer.tokenize_sentsc                 C   s8   | j d g}|d| j d g7 }|ddg7 }||g7 }|S )z«
        This module generates the REPP command to be used at the terminal.

        :param inputfilename: path to the input file
        :type inputfilename: str
        ú	/src/reppz-cú/erg/repp.setz--formatZtriple)r   )r   Zinputfilenamer'   r   r   r   r   b   s
    
z#ReppTokenizer.generate_repp_commandc                 C   s$   t j| t jt jd�}| ¡ \}}|S )N)ÚstdoutÚstderr)Ú
subprocessÚPopenÚPIPEÚcommunicate)r'   Úpr+   r,   r   r   r   r    o   s    zReppTokenizer._executec                 c   sN   t  dt j¡}|  d¡D ]0}dd„ | |¡D ƒ}tdd„ |D ƒƒ}|V  qdS )aZ  
        This module parses the tri-tuple format that REPP outputs using the
        "--format triple" option and returns an generator with tuple of string
        tokens.

        :param repp_output:
        :type repp_output: type
        :return: an iterable of the tokenized sentences as tuples of strings
        :rtype: iter(tuple)
        z^\((\d+), (\d+), (.+)\)$z

c                 S   s$   g | ]\}}}|t |ƒt |ƒf‘qS r   )Úint)Ú.0ÚstartÚendÚtokenr   r   r   Ú
<listcomp>ƒ   s   ÿz4ReppTokenizer.parse_repp_outputs.<locals>.<listcomp>c                 s   s   | ]}|d  V  qdS )é   Nr   )r3   Útr   r   r   Ú	<genexpr>‡   ó    z3ReppTokenizer.parse_repp_outputs.<locals>.<genexpr>N)ÚreÚcompileÚ	MULTILINEÚsplitÚfindallÚtuple)r(   Z
line_regexÚsectionZwords_with_positionsÚwordsr   r   r   r#   u   s    þz ReppTokenizer.parse_repp_outputsc                 C   sJ   t j |¡r|}nt|dd�}t j |d ¡s2J ‚t j |d ¡sFJ ‚|S )zX
        A module to find REPP tokenizer binary and its *repp.set* config file.
        )ZREPP_TOKENIZER)Zenv_varsr)   r*   )ÚosÚpathÚexistsr   )r   Zrepp_dirnameZ	_repp_dirr   r   r   r   Š   s    z ReppTokenizer.find_repptokenizerN)r   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   Ústaticmethodr    r#   r   r   r   r   r   r      s    



r   )rD   r<   r-   Úsysr	   Z	nltk.datar   Znltk.internalsr   Znltk.tokenize.apir   r   r   r   r   r   Ú<module>
   s   