a
     ˜ëj!  ã                   @   s¨   d Z ddlZzddlmZ ddlmZ W n* eyR   dd„ Zdd„ Zd	d
„ ZY n0 e 	d¡Z
G dd„ dƒZdd„ Zdd„ Zefdd„Zefdd„Zdd„ Zdd„ ZdS )zž

A port of the Gale-Church Aligner.

Gale & Church (1993), A Program for Aligning Sentences in Bilingual Corpora.
https://aclweb.org/anthology/J93-1004.pdf

é    N)Úlogsf)Únormc                 C   s�   t | ƒ}ddd|   }|t | | d |d|d|d|d|d|d	|d
|d|d                  ¡ }| dkr„|S d| S dS )zComplementary error function.é   ç      à?gÅ›ˆÀ‰?ô?gˆ5�Ô ð?gê`ygñ×?gðÃÞÙÆ¸?g¥êª‚IØÇ¿gæ#vùØÑ?g€9¦Ë)ò¿g¾¡SöÐ÷?gýÞ…1Oê¿g¤¥v¯(ßÅ?g        g       @N)ÚabsÚmathÚexp)ÚxÚzÚtÚr© r   úY/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/nltk/translate/gale_church.pyÚerfcc   sL    ÿÿÿþÿþÿþÿþÿþÿþþÿr   c                 C   s   ddt | t d¡ ƒ  S )u>   Return the area under the normal distribution from M{-âˆž..x}.r   r   é   )r   r   Úsqrt©r	   r   r   r   Únorm_cdf@   s    r   c                 C   s4   zt  dt| ƒ ¡W S  ty.   tdƒ Y S 0 d S )Nr   ú-inf)r   Úlogr   Ú
ValueErrorÚfloatr   r   r   r   Ú
norm_logsfD   s    r   r   c                   @   s&   e Zd ZdddddddœZdZdZdS )	ÚLanguageIndependentgõÛ×�sF„?g{®Gázì?gbX9´È¶?gºI+‡†?))r   r   )r   r   )r   r   )r   r   )r   r   )r   r   r   g333333@N)Ú__name__Ú
__module__Ú__qualname__ÚPRIORSÚAVERAGE_CHARACTERSÚVARIANCE_CHARACTERSr   r   r   r   r   N   s   ú	r   c           	      C   sØ   g }t |ƒt |ƒf}|dkrÊtdd„ |D ƒƒrÊz| | \}}W n. tyl   |d d |d d f}Y qY n0 t|ƒD ]8}t|ƒD ]*}| |d | d |d | d f¡ q‚qv|d | |d | f}q|ddd… S )aà  
    Traverse the alignment cost from the tracebacks and retrieves
    appropriate sentence pairs.

    :param backlinks: A dictionary where the key is the alignment points and value is the cost (referencing the LanguageIndependent.PRIORS)
    :type backlinks: dict
    :param source_sents_lens: A list of target sentences' lengths
    :type source_sents_lens: list(int)
    :param target_sents_lens: A list of target sentences' lengths
    :type target_sents_lens: list(int)
    )r   r   c                 s   s   | ]}|d kV  qdS )r   Nr   )Ú.0Úpr   r   r   Ú	<genexpr>n   ó    ztrace.<locals>.<genexpr>r   r   Néÿÿÿÿ)ÚlenÚallÚ	TypeErrorÚrangeÚappend)	Ú	backlinksÚsource_sents_lensÚtarget_sents_lensÚlinksÚpositionÚsr   ÚiÚjr   r   r   Útrace`   s    
*r2   c           
         s²   t ‡ ‡fdd„t|d ƒD ƒƒ}t ‡‡fdd„t|d ƒD ƒƒ}z4|||j  d }||j | t ||j ¡ }	W n tyŽ   tdƒ Y S 0 tt	t
|	ƒƒ t |j| ¡  S )aP  Returns the log probability of the two sentences C{source_sents[i]}, C{target_sents[j]}
    being aligned with a specific C{alignment}.

    @param i: The offset of the source sentence.
    @param j: The offset of the target sentence.
    @param source_sents: The list of source sentence lengths.
    @param target_sents: The list of target sentence lengths.
    @param alignment: The alignment type, a tuple of two integers.
    @param params: The sentence alignment parameters.

    @returns: The log probability of a specific alignment between the two sentences, given the parameters.
    c                 3   s   | ]}ˆˆ | d   V  qdS ©r   Nr   ©r    Úoffset)r0   Úsource_sentsr   r   r"   ‰   r#   z!align_log_prob.<locals>.<genexpr>r   c                 3   s   | ]}ˆˆ | d   V  qdS r3   r   r4   )r1   Útarget_sentsr   r   r"   Š   r#   r   r   r   )Úsumr(   r   r   r   r   ÚZeroDivisionErrorr   ÚLOG2r   r   r   r   )
r0   r1   r6   r7   Ú	alignmentÚparamsZl_sZl_tÚmÚdeltar   )r0   r1   r6   r7   r   Úalign_log_prob|   s      ÿ
r?   c                 C   s  t |j ¡ ƒ}g g}i }tt| ƒd ƒD ]Ø}tt|ƒd ƒD ]¢}tdƒ}d}	|D ]b}
d|
d  }||
d  }|t|ƒ k sP|dk r„qP|| | t||| ||
|ƒ }||k rP|}|
}	qP|tdƒkrÄd}|	|||f< |d  |¡ q<t|ƒdkrö| d¡ | g ¡ q(t	|| |ƒS )a¡  Return the sentence alignment of two text blocks (usually paragraphs).

        >>> align_blocks([5,5,5], [7,7,7])
        [(0, 0), (1, 1), (2, 2)]
        >>> align_blocks([10,5,5], [12,20])
        [(0, 0), (1, 1), (2, 1)]
        >>> align_blocks([12,20], [10,5,5])
        [(0, 0), (1, 1), (1, 2)]
        >>> align_blocks([10,2,10,10,2,10], [12,3,20,3,12])
        [(0, 0), (1, 1), (2, 2), (3, 2), (4, 3), (5, 4)]

    @param source_sents_lens: The list of source sentence lengths.
    @param target_sents_lens: The list of target sentence lengths.
    @param params: the sentence alignment parameters.
    @return: The sentence alignments, a list of index pairs.
    r   ÚinfNr$   r   r   )
Úlistr   Úkeysr(   r%   r   r?   r)   Úpopr2   )r+   r,   r<   Zalignment_typesÚDr*   r0   r1   Zmin_distZ	min_alignÚaZprev_iZprev_jr!   r   r   r   Úalign_blocks˜   s4    ÿ
rF   c                    s0   t | ƒt |ƒkrtdƒ‚‡ fdd„t| |ƒD ƒS )aû  Creates the sentence alignment of two texts.

    Texts can consist of several blocks. Block boundaries cannot be crossed by sentence
    alignment links.

    Each block consists of a list that contains the lengths (in characters) of the sentences
    in this block.

    @param source_blocks: The list of blocks in the source text.
    @param target_blocks: The list of blocks in the target text.
    @param params: the sentence alignment parameters.

    @returns: A list of sentence alignment lists
    z>Source and target texts do not have the same number of blocks.c                    s   g | ]\}}t ||ˆ ƒ‘qS r   )rF   )r    Zsource_blockZtarget_block©r<   r   r   Ú
<listcomp>â   s   ÿzalign_texts.<locals>.<listcomp>)r%   r   Úzip)Zsource_blocksZtarget_blocksr<   r   rG   r   Úalign_textsÎ   s    ÿ
þrJ   c                 #   s"   ‡ ‡fdd„}|ˆ   ¡ ƒV  qdS )záSplits an iterator C{it} at values of C{split_value}.

    Each instance of C{split_value} is swallowed. The iterator produces
    subiterators which need to be consumed fully before the next subiterator
    can be used.
    c                 3   s    | }|ˆkr|V  ˆ   ¡ }qd S ©N©Únext)ÚfirstÚv©ÚitÚsplit_valuer   r   Ú_chunk_iteratoró   s    z!split_at.<locals>._chunk_iteratorNrL   )rQ   rR   rS   r   rP   r   Úsplit_atë   s    rT   c                    s   ‡ fdd„t | |ƒD ƒS )z¶Parses a stream of tokens and splits it into sentences (using C{soft_delimiter} tokens)
    and blocks (using C{hard_delimiter} tokens) for use with the L{align_texts} function.
    c                    s    g | ]}d d„ t |ˆ ƒD ƒ‘qS )c                 S   s   g | ]}t d d„ |D ƒƒ‘qS )c                 s   s   | ]}t |ƒV  qd S rK   )r%   )r    Útokenr   r   r   r"     r#   z;parse_token_stream.<locals>.<listcomp>.<listcomp>.<genexpr>)r8   )r    Zsentence_itr   r   r   rH     s   ÿz1parse_token_stream.<locals>.<listcomp>.<listcomp>©rT   )r    Zblock_it©Úsoft_delimiterr   r   rH     s   üþz&parse_token_stream.<locals>.<listcomp>rV   )ÚstreamrX   Zhard_delimiterr   rW   r   Úparse_token_streamý   s    
ûrZ   )Ú__doc__r   r   r   r   Zscipy.statsÚImportErrorr   r   r   r:   r   r2   r?   rF   rJ   rT   rZ   r   r   r   r   Ú<module>	   s    	'
6