a
    ¡˜ëjØ  ã                   @   sŒ   d dl Z d dlmZmZ d dlZd dlmZmZ d dlm	Z	 d dl
mZ eedœdd„Zed	œd
d„Zdeee eedœdd„Zdd„ ZdS )é    N)ÚListÚOptional)ÚMarianMTModelÚMarianTokenizer)Úsent_tokenize)Úchunk_by_attention_window©Úsource_langÚtarget_langc                 C   s   d| › d|› �S )zjConstructs the name of the MarianMT machine translation model based on the
    source and target language.zHelsinki-NLP/opus-mt-ú-© r   r   r   ú^/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/cleaners/translate.pyÚ_get_opus_mt_model_name   s    r   ©Zlanguage_codec                 C   s*   t | tƒrt| ƒdkr&td| › d�ƒ‚d S )Né   zInvalid language code: z,. Language codes must be two letter strings.)Ú
isinstanceÚstrÚlenÚ
ValueErrorr   r   r   r   Ú_validate_language_code   s    
ÿr   Úen)Útextr	   r
   Úreturnc           
      C   sÜ   |   ¡ dkr| S |dur|nt | ¡}| d¡r4d}t|ƒ t|ƒ ||krP| S t||ƒ}td|› �ƒ zt |¡}t	 |¡}W n" t
y¢   td|› d�ƒ‚Y n0 t| |td�}g }|D ]}	| t| ||ƒ¡ qºd |¡S )	aô  Translates the foreign language text. If the source language is not specified, the
    function will attempt to detect it using langdetect.

    Parameters
    ----------
    text: str
        The text to translate
    target_lang: str
        The two letter language code for the target langague. Defaults to "en".
    source_lang: Optional[str]
        The two letter language code for the language of the input text. If source_lang is
        not provided, the function will try to detect it.
    Ú NÚzhzUsing model: z2Transformers could not find the translation model z>. The requested source/target language combo is not supported.)Zsplit_functionú )ÚstripÚ
langdetectÚdetectÚ
startswithr   r   Úprintr   Zfrom_pretrainedr   ÚOSErrorr   r   r   ÚappendÚ_translate_textÚjoin)
r   r	   r
   Z_source_langZ
model_nameÚ	tokenizerÚmodelÚchunksZtranslated_chunksÚchunkr   r   r   Útranslate_text   s.    



ÿ
r)   c              
      sf   t  ¡ �8 t  d¡ |jf i ˆ | gdddd�¤Ž}W d  ƒ n1 sF0    Y  ‡ fdd„|D ƒd S )	z8Translates text using the specified model and tokenizer.ÚignoreÚptT)Zreturn_tensorsÚpaddingZ
truncationNc                    s   g | ]}ˆ j |d dd�‘qS )i   T)Zmax_new_tokensZskip_special_tokens)Údecode)Ú.0Út©r%   r   r   Ú
<listcomp>U   ó    z#_translate_text.<locals>.<listcomp>r   )ÚwarningsÚcatch_warningsÚsimplefilterÚgenerate)r   r&   r%   Z
translatedr   r0   r   r#   K   s    

ÿ$ÿr#   )Nr   )r3   Útypingr   r   r   Ztransformersr   r   Zunstructured.nlp.tokenizer   Z unstructured.staging.huggingfacer   r   r   r   r)   r#   r   r   r   r   Ú<module>   s   3