a
    ¡˜ëj#  ã                   @  s  d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZmZmZmZmZmZmZ e e¡j ¡ Zej eddd¡Zd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z ddœdd„Z!ddœdd„Z"dd„ Z#ddœdd„Z$ej% &dg d ¢¡d!d!dœd"d#„ƒZ'ddœd$d%„Z(ej% &dg d&¢¡d!d!dœd'd(„ƒZ)d)d*„ Z*d+d,„ Z+d-d.„ Z,d/d0„ Z-ddœd1d2„Z.d3d4„ Z/ddœd5d6„Z0ej% &d7d8d9g¡d!d!d7œd:d;„ƒZ1ej% &d<d=gd=fd>d?d@dAgdBfdCgdBfdDg¡d!d!d<œdEdF„ƒZ2dGdH„ Z3ej% &dIdJgdBdJgfdKgdLdJgfdJgdAdJgfdJgdCdJgfdJgdBgdJgfdJgdAgdJgfdMgdNdJdBgfg¡dOdPdOddQœdRdS„ƒZ4ej% &dIg dBdBgfdTgdCdBgfdTgdBdBgfdTgdAdBgfg¡dOd!dOddQœdUdV„ƒZ5ej% &dWg dfdTgdfg¡dOdXdWœdYdZ„ƒZ6d[d\„ Z7ej% &dWg d]gfdTgd^fg¡dOd_dWœd`da„ƒZ8dS )bz=Unit-test suite for the `unstructured.partition.lang` module.é    )ÚannotationsN)ÚLogCaptureFixture)ÚNarrativeTextÚ	PageBreak)Ú_clean_ocr_languages_argÚ/_convert_language_code_to_pytesseract_lang_codeÚapply_lang_metadataÚcheck_language_argsÚdetect_languagesÚprepare_languages_for_tesseractÚtesseract_to_paddle_languagez..zexample-docsc                  C  s   dg} t | ƒdksJ ‚d S ©NÚenÚeng©r   ©Ú	languages© r   úk/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/partition/common/test_lang.pyÚ6test_prepare_languages_for_tesseract_with_one_language   s    r   c                  C  s   ddg} t | ƒdksJ ‚d S r   r   r   r   r   r   Ú>test_prepare_languages_for_tesseract_with_duplicated_languages$   s    r   c                  C  s0   dg} t | ƒdksJ ‚dg} t | ƒdks,J ‚d S )NZosdÚequr   r   r   r   r   Ú1test_prepare_languages_for_tesseract_special_case)   s    r   c                  C  s   ddg} t | ƒdksJ ‚d S )NÚkbdÚeszspa+spa_oldr   r   r   r   r   Ú9test_prepare_languages_for_tesseract_removes_empty_inputs1   s    r   c                  C  s   dg} t | ƒdksJ ‚d S )NÚchiú)chi_sim+chi_sim_vert+chi_tra+chi_tra_vertr   r   r   r   r   Ú6test_prepare_languages_for_tesseract_includes_variants6   s    r   c                  C  s   g d¢} t | ƒdksJ ‚d S )N)ÚjaZafrr   r   zjpn+jpn_vert+afr+eng+equr   r   r   r   r   Ú<test_prepare_languages_for_tesseract_with_multiple_languages;   s    r    r   )Úcaplogc                 C  s*   ddg}t |ƒdksJ ‚d| jv s&J ‚d S )NZzzzr   r   z"not a valid standard language code©r   Útext©r!   r   r   r   r   Ú?test_prepare_languages_for_tesseract_warns_nonstandard_language@   s    r%   c                 C  s*   ddg}t |ƒdksJ ‚d| jv s&J ‚d S )Nr   r   z%not a language supported by Tesseractr"   r$   r   r   r   ÚAtest_prepare_languages_for_tesseract_warns_non_tesseract_languageF   s    r&   c                  C  s>   t jtdd�� d } t| ƒ W d   ƒ n1 s00    Y  d S )Nz`languages` can not be `None`)Úmatch)ÚpytestÚraisesÚ
ValueErrorr   r   r   r   r   Ú3test_prepare_languages_for_tesseract_None_languagesL   s    r+   c                 C  s(   dg}t |ƒdksJ ‚d| jv s$J ‚d S )NÚ r   z>Failed to find any valid standard language code from languagesr"   r$   r   r   r   Ú7test_prepare_languages_for_tesseract_no_valid_languagesR   s    r-   ©Útesseract_langÚexpected_lang))r   r   )Zchi_simÚch)Zchi_traZchinese_cht)ÚdeuÚgerman)ZjpnÚjapan)ÚkorÚkoreanÚstrc                 C  s   |t | ƒksJ ‚d S ©N©r   r.   r   r   r   Ú-test_tesseract_to_paddle_language_valid_codesX   s    r:   c                 C  s&   d}t |ƒdksJ ‚d| jv s"J ‚d S )NZunsupported_langr   z?unsupported_lang is not a language code supported by PaddleOCR,)r   r#   )r!   r/   r   r   r   Ú/test_tesseract_to_paddle_language_invalid_codesg   s    r;   ))ZENGr   )ZFraÚfr)ZDEUr3   c                 C  s   |t | ƒksJ ‚d S r8   r9   r.   r   r   r   Ú2test_tesseract_to_paddle_language_case_sensitivitym   s    	r=   c                  C  s   d} t | ƒdgksJ ‚d S )NúThis is a short sentence.r   ©r
   ©r#   r   r   r   Ú"test_detect_languages_english_autoy   s    rA   c                  C  s"   d} dg}t | |ƒdgksJ ‚d S )NúThis is another short sentence.r   r   r?   ©r#   r   r   r   r   Ú&test_detect_languages_english_provided~   s    rD   c                  C  s   d} t | ƒdgksJ ‚d S )Nu   ì•ˆë…•í•˜ì„¸ìš”r5   r?   r@   r   r   r   Ú!test_detect_languages_korean_auto„   s    rE   c                  C  s   d} t | ƒg d¢ksJ ‚d S )NzMy lubimy mleko i chleb.)ZcesZpolZslkr?   r@   r   r   r   Ú-test_detect_languages_gets_multiple_languages‰   s    rF   c                 C  s2   d}g d¢}t ||ƒdgks J ‚d| jv s.J ‚d S )NrB   )r   ÚautoZrusr   z.rest of the inputted languages will be ignored)r
   r#   )r!   r#   r   r   r   r   Ú4test_detect_languages_warns_for_auto_and_other_inputŽ   s    rH   c                  C  sD   t  t¡�& d} t| dd�dgk W d   ƒ n1 s60    Y  d S )Nr>   r   r   )r(   r)   Ú	TypeErrorr
   r@   r   r   r   Ú<test_detect_languages_raises_TypeError_for_invalid_languages•   s    rJ   c                 C  s@   t dƒtdƒg}tt|dgdd�ƒ}ddd„ | jD ƒvs<J ‚d S )	NzSample text.r,   rG   T)Úelementsr   Zdetect_language_per_elementzNo features in text.c                 S  s   g | ]
}|j ‘qS r   )Úmessage)Ú.0Zrecr   r   r   Ú
<listcomp>¤   ó    zItest_apply_lang_metadata_has_no_warning_for_PageBreak.<locals>.<listcomp>)r   r   Úlistr   Úrecords)r!   rK   r   r   r   Ú5test_apply_lang_metadata_has_no_warning_for_PageBreak›   s    ýÿrR   ©Zlang_inr0   )r   r   )r<   Zfrac                 C  s   |t | ƒksJ ‚d S r8   )r   rS   r   r   r   Ú3test_convert_language_code_to_pytesseract_lang_code§   s    rT   ©Zinput_ocr_langsÚexpectedr   )ú"deu"r2   )ú[deu]r2   )z['deu']r2   rX   r2   rW   )údeu+sparY   c                 C  s   t | ƒ|ksJ ‚d S r8   )r   rU   r   r   r   Útest_clean_ocr_languages_arg²   s    rZ   c                  C  s    t ddgd�} | dgksJ ‚d S )Nz Sample text longer than 5 words.ZSpanishrC   Úspar?   r   r   r   r   Ú3test_detect_languages_handles_spelled_out_languagesÂ   s    r\   )r   Úocr_languagesÚexpected_langsr[   ÚspanishÚenglishzspa+deuzeng+deuz	list[str]zlist[str] | str)r   r]   r^   r!   c                 C  s4   t | |d�}|D ]}||v s J ‚d|jv sJ ‚qd S ©N©r   r]   r]   ©r	   r#   ©r   r]   r^   r!   Úreturned_langsÚlangr   r   r   ÚYtest_check_language_args_uses_languages_when_ocr_languages_and_languages_are_both_definedÇ   s    þrg   r,   c                 C  s4   t | |d�}|D ]}||v s J ‚d|jv sJ ‚qd S ra   rc   rd   r   r   r   ÚKtest_check_language_args_uses_ocr_languages_when_languages_is_empty_or_Noneâ   s    rh   rb   ÚNonec                 C  s   t | |d�}|d u sJ ‚d S ©Nrb   ©r	   )r   r]   re   r   r   r   Ú%test_check_language_args_returns_Noneø   s    rl   c                   C  s   t g d¢d d�dgksJ ‚d S )N)r   r[   rG   rb   rG   rk   r   r   r   r   Ú%test_check_language_args_returns_auto  s    rm   rG   zeng+autozstr | list[str]c                 C  s:   t  t¡� t| |d� W d   ƒ n1 s,0    Y  d S rj   )r(   r)   r*   r	   rb   r   r   r   ÚFtest_check_language_args_raises_error_when_ocr_languages_contains_auto  s
    þrn   )9Ú__doc__Ú
__future__r   ÚosÚpathlibr(   Ztest_unstructured.unit_utilsr   Zunstructured.documents.elementsr   r   Z"unstructured.partition.common.langr   r   r   r	   r
   r   r   ÚPathÚ__file__ÚparentÚresolveZ	DIRECTORYÚpathÚjoinZEXAMPLE_DOCS_DIRECTORYr   r   r   r   r   r    r%   r&   r+   r-   ÚmarkZparametrizer:   r;   r=   rA   rD   rE   rF   rH   rJ   rR   rT   rZ   r\   rg   rh   rl   rm   rn   r   r   r   r   Ú<module>   sº   $
þþþþùþùþ
ûþ
þþþþ