a
    ¡˜ëj>  ã                   @   sx   d dl mZmZ d dlmZmZ d dlmZ ee eeeef  dœdd„Z	dd„ Z
d	d
„ Zdd„ Zdd„ Zdd„ ZdS )é    )ÚListÚTuple)Úmock_sent_tokenizeÚmock_word_tokenize)Útokenize)ÚtokensÚreturnc                 C   s<   g }| D ].}|  ¡ dkr(| |df¡ q| |df¡ q|S )NÚaskZVBÚ )ÚlowerÚappend)r   Zpos_tagsÚtoken© r   úb/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/test_unstructured/nlp/test_tokenize.pyÚmock_pos_tag   s    r   c                  C   s   t  d¡} | g d¢ksJ ‚d S )NzITEM 2A. PROPERTIES))ÚITEMZNNP)Z2AZCD)Ú.r   )Z
PROPERTIESZNN)r   Úpos_tag)Zparts_of_speechr   r   r   Útest_pos_tag   s    
r   c                 C   s\   |   tdt¡ |   tdt¡ tj ¡  tj ¡ jdks:J ‚t d¡ tj ¡ jdksXJ ‚d S ©NÚ_word_tokenizeÚ_pos_tagr   ú!Greetings! I am from outer space.é   )Úsetattrr   r   r   Úword_tokenizeÚcache_clearÚ
cache_infoÚcurrsize©Zmonkeypatchr   r   r   Útest_word_tokenize_caches   s    

r    c                 C   sj   |   tdt¡ |   tdt¡ |   tdt¡ tj ¡  tj ¡ jdksHJ ‚t d¡ tj ¡ jdksfJ ‚d S )NZ_sent_tokenizer   r   r   r   r   )	r   r   r   r   r   Úsent_tokenizer   r   r   r   r   r   r   Útest_sent_tokenize_caches$   s    

r"   c                 C   s\   |   tdt¡ |   tdt¡ tj ¡  tj ¡ jdks:J ‚t d¡ tj ¡ jdksXJ ‚d S r   )r   r   r   r   r   r   r   r   r   r   r   r   Útest_pos_tag_caches.   s    

r#   c                  C   s&   d} t  | ¡ t  | ¡ t  | ¡ d S )Nz$I am a big brown bear. What are you?)r   r!   r   r   )Zsentencer   r   r   Útest_tokenizers_functions_run7   s    

r$   N)Útypingr   r   Ztest_unstructured.nlp.mock_nltkr   r   Zunstructured.nlpr   Ústrr   r   r    r"   r#   r$   r   r   r   r   Ú<module>   s    

	
	