a
    ”˜ëjÄ  ã                
   @   s
  d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	m
Z
 e e¡ZdZdZdd	 d
d„ eD ƒ¡ d Zdd	 dd„ eD ƒ¡ d Zde› de› d�Zddœeeeejdf ee dœdd„Zddddddœeeee eeejdf eee eee dœdd„ZdS )z Utilities for working with HTML.é    N)ÚSequence)ÚOptionalÚUnion)ÚurljoinÚurlparse)zjavascript:zmailto:ú#)z.cssz.jsz.icoz.pngz.jpgz.jpegz.gifz.svgz.csvz.bz2z.zipz.epubz(?!ú|c                 C   s   g | ]}t  |¡d  ‘qS )z[\#'\"]©ÚreÚescape©Ú.0Ús© r   úX/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain_core/utils/html.pyÚ
<listcomp>   ó    r   ú)c                 C   s   g | ]}t  |¡‘qS r   r	   r   r   r   r   r      r   z
href=[\"']z((?:z.)*?)[\#'\"]©Úpattern)Úraw_htmlr   Úreturnc                C   s   |pt }ttt || ¡ƒƒS )zÇExtract all links from a raw HTML string.

    Args:
        raw_html: original HTML.
        pattern: Regex to use for extracting links from raw HTML.

    Returns:
        list[str]: all links
    )ÚDEFAULT_LINK_REGEXÚlistÚsetr
   Úfindall)r   r   r   r   r   Úfind_all_links%   s    r   Tr   F)Úbase_urlr   Úprevent_outsideÚexclude_prefixesÚcontinue_on_failure)r   Úurlr   r   r   r   r    r   c                   sH  |dur|n|}t |ƒ}t |ƒ}	t| |d�}
tƒ }|
D ]®}zdt |ƒ}|jdv rT|}n>| d¡rp|	j› d|› �}n"t||jƒ}|jr’|d|j› �7 }| |¡ W q6 t	yâ } z,|rÌt
 d||¡ W Y d}~q6‚ W Y d}~q6d}~0 0 q6g }|D ]T‰ t‡ fdd	„|D ƒƒ�rqî|�r8t ˆ ƒ}|j|jk�r*qîˆ  |¡�s8qî| ˆ ¡ qî|S )
a–  Extract all links from a raw HTML string and convert into absolute paths.

    Args:
        raw_html: original HTML.
        url: the url of the HTML.
        base_url: the base URL to check for outside links against.
        pattern: Regex to use for extracting links from raw HTML.
        prevent_outside: If True, ignore external links which are not children
            of the base URL.
        exclude_prefixes: Exclude any URLs that start with one of these prefixes.
        continue_on_failure: If True, continue if parsing a specific link raises an
            exception. Otherwise, raise the exception.

    Returns:
        list[str]: sub links.
    Nr   >   ÚhttpsÚhttpz//ú:ú?z-Unable to load link %s. Raised exception:

%sc                 3   s   | ]}ˆ   |¡V  qd S )N)Ú
startswith)r   Zexclude_prefix©Úpathr   r   Ú	<genexpr>l   r   z$extract_sub_links.<locals>.<genexpr>)r   r   r   Úschemer&   r   r(   ÚqueryÚaddÚ	ExceptionÚloggerÚwarningÚanyÚnetlocÚappend)r   r!   r   r   r   r   r    Zbase_url_to_useZparsed_base_urlÚ
parsed_urlZ	all_linksZabsolute_pathsÚlinkZparsed_linkZabsolute_pathÚeÚresultsZparsed_pathr   r'   r   Úextract_sub_links5   sF    

ÿr7   )Ú__doc__Úloggingr
   Úcollections.abcr   Útypingr   r   Úurllib.parser   r   Ú	getLoggerÚ__name__r.   ZPREFIXES_TO_IGNOREZSUFFIXES_TO_IGNOREÚjoinZSUFFIXES_TO_IGNORE_REGEXZPREFIXES_TO_IGNORE_REGEXr   ÚstrÚPatternr   r   Úboolr7   r   r   r   r   Ú<module>   sD   
ÿÿÿÿþø÷