a
    •˜ëj#B  ã                   @   s’   U d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	 erRd dl
mZmZmZ e  e¡Zh d£Zee ed< G dd„ deƒZG d	d
„ d
ƒZdS )é    N)Úplatform)ÚTYPE_CHECKINGÚAnyÚOptionalÚ	TypedDictÚUnion)ÚBrowserÚ
CDPSessionÚPage>   ÚheadÚiframeÚmetaÚsvgÚstyleÚbrÚtitleÚscriptÚpathz::markerÚbodyÚhtmlÚblack_listed_elementsc                   @   sn   e Zd ZU dZeed< eed< ee ed< ee ed< ee ed< e	ed< eed< eed	< eed
< eed< dS )ÚElementInViewPortzIA typed dictionary containing information about elements in the viewport.Ú
node_indexÚbackend_node_idÚ	node_nameÚ
node_valueÚ	node_metaÚis_clickableÚorigin_xÚorigin_yÚcenter_xÚcenter_yN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__Úintr   ÚlistÚbool© r+   r+   ú^/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/langchain/chains/natbot/crawler.pyr   !   s   
r   c                   @   s�   e Zd ZdZddœdd„Zeddœdd„Zedd	œd
d„Zeee	f ddœdd„Z
eee	f eddœdd„Zddœdd„Zee dœdd„ZdS )ÚCrawlera   A crawler for web pages.

    **Security Note**: This is an implementation of a crawler that uses a browser via
        Playwright.

        This crawler can be used to load arbitrary webpages INCLUDING content
        from the local file system.

        Control access to who can submit crawling requests and what network access
        the crawler has.

        Make sure to scope permissions to the minimal permissions necessary for
        the application.

        See https://python.langchain.com/docs/security for more information.
    N)Úreturnc              
   C   s„   zddl m} W n2 tyB } zd}t|ƒ|‚W Y d}~n
d}~0 0 |ƒ  ¡ jjdd�| _| j ¡ | _| j 	ddd	œ¡ |  |  dS )
zInitialize the crawler.r   )Úsync_playwrightz\Could not import playwright python package. Please install it with `pip install playwright`.NF)Zheadlessi   i8  )ÚwidthÚheight)
Úplaywright.sync_apir/   ÚImportErrorÚstartZchromiumZlaunchZbrowserZnew_pageÚpageZset_viewport_size)Úselfr/   ÚeÚmsgr+   r+   r,   Ú__init__B   s    ÿ ÿzCrawler.__init__)Úurlr.   c                 C   s:   | j jd|v r|nd| d� | j j | j ¡| _i | _dS )z°Navigate to the given URL.

        Args:
            url: The URL to navigate to. If it does not contain a scheme, it will be
                prefixed with "http://".
        z://zhttp://)r:   N)r5   ÚgotoÚcontextZnew_cdp_sessionÚclientÚpage_element_buffer)r6   r:   r+   r+   r,   Ú
go_to_pageT   s    zCrawler.go_to_page)Ú	directionr.   c                 C   s.   |dkr| j  d¡ n|dkr*| j  d¡ dS )z‰Scroll the page in the given direction.

        Args:
            direction: The direction to scroll in, either "up" or "down".
        Zupz…(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop - window.innerHeight;Zdownz…(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;N)r5   Úevaluate)r6   r@   r+   r+   r,   Úscroll_   s    ÿÿzCrawler.scroll)Úid_r.   c                 C   sR   d}| j  |¡ | j t|ƒ¡}|rF|d }|d }| j j ||¡ ntdƒ dS )zrClick on an element with the given id.

        Args:
            id_: The id of the element to click on.
        z‡
		links = document.getElementsByTagName("a");
		for (var i = 0; i < links.length; i++) {
			links[i].removeAttribute("target");
		}
		r    r!   zCould not find elementN)r5   rA   r>   Úgetr(   ZmouseÚclickÚprint)r6   rC   ZjsÚelementÚxÚyr+   r+   r,   rE   r   s    zCrawler.click)rC   Útextr.   c                 C   s   |   |¡ | jj |¡ dS )z®Type text into an element with the given id.

        Args:
            id_: The id of the element to type into.
            text: The text to type into the element.
        N)rE   r5   ÚkeyboardÚtype)r6   rC   rJ   r+   r+   r,   rL   Š   s    
zCrawler.typec                 C   s   | j j d¡ dS )zPress the Enter key.ZEnterN)r5   rK   Zpress)r6   r+   r+   r,   Úenter”   s    zCrawler.enterc           K         sx  | j }| j}t ¡ }g }| d¡}tdkr6|dkr6d}| d¡}| d¡}| d¡}| d¡}	|| }
||	 }d}d}| d	d	d
|d›d|d›d�dœ¡ | j dg dddœ¡}|d ‰|d d	 }|d }|d }|d }|d }|d ‰|d ‰t|d d ƒ}|d }|d }|d }|d }|d }|d  }d	}i }g }d!d"i}d!d"i}t	t
 t	t t
d#œd$d%„} tttf tt
 tt
t
f d&œ‡fd'd(„}!tt
ttt	t f f t
tt	t
 tttt	t f d)œ‡ ‡‡‡fd*d+„‰ tˆƒD �]Þ\}"}#ˆ|" }$ˆ|#  ¡ }%ˆ |d,|"|%|$ƒ\}&}'ˆ |d-|"|%|$ƒ\}(})z| |"¡}W n t�yP   Y �qâY n0 |%tv �r`�qâ|| \}*}+},}-|*| }*|+| }+|,| },|-| }-|*}.|+}/|*|, }0|+|- }1|.|
k �oÌ|0|k�oÌ|/|k �oÌ|1|k}2|2�sØ�qâg }3|!||" g d.¢ƒ}4|&�pö|(}5|5�sd/n|&�rt
|'ƒnt
|)ƒ}6|5�s"d/n| t
|6ƒg ¡}7|%d0k�rt|5�rt|7�rtˆ||"  }8|8d1v �rb�qâ|7 d2|8d3œ¡ nr|%d4k�rŽ|4 d2¡d5k�s˜|%d-k�r¨d-}%|4 d2d/¡ |4D ]8}9|5�rÔ|7�rÔ|7 d6|9|4|9 d7œ¡ n|3 |4|9 ¡ �q¬d/}:||" d	k�rˆ||"  }:|:d8k�r`�qânL|%d4k�r`|"|v �r`|:d/u �r`| |"¡};||; }<|;d	k�r`|<d	k�r`ˆ|< }:|5�rt|%d9v�rt�qâ| t
|"ƒ||" |%|:|3|"|v t|*ƒt|+ƒt|*|,d  ƒt|+|-d  ƒd:œ
¡ �qâg }=d	}>|D �]†}?|? d;¡}@|? d<¡}%|? d=¡}:|? d>¡}A|? d?¡}B|:�r|:› d@�ndA}CdA}D|@|v �rŽ||@ D ]Z}E|E d2¡}F|E d¡}G|Fd6k�r||B�r||E dB¡}H|B |H› dC|G› dD�¡ n|C|G› d@�7 }C�q2|B�r¨d@ |B¡}Id@|I› �}D|CdAk�r¼|C ¡ › }C| |%|Aƒ}J|Jd-k�sÚ|DdAk�rö|JdEv�rö|C ¡ dAk�rö�qÐ|?||>< |CdAk�r2|= dF|J› dG|>› |D› dH|C› dI|J› dH�
¡ n|= dF|J› dG|>› |D› dJ�¡ |>d7 }>�qÐtdKt ¡ | d›dL�ƒ |=S )MzfCrawl the current page.

        Returns:
            A list of the elements in the viewport.
        zwindow.devicePixelRatioÚdarwiné   é   zwindow.pageYOffsetzwindow.pageXOffsetzwindow.screen.widthzwindow.screen.heightr   z[scrollbar z0.2fú-z%])rH   rI   rJ   zDOMSnapshot.captureSnapshotT)ZcomputedStylesZincludeDOMRectsZincludePaintOrderÚstringsZ	documentsÚnodesZbackendNodeIdÚ
attributesZ	nodeValueZparentIndexÚnodeNameZisClickableÚindexZ
inputValueÚvalueÚlayoutÚ	nodeIndexÚboundsz-1©FN)r   Úhas_click_handlerr.   c                 S   s8   | dkrdS | dkrdS | dkr$dS | dks0|r4dS dS )NÚaÚlinkÚinputÚimgÚbuttonrJ   r+   )r   r\   r+   r+   r,   Úconvert_nameÝ   s    ÿÿz#Crawler.crawl.<locals>.convert_name)rT   Úkeysr.   c                    sd   i }t t| ƒfd Ž D ]H\}}|dk r(qˆ | }ˆ | }||v r|||< | |¡ |s|  S q|S )NrP   r   )ÚzipÚiterÚremove)rT   rc   ÚvaluesZ	key_indexZvalue_indexÚkeyrW   )rR   r+   r,   Úfind_attributesí   s    

z&Crawler.crawl.<locals>.find_attributes)Ú	hash_treeÚtagÚnode_idr   Ú	parent_idr.   c                    sx   t |ƒ}|| vr8ˆˆ|   ¡ }ˆ| }ˆ | ||||ƒ | | \}}	||krVd|f}
n|rdd|	f}
nd}
|
| t |ƒ< |
S )NTr[   )r&   Úlower)rj   rk   rl   r   rm   Zparent_id_strÚparent_nameZgrand_parent_idZis_parent_desc_anchorÚ	anchor_idrW   ©Úadd_to_hash_treeZ
node_namesÚparentrR   r+   r,   rr     s     
ÿ
ÿ
z'Crawler.crawl.<locals>.add_to_hash_treer]   ra   )rL   Úplaceholderz
aria-labelr   ZaltNz#text>   u   â€¢ú|rL   )rL   rW   r_   ZsubmitÚ	attribute)rL   rh   rW   ru   >   ra   r]   )
r   r   r   r   r   r   r   r   r    r!   r   r   r   r   r   ú Ú rh   z="ú">   r`   r^   r_   Útextareaú<z id=ú>z</z/>zParsing time: z seconds)r5   r>   ÚtimerA   r   Úappendr=   ÚsendÚsetr   r&   r*   Údictr(   r   r)   ÚtupleÚ	enumeratern   rV   Ú
ValueErrorr   Ú
setdefaultrD   ÚpopÚjoinÚstriprF   )Kr6   r5   r>   r4   Zpage_state_as_textZdevice_pixel_ratioZwin_upper_boundZwin_left_boundZ	win_widthZ
win_heightZwin_right_boundZwin_lower_boundZpercentage_progress_startZpercentage_progress_endÚtreeÚdocumentrS   r   rT   r   r   Zinput_valueZinput_value_indexZinput_value_valuesrX   Zlayout_node_indexrZ   ÚcursorZchild_nodesZelements_in_view_portZanchor_ancestryZbutton_ancestryrb   ri   rV   Znode_name_indexZnode_parentr   Zis_ancestor_of_anchorrp   Zis_ancestor_of_buttonZ	button_idrH   rI   r0   r1   Zelem_left_boundZelem_top_boundZelem_right_boundZelem_lower_boundZpartially_is_in_viewportZ	meta_dataZelement_attributesZancestor_exceptionZancestor_node_keyZancestor_noderJ   rh   Zelement_node_valueZnode_input_text_indexZ
text_indexZelements_of_interestZ
id_counterrG   r   Znode_is_clickableZnode_meta_dataZ
inner_textr   ÚchildZ
entry_typeZentry_valueZ	entry_keyZmeta_stringZconverted_node_namer+   rq   r,   Úcrawl˜   s¤   





ÿýÿ	
þý
þú&
ÿ
ÿ

ÿþüÿ
ÿÿû	ÿý
ÿÿþÿýÿÿÿþý
öÿ














ÿÿþ
ý
ÿÿÿzCrawler.crawl)r"   r#   r$   r%   r9   r&   r?   rB   r   r(   rE   rL   rM   r)   r�   r+   r+   r+   r,   r-   0   s   
r-   )Úloggingr}   Úsysr   Útypingr   r   r   r   r   r2   r   r	   r
   Ú	getLoggerr"   Úloggerr   r€   r&   r'   r   r-   r+   r+   r+   r,   Ú<module>   s   

