a
    <Bìj?  ã                   @  s–  d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7 e	 8e9¡Z:ee;e;f Z<dddœdd„Z=G dd„ de>ƒZ?ddd œd!d"„Z@G d#d$„ d$e>ƒZAdd%dd&œd'd(„ZBdd%dd&œd)d*„ZCd+dd,œd-d.„ZDG d/d0„ d0ƒZEG d1d2„ d2eƒZFd2d2d3œd4d5„ZGeGd6d7d8œd9d:„ƒZHed;d<�G d=d6„ d6ƒƒZIG d>d?„ d?eƒZJdRd@dAdBddCœdDdE„ZKdSddFd6dGœdHdI„ZLd@d%dJdKœdLdM„ZMG dNdO„ dOeƒZNG dPdQ„ dQƒZOdS )TzO
The main purpose of this module is to expose LinkCollector.collect_sources().
é    )ÚannotationsN)ÚIterableÚMutableMappingÚSequence)Ú	dataclass)Ú
HTMLParser)ÚValues)ÚCallableÚ
NamedTupleÚProtocol)Úrequests)ÚResponse)Ú
RetryErrorÚSSLError)ÚNetworkConnectionError)ÚLink)ÚSearchScope)Ú
PipSession)Úraise_for_status)Úis_archive_file©Úredact_auth_from_url)Úurl_to_path)Úvcsé   )ÚCandidatesFromPageÚ
LinkSourceÚbuild_sourceÚstrú
str | None©ÚurlÚreturnc                 C  s6   t jD ]*}|  ¡  |¡r| t|ƒ dv r|  S qdS )zgLook for VCS schemes in the URL.

    Returns the matched VCS scheme, or None if there's no match.
    z+:N)r   ÚschemesÚlowerÚ
startswithÚlen)r!   Úscheme© r(   ú\/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/pip/_internal/index/collector.pyÚ_match_vcs_scheme.   s    

r*   c                      s&   e Zd Zddddœ‡ fdd„Z‡  ZS )Ú_NotAPIContentr   ÚNone)Úcontent_typeÚrequest_descr"   c                   s   t ƒ  ||¡ || _|| _d S ©N)ÚsuperÚ__init__r-   r.   )Úselfr-   r.   ©Ú	__class__r(   r)   r1   :   s    z_NotAPIContent.__init__)Ú__name__Ú
__module__Ú__qualname__r1   Ú__classcell__r(   r(   r3   r)   r+   9   s   r+   r   r,   )Úresponser"   c                 C  s6   | j  dd¡}| ¡ }| d¡r$dS t|| jjƒ‚dS )z°
    Check the Content-Type header to ensure the response contains a Simple
    API Response.

    Raises `_NotAPIContent` if the content type is not a valid content-type.
    úContent-TypeÚUnknown)z	text/htmlz#application/vnd.pypi.simple.v1+htmlú#application/vnd.pypi.simple.v1+jsonN)ÚheadersÚgetr$   r%   r+   ÚrequestÚmethod)r9   r-   Úcontent_type_lr(   r(   r)   Ú_ensure_api_header@   s    ÿrB   c                   @  s   e Zd ZdS )Ú_NotHTTPN)r5   r6   r7   r(   r(   r(   r)   rC   V   s   rC   r   )r!   Úsessionr"   c                 C  sF   t j | ¡\}}}}}|dvr$tƒ ‚|j| dd�}t|ƒ t|ƒ dS )zõ
    Send a HEAD request to the URL, and ensure the response contains a simple
    API Response.

    Raises `_NotHTTP` if the URL is not available for a HEAD request, or
    `_NotAPIContent` if the content type is not a valid content type.
    >   ÚhttpsÚhttpT)Úallow_redirectsN)ÚurllibÚparseÚurlsplitrC   Úheadr   rB   )r!   rD   r'   ÚnetlocÚpathÚqueryÚfragmentÚrespr(   r(   r)   Ú_ensure_api_responseZ   s    rQ   c                 C  sx   t t| ƒjƒrt| |d� t dt| ƒ¡ |j| d g d¢¡ddœd�}t	|ƒ t
|ƒ t dt| ƒ|j d	d
¡¡ |S )aY  Access an Simple API response with GET, and return the response.

    This consists of three parts:

    1. If the URL looks suspiciously like an archive, send a HEAD first to
       check the Content-Type is HTML or Simple API, to avoid downloading a
       large file. Raise `_NotHTTP` if the content type cannot be determined, or
       `_NotAPIContent` if it is not HTML or a Simple API.
    2. Actually perform the request. Raise HTTP exceptions on network failures.
    3. Check the Content-Type header to make sure we got a Simple API response,
       and raise `_NotAPIContent` otherwise.
    ©rD   zGetting page %sz, )r<   z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z	max-age=0)ÚAcceptzCache-Control)r=   zFetched page %s as %sr:   r;   )r   r   ÚfilenamerQ   ÚloggerÚdebugr   r>   Újoinr   rB   r=   )r!   rD   rP   r(   r(   r)   Ú_get_simple_responsel   s&    ÿëþýrX   ÚResponseHeaders)r=   r"   c                 C  s<   | r8d| v r8t j ¡ }| d |d< | d¡}|r8t|ƒS dS )z=Determine if we have any encoding information in our headers.r:   zcontent-typeÚcharsetN)ÚemailÚmessageÚMessageÚ	get_paramr   )r=   ÚmrZ   r(   r(   r)   Ú_get_encoding_from_headers«   s    

r`   c                   @  s:   e Zd Zdddœdd„Zdddœd	d
„Zddœdd„ZdS )ÚCacheablePageContentÚIndexContentr,   ©Úpager"   c                 C  s   |j s
J ‚|| _d S r/   )Úcache_link_parsingrd   ©r2   rd   r(   r(   r)   r1   ·   s    
zCacheablePageContent.__init__ÚobjectÚbool)Úotherr"   c                 C  s   t |t| ƒƒo| jj|jjkS r/   )Ú
isinstanceÚtyperd   r!   )r2   ri   r(   r(   r)   Ú__eq__»   s    zCacheablePageContent.__eq__Úint©r"   c                 C  s   t | jjƒS r/   )Úhashrd   r!   ©r2   r(   r(   r)   Ú__hash__¾   s    zCacheablePageContent.__hash__N)r5   r6   r7   r1   rl   rq   r(   r(   r(   r)   ra   ¶   s   ra   c                   @  s   e Zd Zdddœdd„ZdS )Ú
ParseLinksrb   úIterable[Link]rc   c                 C  s   d S r/   r(   rf   r(   r(   r)   Ú__call__Ã   ó    zParseLinks.__call__N)r5   r6   r7   rt   r(   r(   r(   r)   rr   Â   s   rr   )Úfnr"   c                   s>   t jdddœ‡ fdd„ƒ‰t  ˆ ¡dddœ‡ ‡fdd	„ƒ}|S )
zÚ
    Given a function that parses an Iterable[Link] from an IndexContent, cache the
    function's result (keyed by CacheablePageContent), unless the IndexContent
    `page` has `page.cache_link_parsing == False`.
    ra   z
list[Link])Úcacheable_pager"   c                   s   t ˆ | jƒƒS r/   )Úlistrd   )rw   )rv   r(   r)   ÚwrapperÍ   s    z*with_cached_index_content.<locals>.wrapperrb   rc   c                   s   | j rˆt| ƒƒS tˆ | ƒƒS r/   )re   ra   rx   )rd   ©rv   ry   r(   r)   Úwrapper_wrapperÑ   s    z2with_cached_index_content.<locals>.wrapper_wrapper)Ú	functoolsÚcacheÚwraps)rv   r{   r(   rz   r)   Úwith_cached_index_contentÆ   s
    r   rb   rs   rc   c           
      c  sº   | j  ¡ }| d¡rTt | j¡}| dg ¡D ]"}t || j	¡}|du rHq,|V  q,dS t
| j	ƒ}| jpfd}| | j |¡¡ | j	}|jpˆ|}|jD ]$}	tj|	||d�}|du r®q�|V  q�dS )z\
    Parse a Simple API's Index Content, and yield its anchor elements as Link objects.
    r<   ÚfilesNzutf-8)Úpage_urlÚbase_url)r-   r$   r%   ÚjsonÚloadsÚcontentr>   r   Ú	from_jsonr!   ÚHTMLLinkParserÚencodingÚfeedÚdecoder‚   ÚanchorsZfrom_element)
rd   rA   ÚdataÚfileÚlinkÚparserrˆ   r!   r‚   Úanchorr(   r(   r)   Úparse_linksÚ   s&    





r‘   T)Úfrozenc                   @  sL   e Zd ZU dZded< ded< ded< ded< d	Zd
ed< ddœdd„ZdS )rb   aŒ  Represents one response (or page), along with its URL.

    :param encoding: the encoding to decode the given content.
    :param url: the URL from which the HTML was downloaded.
    :param cache_link_parsing: whether links parsed from this page's url
                               should be cached. PyPI index urls should
                               have this set to False, for example.
    Úbytesr…   r   r-   r   rˆ   r!   Trh   re   rn   c                 C  s
   t | jƒS r/   )r   r!   rp   r(   r(   r)   Ú__str__  s    zIndexContent.__str__N)r5   r6   r7   Ú__doc__Ú__annotations__re   r”   r(   r(   r(   r)   rb   ÷   s   
	c                      sJ   e Zd ZdZdddœ‡ fdd„Zddddœd	d
„Zdddœdd„Z‡  ZS )r‡   zf
    HTMLParser that keeps the first base HREF and a list of all anchor
    elements' attributes.
    r   r,   r    c                   s$   t ƒ jdd� || _d | _g | _d S )NT)Úconvert_charrefs)r0   r1   r!   r‚   r‹   )r2   r!   r3   r(   r)   r1     s    zHTMLLinkParser.__init__zlist[tuple[str, str | None]])ÚtagÚattrsr"   c                 C  sH   |dkr,| j d u r,|  |¡}|d urD|| _ n|dkrD| j t|ƒ¡ d S )NÚbaseÚa)r‚   Úget_hrefr‹   ÚappendÚdict)r2   r˜   r™   Úhrefr(   r(   r)   Úhandle_starttag  s    
zHTMLLinkParser.handle_starttagr   )r™   r"   c                 C  s"   |D ]\}}|dkr|  S qd S )NrŸ   r(   )r2   r™   ÚnameÚvaluer(   r(   r)   rœ   !  s    
zHTMLLinkParser.get_href)r5   r6   r7   r•   r1   r    rœ   r8   r(   r(   r3   r)   r‡     s   r‡   r   zstr | ExceptionzCallable[..., None] | None)rŽ   ÚreasonÚmethr"   c                 C  s   |d u rt j}|d| |ƒ d S )Nz%Could not fetch URL %s: %s - skipping)rU   rV   )rŽ   r£   r¤   r(   r(   r)   Ú_handle_get_simple_fail(  s    r¥   rh   )r9   re   r"   c                 C  s&   t | jƒ}t| j| jd || j|d�S )Nr:   )rˆ   r!   re   )r`   r=   rb   r…   r!   )r9   re   rˆ   r(   r(   r)   Ú_make_index_content2  s    
ûr¦   úIndexContent | None)rŽ   rD   r"   c             
   C  sè  | j  dd¡d }t|ƒ}|r0t d|| ¡ d S | d¡rvtj t	|ƒ¡rv| 
d¡s\|d7 }tj |d¡}t d|¡ zt||d	�}W �nL ty¨   t d
| ¡ Y �n< tyà } z"t d| |j|j¡ W Y d }~�nd }~0  t�y } zt| |ƒ W Y d }~nÞd }~0  t�y< } zt| |ƒ W Y d }~n°d }~0  t�y€ } z,d}|t|ƒ7 }t| |tjd� W Y d }~nld }~0  tj�y¶ } zt| d|› �ƒ W Y d }~n6d }~0  tj�yÔ   t| dƒ Y n0 t|| jd�S d S )Nú#r   r   zICannot look at %s URL %s because it does not support lookup as web pages.zfile:ú/z
index.htmlz# file: URL is directory, getting %srR   z`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zºSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )r¤   zconnection error: z	timed out)re   )r!   Úsplitr*   rU   Úwarningr%   ÚosrM   Úisdirr   ÚendswithrH   rI   ÚurljoinrV   rX   rC   r+   r.   r-   r   r¥   r   r   r   Úinfor   ÚConnectionErrorÚTimeoutr¦   re   )rŽ   rD   r!   Ú
vcs_schemerP   Úexcr£   r(   r(   r)   Ú_get_index_content?  sT    ý
ý
ú$$rµ   c                   @  s   e Zd ZU ded< ded< dS )ÚCollectedSourceszSequence[LinkSource | None]Ú
find_linksÚ
index_urlsN)r5   r6   r7   r–   r(   r(   r(   r)   r¶   {  s   
r¶   c                   @  sp   e Zd ZdZddddœdd„Zeddd	d
d dœdd„ƒZeddœdd„ƒZdddœdd„Z	ddddœdd„Z
dS )ÚLinkCollectorzµ
    Responsible for collecting Link objects from all configured locations,
    making network requests as needed.

    The class's main method is its collect_sources() method.
    r   r   r,   )rD   Úsearch_scoper"   c                 C  s   || _ || _d S r/   )rº   rD   )r2   rD   rº   r(   r(   r)   r1   ˆ  s    zLinkCollector.__init__Fr   rh   )rD   ÚoptionsÚsuppress_no_indexr"   c                 C  sd   |j g|j }|jr8|s8t dd dd„ |D ƒ¡¡ g }|jp@g }tj|||jd�}t	||d�}|S )zÆ
        :param session: The Session to use to make requests.
        :param suppress_no_index: Whether to ignore the --no-index option
            when constructing the SearchScope object.
        zIgnoring indexes: %sú,c                 s  s   | ]}t |ƒV  qd S r/   r   )Ú.0r!   r(   r(   r)   Ú	<genexpr>   ru   z'LinkCollector.create.<locals>.<genexpr>)r·   r¸   Úno_index)rD   rº   )
Ú	index_urlÚextra_index_urlsrÀ   rU   rV   rW   r·   r   Úcreater¹   )ÚclsrD   r»   r¼   r¸   r·   rº   Úlink_collectorr(   r(   r)   rÃ   �  s$    
þ
ýþzLinkCollector.createz	list[str]rn   c                 C  s   | j jS r/   )rº   r·   rp   r(   r(   r)   r·   ²  s    zLinkCollector.find_linksr   r§   )Úlocationr"   c                 C  s   t || jd�S )z>
        Fetch an HTML page containing package links.
        rR   )rµ   rD   )r2   rÆ   r(   r(   r)   Úfetch_response¶  s    zLinkCollector.fetch_responser   r   r¶   )Úproject_nameÚcandidates_from_pager"   c                   sª   t  ‡ ‡‡fdd„ˆj ˆ¡D ƒ¡ ¡ }t  ‡ ‡‡fdd„ˆjD ƒ¡ ¡ }t tj	¡r–dd„ t
 ||¡D ƒ}t|ƒ› dˆ› d�g| }t d |¡¡ tt|ƒt|ƒd	�S )
Nc              	   3  s&   | ]}t |ˆ ˆjjd d ˆd�V  qdS )F©rÉ   Zpage_validatorÚ
expand_dirre   rÈ   N©r   rD   Úis_secure_origin©r¾   Úloc©rÉ   rÈ   r2   r(   r)   r¿   Â  s   	øúz0LinkCollector.collect_sources.<locals>.<genexpr>c              	   3  s&   | ]}t |ˆ ˆjjd d ˆd�V  qdS )TrÊ   NrÌ   rÎ   rÐ   r(   r)   r¿   Í  s   	øúc                 S  s*   g | ]"}|d ur|j d urd|j › �‘qS )Nz* )rŽ   )r¾   Úsr(   r(   r)   Ú
<listcomp>Ú  s   þz1LinkCollector.collect_sources.<locals>.<listcomp>z' location(s) to search for versions of ú:Ú
)r·   r¸   )ÚcollectionsÚOrderedDictrº   Úget_index_urls_locationsÚvaluesr·   rU   ÚisEnabledForÚloggingÚDEBUGÚ	itertoolsÚchainr&   rV   rW   r¶   rx   )r2   rÈ   rÉ   Zindex_url_sourcesZfind_links_sourcesÚlinesr(   rÐ   r)   Úcollect_sources¼  s*    	
÷	÷
þ
ÿÿýþzLinkCollector.collect_sourcesN)F)r5   r6   r7   r•   r1   ÚclassmethodrÃ   Úpropertyr·   rÇ   rß   r(   r(   r(   r)   r¹   €  s    ü!r¹   )N)T)Pr•   Ú
__future__r   rÕ   Úemail.messager[   r|   rÜ   rƒ   rÚ   r¬   Úurllib.parserH   Úcollections.abcr   r   r   Zdataclassesr   Úhtml.parserr   Úoptparser   Útypingr	   r
   r   Úpip._vendorr   Zpip._vendor.requestsr   Zpip._vendor.requests.exceptionsr   r   Úpip._internal.exceptionsr   Úpip._internal.models.linkr   Ú!pip._internal.models.search_scoper   Úpip._internal.network.sessionr   Úpip._internal.network.utilsr   Úpip._internal.utils.filetypesr   Úpip._internal.utils.miscr   Úpip._internal.utils.urlsr   Úpip._internal.vcsr   Úsourcesr   r   r   Ú	getLoggerr5   rU   r   rY   r*   Ú	Exceptionr+   rB   rC   rQ   rX   r`   ra   rr   r   r‘   rb   r‡   r¥   r¦   rµ   r¶   r¹   r(   r(   r(   r)   Ú<module>   sf   
? ý ÿ<