o Eeɕ@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+dd l,m-Z-e.d Z/e.d ej0Z1e.d Z2e.d ej0j3Z4d5Z6gdZ7dZ8dZ9e9j:dj:ej;edZddZ?dBddZ@dBddZAdBddZBdedfd d!ZCd"d#ZDe.d$ej0ZE eDd%d&ZFGd'd(d(ZGGd)d*d*eGZHGd+d,d,eZIe.d-jJZKd.d/ZLd0d1ZMdCd2d3ZNd4d5ZOGd6d7d7ZPGd8d9d9e jQZRejSjTfd:d;ZUdd?ZWd@dAZXdS)Dz$PyPI and direct package downloading.Nwraps) CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST parse_version)logDistutilsError) translate)Wheelunique_everseenz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z.tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_namezsz(interpret_distro_name..Nr6) py_versionrDrW)r:anyrangerSisdigitrjoin)rMrTrIrarDrWr<r^r$r$r%rs$  rcstfdd}|S)zs Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. cst|i|SNr)argskwargsfuncr$r%wrapperszunique_values..wrapperr)rjrkr$rir% unique_valuessrlz3<([^>]*\srel\s{0,10}=\s{0,10}['"]?([^'" >]+)[^>]*)>c cst|D]2}|\}}tttj|d}d|vs#d|vr8t |D]}t j |t |dVq(qdD] }||}|dkr[t ||}|r[t j |t |dVq;dS)zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,homepager3r6)z Home PagezDownload URLr2N)RELfinditergroupssetmapstrstripr*r:HREFr7r urljoin htmldecoderHfindsearch)r;pagerGtagrelrelsposr$r$r%find_external_linkss    rc@(eZdZdZddZddZddZdS) ContentCheckerzP A null content checker that defines the interface for checking content cCdS)z3 Feed a block of data to the hash. Nr$selfblockr$r$r%feedzContentChecker.feedcCr)zC Check the hash. Return False if validation fails. Tr$rr$r$r%is_validrzContentChecker.is_validcCr)zu Call reporter with information about the checker (hash name) substituted into the template. Nr$)rreportertemplater$r$r%reportszContentChecker.reportN)__name__ __module__ __qualname____doc__rrrr$r$r$r%rs  rc@sBeZdZedZddZeddZddZ dd Z d d Z d S) HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)cCs||_t||_||_dSrf) hash_namehashlibnewhashexpected)rrrr$r$r%__init__s  zHashChecker.__init__cCsBtj|d}|s tS|j|}|stS|di|S)z5Construct a (possibly null) ContentChecker from a URLr2Nr$)r7r r8rpatternrz groupdict)clsr;rBrGr$r$r%from_urls zHashChecker.from_urlcCs|j|dSrf)rupdaterr$r$r%rzHashChecker.feedcCs|j|jkSrf)r hexdigestrrr$r$r%r"rzHashChecker.is_validcCs||j}||Srf)r)rrrmsgr$r$r%r%s zHashChecker.reportN) rrrr\compilerr classmethodrrrrr$r$r$r%r s   rcsdeZdZdZ    dNfdd Zfdd ZdOd d ZdOd dZdOddZddZ ddZ ddZ ddZ ddZ dPddZddZdPfdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Z dQd+d,ZdRd-d.Zd/d0Zd1Zd2d3Zd4d5ZdPd6d7Zd8d9Zd:d;Zdd?Z d@dAZ!e"dOdBdCZ#dDdEZ$dFdGZ%dHdIZ&dJdKZ'dLdMZ(Z)S)Srz;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTcsltj|i||dd|d |_i|_i|_i|_td t t |j |_ g|_tjj|_dS)Nr1|)superrr+ index_url scanned_urls fetched_urls package_pagesr\rrersrrGallowsto_scanr7requesturlopenopener)rrhosts ca_bundle verify_sslrgkw __class__r$r%r-s zPackageIndex.__init__cs0zt|jWn tyYdSwt|Srf)rrO Exceptionradd)rrJrr$r%r?s   zPackageIndex.addFc Cs||jvr |s dSd|j|<t|s||dStt|}|r.||s(dS|d||s7|r7||jvrAtt|j |dS||sMd|j|<dS| d|d|j|<d}| |||}|durhdSt |t jjr||jdkr|| d|jd|j|j<d|jd d vr|dS|j}|}t |tst |t jjrd }n|jd pd }||d }|t|D]} t j|t| d} |!| q|"|j#rt$|dddkr|%||}dSdSdS)z.)filterrYr?rr itertoolsstarmap scan_egg_link)r search_pathdirs egg_linksr$r$r%scan_egg_linkss zPackageIndex.scan_egg_linkscCsttj||}ttdttj|}Wdn1swYt |dkr,dS|\}}t tj||D]}tjj|g|R|_ t |_ ||q9dS)Nr`)openrYr?rerrrsrtrurSr rMrrDr)rr?r raw_lineslinesegg_path setup_pathrJr$r$r%rs  zPackageIndex.scan_egg_linkcCsd}||js |Stttjj|t|jdd}t|dks)d|dvr+|St |d}t |d}d|j | i|<t|t|fS)N)NNr1r`r5r6rT)r,rrrsr7r r9rSr:r r r setdefaultr*r )rrNO_MATCH_SENTINELr<pkgverr$r$r%_scans &  zPackageIndex._scanc Cst|D]}z|tj|t|dWqty!Yqw||\}}|s-dSt ||D]$}t |\}}| drQ|sQ|rL|d||f7}n| || |q2tdd|S)z#Process the contents of a PyPI pager6r.pyz #egg=%s-%scSsd|dddS)Nz%sr6r`)rH)mr$r$r%sz,PackageIndex.process_index..)rvrprr7r rwrxrHr!rrCr+need_version_infoscan_urlPYPI_MD5sub) rr;r{rGrrnew_urlr.fragr$r$r%rs&"    zPackageIndex.process_indexcCs|d|dS)NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allrr;r$r$r%rszPackageIndex.need_version_infocGs<|j|jvr|r|j|g|R|d||jdS)Nz6Scanning index of all packages (this may take a while))rrrrrrrrgr$r$r%rs  zPackageIndex.scan_allcCsz||j|jd|j|js||j|jd|j|js)||t|j|jdD]}||q3dS)Nr1r$) rr unsafe_namerrkeyrNnot_found_in_indexr)r requirementr;r$r$r% find_packagess  zPackageIndex.find_packagescsR|||||jD]}||vr|S|d||qtt|||S)Nz%s does not match %s)prescanrrrrrobtain)rr installerrJrr$r%r s zPackageIndex.obtaincCsL||jd||s$|t|td|jjtj |fdS)z- checker is a ContentChecker zValidating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) rrrrrYunlinkrrr-r?rT)rcheckerrZtfpr$r$r% check_hash s zPackageIndex.check_hashcCsN|D]"}|jdust|r|dstt|r||q|j|qdS)z;Add `urls` to the list that will be prescanned for searchesNfile:)rrr,rrrappend)rurlsr;r$r$r%add_find_linkss   zPackageIndex.add_find_linkscCs"|jr tt|j|jd|_dS)z7Scan urls scheduled for prescanning (e.g. --find-links)N)rrrsrrr$r$r%r &s zPackageIndex.prescancCs<||jr |jd}}n|jd}}|||j|dS)Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rrrrr)rrmethrr$r$r%r,s   zPackageIndex.not_found_in_indexcCs~t|ts5t|}|r)||d||}t|\}}|dr'||||}|Stj |r1|St |}t | ||ddS)aLocate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). If it is the URL of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method returns the location of a matching distribution (possibly after downloading it to `tmpdir`). If `spec` is a locally existing file or directory name, it is simply returned unchanged. If `spec` is a URL, it is downloaded to a subpath of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. r6rrMN)rrr _download_urlrHrCr+ gen_setuprYr?rr&rfetch_distribution)rr"tmpdirr=foundr.rBr$r$r%r37s    zPackageIndex.downloadc sd|id}d fdd }|r$|||}|s/|dur/|||}|dur@jdur<||}|durO|sO|||}|dur`drZdp[d|dSd||j|jd S) a|Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, the returned distribution's ``location`` is the value you would have gotten from calling the ``download()`` method with the matching distribution's URL or filename. If no matching distribution is found, ``None`` is returned. If the `source` flag is set, only source distributions and source checkout links will be considered. Unless the `develop_ok` flag is set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. zSearching for %sNcs|dur}||jD];}|jtkr#s#|vr"d|d|<q ||vo.|jtkp. }|rF|j}||_tj |jrF|Sq dS)Nz&Skipping development or system egg: %sr6) rrDrrrr3rMdownload_locationrYr?r)reqenvrJtestloc develop_okrskippedsourcerr$r%ryws&z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of rzBest match: %s)rMrf)rr rrrcloner) rrr force_scanr"r  local_indexrJryr$rr%rYs2         zPackageIndex.fetch_distributioncCs"|||||}|dur|jSdS)a3Obtain a file suitable for fulfilling `requirement` DEPRECATED; use the ``fetch_distribution()`` method now instead. For backward compatibility, this routine is identical but returns the ``location`` of the downloaded distribution instead of a distribution object. N)rrM)rrrr$r"rJr$r$r%fetchszPackageIndex.fetchc Cst|}|rddt||ddDpg}t|dkrztj|}tj||krGtj ||}tj |r?tj ||sGt |||}ttj |dd}|d|dj|djtj|dfWd|S1sswY|S|rtd||ftd ) NcSsg|]}|jr|qSr$)rO)r]dr$r$r% s z*PackageIndex.gen_setup..r6zsetup.pywzIfrom setuptools import setup setup(name=%r, version=%r, py_modules=[%r]) rzCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rFrGrrHrSrYr?rTdirnamerersamefileshutilcopy2rwriterNrOsplitextr) rrZrBrrGrrTdstrr$r$r%rsJ      zPackageIndex.gen_setupi c Cs2|d|d}zt|}||}t|tjjr%td||j |j f|}d}|j }d}d|vrI| d} t tt| }||||||t|d1} ||} | ro|| | | |d 7}||||||nnqP|||| Wdn1swY|W|r|SS|r|ww) NzDownloading %szCan't download %s: %s %srr2zcontent-lengthzContent-LengthwbTr6)rrrrrr7rrrrr dl_blocksizeget_allmaxrsint reporthookrrrr.rr) rr;rZfpr rblocknumbssizesizesrrr$r$r% _download_tosF            zPackageIndex._download_tocCsdSrfr$)rr;rZr8blksizer:r$r$r%r6 zPackageIndex.reporthookc Cs|dr t|Szt||jWSttjjfyD}z$ddd|j D}|r0| ||n t d||f|WYd}~dSd}~wt j jyY}z|WYd}~Sd}~wt j jy}z|rl| ||jn t d||jf|WYd}~dSd}~wtjjy}z|r| ||jn t d||jf|WYd}~dSd}~wtjjtfy}z|r| ||n t d||f|WYd}~dSd}~ww)Nr cSsg|]}t|qSr$)rt)r]argr$r$r%r(sz)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r, local_openopen_with_authrr!httpclient InvalidURLrergrrr7rrURLErrorreason BadStatusLineline HTTPExceptionOSError)rr;warningvrr$r$r%r sV  zPackageIndex.open_urlcCst|\}}|rd|vr|dddd}d|vs nd}|dr(|dd}tj||}|dks8|d r>|||S|d ksG|d rM|||S|d rX| ||S|d krht j t j |dS||d|||S)Nz...\___downloaded__rKr)svnzsvn+gitzgit+zhg+rr`T)rCreplacer+rYr?rer, _download_svn _download_git _download_hgr7r url2pathnamer r8r_attempt_download)rr=r;rr-rBrZr$r$r%r/s(         zPackageIndex._download_urlcCs||ddS)NT)rrr$r$r%rLrzPackageIndex.scan_urlcCs2|||}d|ddvr||||S|S)Nrrr)r<rr*_invalid_download_html)rr;rZrr$r$r%rYOs zPackageIndex._attempt_downloadcCst|td|)NzUnexpected HTML page found at )rYr r)rr;rrZr$r$r%rZVs z#PackageIndex._invalid_download_htmlcCstd|)Nz/Invalid config, SVN download is not supported: r)rr; _filenamer$r$r%rUZszPackageIndex._download_svncCsptj|\}}}}}|ddd}|ddd}d}d|vr)|dd\}}tj||||df}||fS)N+r6r2r5r@r)r7r urlsplitr:rsplit urlunsplit)r; pop_prefixr=netlocr?rArrevr$r$r%_vcs_split_rev_from_url]sz$PackageIndex._vcs_split_rev_from_urlcCl|ddd}|j|dd\}}|d||td||f|dur4|d|td ||f|S) Nr5r6rTrazDoing git clone from %s to %szgit clone --quiet %s %szChecking out %szgit -C %s checkout --quiet %sr:rdrrYsystemrr;rZrcr$r$r%rVo zPackageIndex._download_gitcCre) Nr5r6rTrfzDoing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -qrgrir$r$r%rWrjzPackageIndex._download_hgcGtj|g|RdSrf)rrrr$r$r%rzPackageIndex.debugcGrkrf)rrrr$r$r%rrlzPackageIndex.infocGrkrf)rrrr$r$r%rrlzPackageIndex.warn)rrNT)Frf)FFFN)FF)*rrrrrrrrrrrrrrrrr rrr rr3rr&rr2r<r6rrrrYrZrU staticmethodrdrVrWrrr __classcell__r$r$rr%r*s^  5     & O .$ " rz!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?cCs|d}t|S)Nr)rHrunescape)rGwhatr$r$r% decode_entitys  rqcCs tt|S)a Decode HTML entities in the given text. >>> htmldecode( ... 'https://../package_name-0.1.2.tar.gz' ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' ) entity_subrq)textr$r$r%rxs rxcsfdd}|S)Ncsfdd}|S)Nc s:t}tz |i|Wt|St|wrf)socketgetdefaulttimeoutsetdefaulttimeout)rgrh old_timeout)rjtimeoutr$r%_socket_timeouts  z@socket_timeout.._socket_timeout.._socket_timeoutr$)rjryrxrir%rysz'socket_timeout.._socket_timeoutr$)rxryr$rzr%socket_timeouts  r{cCs2tj|}|}t|}|}|ddS)a9 Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' Long auth strings should not cause a newline to be inserted. >>> long_auth = 'username:' + 'password'*10 >>> chr(10) in str(_encode_auth(long_auth)) False  r)r7r r9encodebase64 b64encoderrT)authauth_s auth_bytes encoded_bytesencodedr$r$r% _encode_auths   rc@r) Credentialz: A username/password pair. Use like a namedtuple. cCs||_||_dSrfusernamepassword)rrrr$r$r%rs zCredential.__init__ccs|jV|jVdSrfrrr$r$r%__iter__s zCredential.__iter__cCs dt|S)Nz%(username)s:%(password)s)varsrr$r$r%__str__s zCredential.__str__N)rrrrrrrr$r$r$r%rs  rcs8eZdZfddZeddZddZddZZS) PyPIConfigcsPtgdd}t|tjtjdd}tj|r&| |dSdS)z% Load from ~/.pypirc )rr repositoryr~z.pypircN) dictfromkeysrrrYr?re expanduserrr)rdefaultsrcrr$r%rs   zPyPIConfig.__init__cs&fddD}ttj|S)Ncs g|] }|dr|qS)r)rru)r]sectionrr$r%r(s z2PyPIConfig.creds_by_repository..)sectionsrrs_get_repo_cred)rsections_with_repositoriesr$rr%creds_by_repositorys zPyPIConfig.creds_by_repositorycCs6||d}|t||d||dfS)Nrrr)rrur)rrrepor$r$r%rs zPyPIConfig._get_repo_credcCs*|jD] \}}||r|SqdS)z If the URL indicated appears to be a repository defined in this config, return the credential for that repository. N)ritemsr,)rr;rcredr$r$r%find_credentials  zPyPIConfig.find_credential) rrrrpropertyrrrrnr$r$rr%rs    rcCs:tj|}|\}}}}}}|drtjd|dvr$t|\} } nd} | sBt |} | rBt | } | j |f} t j dg| R| redt| } || ||||f} tj| }tj|}|d| ntj|}|dt||}| rtj|j\}}}}}}||kr|| kr||||||f} tj| |_|S) z4Open a urllib2 request, handling HTTP authentication:znonnumeric port: '')rChttpsNz*Authenticating as %s for %s (from .pypirc)zBasic Authorizationz User-Agent)r7r r8r+rCrDrE _splituserrrrtrrrr urlunparserRequest add_header user_agentr;)r;rparsedr=rbr?paramsrArraddressrrr<rrr7s2h2path2param2query2frag2r$r$r%rBs8          rBcCs$|d\}}}|r||fSd|fS)zNsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.r]N) rpartition)hostuserdelimr$r$r%r@srcCs|Srfr$)r;r$r$r% fix_sf_urlKr>rc Cs*tj|\}}}}}}tj|}tj|rtj|S| dr}tj |r}g}t |D];} tj || } | dkrXt | d } | } Wdn1sQwYn tj | rb| d7} |dj| dq0d} | j|d |d } d \}}nd \}}} d d i}t| }tj|||||S)z7Read a local path, with special support for directoriesr1z index.htmlrNz{name})r-zB{url}{files}r|)r;files)OK)rzPath not foundz Not foundrz text/html)r7r r8rrXrYr?isfilerr+rrrerrrformatioStringIOrr)r;r=r>r?paramrArrZrrfilepathr7bodyrstatusmessager body_streamr$r$r%rAOs0         rArf)r)YrsysrYr\rr,rtr~rr configparserr http.clientrC urllib.parser7urllib.request urllib.error functoolsrr pkg_resourcesrrrrrr r r r r rrrr distutilsrdistutils.errorsrfnmatchrsetuptools.wheelr setuptools.extern.more_itertoolsrrrFIrvrrGrr:rR__all___SOCKET_TIMEOUT_tmplr version_inforr&rrCrrEr[rrlrorrrrrrrrqrxr{rrRawConfigParserrrrrBrrrAr$r$r$r%s @          # "  ! { '/