o ]Lb-Y@sddlmZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZmZmZmZmZddlmZmZddlmZmZmZejZdadadadadZ d d Z!d d Z"d dZ#Gddde$Z%ddZ&ddZ'ddZ(Gddde$Z)dS))absolute_importN)_)binhex)errorpycompatrevlogsshpeerutilwireprotov1peer)hashutilprocutil) constants contentstore metadatastorecCs4tt|}tj||dd|dd|S)Nrr sha1digestospathjoin)reponamefileidpathhashrF/usr/lib/python3/dist-packages/hgext/remotefilelog/fileserverclient.py getcachekey0s"rcCs tt|}tj||SNr)rrrrrr getlocalkey5sr!cs Gfddd|j|_dS)NcsPeZdZejddZejddZddZfddZfd d Z Z S) z!peersetup..remotefilepeercs0|ds tdfdd}d|fS)N x_rfl_getfiles5configured remotefile server does not support getfilecs*|dd\}}t|rt||S)Nr)splitintr LookupError)dcodedatarnoderrdecodeCsz?peersetup..remotefilepeer.x_rfl_getfile..decodesfilesnodecapablerAbort)selfrr+r,rr*r x_rfl_getfile<s z/peersetup..remotefilepeer.x_rfl_getfilecSs(|ds tddd}d|i|fS)Nsx_rfl_getflogheadss:configured remotefile server does not support getflogheadscSs|r|dSgS)N )r$)r'rrrr,SzDpeersetup..remotefilepeer.x_rfl_getflogheads..decodespathr.)r1rr,rrrx_rfl_getflogheadsKs  z4peersetup..remotefilepeer.x_rfl_getflogheadscSs|dkrdStj|vrdSt|dsdStj|jjvr dS|d}|r+|g}ng}| tj |jj rGd |jj }d|}| ||jj r[d |jj }d|}| |d ||d<dS)Ns getbundle _localrepos bundlecapsr#sincludepattern=sexcludepattern=,)rNETWORK_CAP_LEGACY_SSH_GETFILES capabilitiesr safehasattrSHALLOWREPO_REQUIREMENTr6 requirementsgetappendBUNDLE2_CAPABLITYincludepatternrexcludepattern)r1commandopts bundlecapspatterns includecap excludecaprrr_updatecallstreamoptsXs4     z7peersetup..remotefilepeer._updatecallstreamoptscs&|||t|j||fi|Sr )rHsuper _sendrequest)r1rBargsrC __class__remotefilepeerrrrJs  z.peersetup..remotefilepeer._sendrequestcs@t|}t|ds||t|t|j|fi|S)NrJ)rIr r:rHr byteskwargs _callstream)r1rBrC supertyperLrrrPs  z-peersetup..remotefilepeer._callstream) __name__ __module__ __qualname__r batchabler2r5rHrJrP __classcell__rrNrMrrN;s   (rNrX)uipeerrrWr peersetup:s Qr[c@s:eZdZdZddZddZddZdd d Zd d Zd S)cacheconnectionzThe connection for communicating with the remote cache. Performs gets and sets by communicating with an external process that has the cache-specific implementation. cCs"d|_|_|_d|_d|_dS)NF)pipeopipeipipee subprocess connectedr1rrr__init__s zcacheconnection.__init__cCs8|jr ttdt|\|_|_|_|_d|_ dS)Nscache connection already openT) r]rr0rr popen4r^r_r`ra)r1 cachecommandrrrconnects  zcacheconnection.connectcCsdd}|jrEz|jdWn tyYnw||jd|_||jd|_||jd|_z|jWn tyAYnwd|_d|_dS)NcSs$z|WdStyYdSwr )close Exception)piperrrtrycloses  z'cacheconnection.close..tryclosesexit F)rar^writerhr]r_r` communicate)r1rjrrrrgs(      zcacheconnection.closeTcCsN|jr%z|j||r|jWdSWdSty$|YdSwdSr )rar^rkflushIOErrorrg)r1requestrmrrrros  zcacheconnection.requestcCsR|jsdSz|jdd}|s|W|SW|Sty(|Y|Sw)N)rar]readlinergrn)r1resultrrr receivelines  zcacheconnection.receivelineNT) rRrSrT__doc__rcrfrgrorsrrrrr\s  r\c Cs|Q}g}|D]}||d|||dddq t|D]*\} }|| } d|| <||} |dd} |tdt| | f| | |q"WddS1sXwYdS)Nr"r-s%d %s)commandexecutorr> callcommand enumeraterrioBytesIOlen) remotereceivemissing progresstickmissedidmap batchsizeefuturesmirfile_r+rrr_getfilesbatchs"   "rcCs|dd}|j}|j}|t|krd|} tt|| |} | }|| | D]} | dd} || } d| | f}||q&||| | D]} | dd} || } ||| | |qH|t|ks|d|dS)Nx_rfl_getfilesrrv%s%s r3)rP_pipeo_pipeir|minrkrm)r}r~rrrsteprr]r^startend missingid versionidr sshrequestrrr_getfiles_optimistics,         rc s|d|j|j}fdd}tj|d}d|_|D]} | dd} | } ||| | |q"|d dS)Nrcs@D]}|dd}|}d||f}|qdS)Nrvr)rkrm)rrrrrrr]rrwriters     z"_getfiles_threaded..writer)targetTrvr3) rPrr threadingThreaddaemonrrrkrm) r}r~rrrrr^r writerthreadrrrrrr_getfiles_threadeds      rc@sdeZdZdZddZddZddZdd Zd d Zd d Z ddZ dddZ ddZ ddZ dS)fileserverclientz:A client for requesting files from the remote file server.cCs\|j}||_||_|dd|_|jrt|j|_|dd|_|dd|_t |_ dS)N remotefilelogs cacheprocessscacheprocess.includepathsdebug) rYrepoconfig cacheprocessr expandpath configboolcacheprocesspasspath debugoutputr\ remotecache)r1rrYrrrrc5s zfileserverclient.__init__cCs||_||_||_||_dSr ) datastore historystore writedata writehistory)r1rrrrrrrsetstoreGs zfileserverclient.setstorecCs|jj|jjSr )rconnectionpoolr= fallbackpathrbrrr_connectMr4zfileserverclient._connectc CsH|jjs||j}|j}|j}t|}d|}i}|j}|D]\} } t|| | } |jr3|| d7}|| d7}| || <q | ||j j t d|d} | dg} |}|svt| }|D] }||vrl| |qa|j t dn|d kr{n|d r|d }| t|d qT| |qUtt| 7a|t| }| j ||d|j jd d||||dtd }z| r|j j}d|j _zz|s}|j}|tjrt |t!st"#d|j $d d}|j %d d}|dkrt&}nt'}|||j(| j| ||n4|dr2|drd}nd}|j $d d|}|j )dt| t*||j(| j| ||nt"#dWdn 1sBwY|j jdd| j+|||dWnt,yo|j jdd | j+|||dwW||j _n||j _wd!t| d-| f}| || .|/|jj0Wt|dSt|w)"zTakes a list of filename/node pairs and fetches them from the server. Files are stored in the local cache. A list of nodes that the server couldn't find is returned. If the connection fails, an exception is raised. sget %d r#r3s downloading)totalrTs@warning: cache connection closed early - falling back to server 0s_hits__rrs"remote cache hit rate is %r of %r )hitrFs"remotefilelog requires ssh serverss getfilessteps getfilestypesthreadedr"sbatchd s batchsizes1requesting %d files from remotefilelog server... s>configured remotefilelog server does not support remotefilelogNsremotefilefetchlogsSuccess ) fetched_filestotal_to_fetchsFail s set %d %s )1rrarfrrr|namerrrorY makeprogressrupdaterssetr>warn startswithr$ incrementr% fetchmisseslogrumaskverboserrZr/rr8 isinstance _sshv1peerrr0 configintrrrr~debugrposrhrcompletemarkrepor)r1fileidscacherrrrorrrrfullidprogressrr missedsetparts fromcacheoldumaskrconnr}r getfilestype _getfiles batchdefaultrrrrroPs                   5  zfileserverclient.requestcCs|dd}|sttdtdt|}||}t||kr3ttdtdt||f|j|t |t |dS)Nrps error downloading file contents:sconnection closed earlysonly received %s of %s bytes) rqr ResponseErrorrr%readr|raddremotefilelognoderzlib decompress)r1rifilenamer+linesizer)rrrr~s   zfileserverclient.receivemissingcCsD|jrd|j|jjf}|j|dSGdddt}||_dS)Ns%s %sc@s.eZdZddZddZd ddZdd Zd S) z-fileserverclient.connect..simplecachecSsg|_d|_dS)NT) missingidsrarbrrrrc s z6fileserverclient.connect..simplecache.__init__cSsdSr rrbrrrrgsz3fileserverclient.connect..simplecache.closeTcSs8|d}|ddkr dS|dd|_|jddS)Nr3rsgetrrpr)r$rr>)r1valuermlinesrrrros  z5fileserverclient.connect..simplecache.requestcSst|jdkr |jdSdS)Nr)r|rpoprbrrrrss z9fileserverclient.connect..simplecache.receivelineNrt)rRrSrTrcrgrorsrrrr simplecache s   r)rr_pathrrfobject)r1cmdrrrrrfs  zfileserverclient.connectcCsztr0dttttttttdtf}|jr|j||jjd| ddttttdd|j j r;|j dSdS)NsN%d files fetched over %d fetches - (%d misses, %0.2f%% hit ratio) over %0.2fs gY@sremotefilelog.prefetch%s%%i)remotefilelogfetchedremotefilelogfetchesremotefilelogfetchmissesremotefilelogfetchtime) fetchesfetchedrfloat fetchcostrrYrrreplacerrarg)r1msgrrrrgs,   zfileserverclient.closeFTcsj}g}|D]\}}|dkst|dks||sq||t|fqj} j} |r9tj|j } t j |j } t } |rF| | ||rP| | |tfdd| D} | rpfdd| D} |jjd| dd| rtd 7atd krtd krtd krjd d } | rj| dddt| D} tt| 7at}| } | rttdt| tt|7a|dSdS)z.downloads the given file versions to the caches.hgtags*cs g|] \}}|jjkrdqSr rnullid).0unusedrrbrr ^s z-fileserverclient.prefetch..cs$g|]\}}|jjkr||fqSrr)rfrrbrrrassDremotefilelog not fetching %d null revs - this is likely hiding bugssremotefilelog-ext)rrrs fetchwarningr3cSsg|] \}}|t|fqSr)r)rrrrrrryssunable to download %d filesN)rr| shallowmatchr>rrrrunioncontentstoreshareddatastoresrunionmetadatastoresharedhistorystoresrr getmissingrY develwarnrrr logstacktracesortedrtimerorr0rr _lfsprefetch)r1rforce fetchdata fetchhistoryr idstocheckrrrrrnullids fetchwarningrrrbrprefetch9sn      zfileserverclient.prefetchc str t|jjds dStj|jsdSg}|jjj|D]/\}}t|}|j |}| |t j @rM| |}tj|}|} | sM||qt|dkrj|jjj|tfdd|DslJdSdS)Nslfslocalblobstorerc3s|] }|VqdSr )hasoid)rpstorerr sz0fileserverclient._lfsprefetch..)_lfsmodr r:rsvfswrapper candownloadlfslocalblobstorerrflagsrREVIDX_EXTSTOREDrawdatapointer deserializer r r>r|lfsremoteblobstore readbatchall) r1rpointersrrr+rlogtextrr rrrrs.        zfileserverclient._lfsprefetchc Cs0ddl}|jddddd|DdS)Nrrs"excess remotefilelog fetching: %s css|]}t|VqdSr )rsysbytes)rsrrrrsz1fileserverclient.logstacktrace..) tracebackrYrr format_stack)r1r%rrrrs zfileserverclient.logstacktraceN)FTF)rRrSrTrurcrrror~rfrgr rrrrrrr2s K r)* __future__rrzrrrrmercurial.i18nrmercurial.noderr mercurialrrrr r r mercurial.utilsr r rrr sshv1peerrrrrrrrr!r[rr\rrrrrrrrs2   UE