o ]Lb@sdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZmZddlmZmZmZddlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,d d l-m.Z.m/Z/d Z0iZ1e(2e1Z2e2d d dde2d ddde2d ddde2d ddde2d ddde2d ddde2dddde2d ddde2d ddde2dddde2dddddZ3dZ4dZ5e.j6Z6e(7Z7e(8Z8dda9e :dj;Zd$d%Z?d&d'Z@Gd(d)d)eAZBd*d+ZCd,d-ZDd.d/ZEd0d1ZFd2d3ZGd4d5ZHd6d7ZId8d9ZJdyd:d;ZKe+jLdd?ZNd@dAZOdBdCZPdDdEZQdFdGZRdHdIZSdzdJdKZTdLdMZUd{dOdPZVdQdRZWdSdTZXdUdVZYdWdXZZdYdZZ[d[d\Z\e%j]fd]d^Z^e!_e6d_d`Z`daejae.j6<dbdcZbejcdddeZddfdgZedhdiZfdjdkZgdldmZheie6dndodpZjdqdrZkdsdtZldudvZmdwdxZndS)|ax store some pushes in a remote blob store on the server (EXPERIMENTAL) IMPORTANT: if you use this extension, please contact mercurial-devel@mercurial-scm.org ASAP. This extension is believed to be unused and barring learning of users of this functionality, we will delete this code at the end of 2020. [infinitepush] # Server-side and client-side option. Pattern of the infinitepush bookmark branchpattern = PATTERN # Server or client server = False # Server-side option. Possible values: 'disk' or 'sql'. Fails if not set indextype = disk # Server-side option. Used only if indextype=sql. # Format: 'IP:PORT:DB_NAME:USER:PASSWORD' sqlhost = IP:PORT:DB_NAME:USER:PASSWORD # Server-side option. Used only if indextype=disk. # Filesystem path to the index store indexpath = PATH # Server-side option. Possible values: 'disk' or 'external' # Fails if not set storetype = disk # Server-side option. # Path to the binary that will save bundle to the bundlestore # Formatted cmd line will be passed to it (see `put_args`) put_binary = put # Serser-side option. Used only if storetype=external. # Format cmd-line string for put binary. Placeholder: {filename} put_args = {filename} # Server-side option. # Path to the binary that get bundle from the bundlestore. # Formatted cmd line will be passed to it (see `get_args`) get_binary = get # Serser-side option. Used only if storetype=external. # Format cmd-line string for get binary. Placeholders: {filename} {handle} get_args = {filename} {handle} # Server-side option logfile = FIlE # Server-side option loglevel = DEBUG # Server-side option. Used only if indextype=sql. # Sets mysql wait_timeout option. waittimeout = 300 # Server-side option. Used only if indextype=sql. # Sets mysql innodb_lock_wait_timeout option. locktimeout = 120 # Server-side option. Used only if indextype=sql. # Name of the repository reponame = '' # Client-side option. Used by --list-remote option. List of remote scratch # patterns to list if no patterns are specified. defaultremotepatterns = ['*'] # Instructs infinitepush to forward all received bundle2 parts to the # bundle for storage. Defaults to False. storeallparts = True # routes each incoming push to the bundlestore. defaults to False pushtobundlestore = True [remotenames] # Client-side option # This option should be set only if remotenames extension is enabled. # Whether remote bookmarks are tracked by remotenames extension. bookmarks = True )absolute_importN)binhex)_)getattropen)procutil stringutilurlutil)bundle2 changegroupcommands discoveryencodingerrorexchange extensionshg localrepophasespushkeypycompat registrarutilwireprototypeswireprotov1peerwireprotov1server) bundlepartscommonsships-with-hg-core infinitepushserverF)default storetype indextypes indexpath storeallpartsreponames scratchbranchs storepath branchpatternpushtobundlestores experimentalsserver-bundlestore-bookmarksinfinitepush-scratchpushcCsdSNF)xr+r+=/usr/lib/python3/dist-packages/hgext/infinitepush/__init__.pysr.z ^[a-f0-9]+$cCsl|ddg}|dd}|std|ddg}|dd}|s(tddd lm}|||||S) Nr sput_argss put_binarysput binary is not specifiedsget_argss get_binarysget binary is not specifiedrstore) configlistconfigrAbortr0externalbundlestore)uiput_args put_binaryget_args get_binaryr0r+r+r-_buildexternalbundlestores     r;c Cs|dd}|sttd|d\}}}}}|dd}|s(ttd|ddd}|dd d } |dd d } d dlm} | j|||||||t|| | d S)Nr ssqlhostsplease set infinitepush.sqlhost:r's please set infinitepush.reponameslogfiler$s waittimeouti,s locktimeoutxr) sqlindexapi) waittimeout locktimeout) r2rr3rsplit configintr4r> _getloglevel) r6sqlhosthostportdbuserpasswordreponamelogfiler?r@r>r+r+r-_buildsqlindexs.   rLcCs>|ddd}tt|d}t|tsttd||S)Nr sloglevelsDEBUGsinvalid log level %s) r2rloggingupper isinstanceintrr3r)r6loglevelnumeric_loglevelr+r+r-rC s  rCcCs8t|r|ddd}||r|t|dS|S)a)returns a bookmarks with hoisted part removed Remotenames extension has a 'hoist' config that allows to use remote bookmarks without specifying remote path. For example, 'hg update master' works as well as 'hg update remote/master'. We want to allow the same in infinitepush. remotenamess hoistedpeer/N)risremotebooksenabledr2 startswithlen)r6remotebookmarkhoistr+r+r- _tryhoists  rZc@seZdZddZdS) bundlestorecCs||_|jjdd}|dkr ddlm}||jj|j|_n|dkr,t|jj|_n tt d||jjdd}|dkrPdd lm }| |j|_ dS|d kr]t |jj|_ dStt d |) Nr r#sdiskrr/sexternals,unknown infinitepush store type specified %sr%) fileindexapissqls,unknown infinitepush index type specified %s) _repor6r2r4r0filebundlestorer;rr3rr\indexrL)selfrepo storetyper0 indextyper\r+r+r-__init__#s&    zbundlestore.__init__N)__name__ __module__ __qualname__rdr+r+r+r-r["s r[cCs |ddS)Nr r!) configboolr6r+r+r- _isserver>s rjcCs&t|r|rt||_dSdSdSN)rjlocalr[)r6rar+r+r- reposetupBsrmcCs(t|t|rt|dSt|dSrk) commonsetuprjserverextsetupclientextsetuprir+r+r-extsetupGs  rqcCs6tdftjd<|dd}|rt|\}}adSdS)Nsnamespace patternsslistkeyspatternsr r()wireprotolistkeyspatternsrr r2r stringmatcher_scratchbranchmatcher)r6scratchbranchpatkindpatr+r+r-rnOs rncstjdfdd}j|_|tjd<tjdfdd}j|_|tjd<ttjdttt j dd d ft j d<tt d t ttd t dS) Npushkeycstg|Ri|dSrk)bundle2pushkeyargskwargs)origpushkeyhandlerr+r-newpushkeyhandler_sz)serverextsetup..newpushkeyhandlers phase-headscstg|Ri|Srk)bundle2handlephasesrz)orighandlephasehandlerr+r-r.fsz serverextsetup..listkeysslookuprkeysgetbundlechunkss processparts)r parthandlermappingparamsr wrapfunctionrlocalrepositorylocalrepolistkeys _lookupwraprr rgetbundlechunks processparts)r6r~newphaseheadshandlerr+)rr}r-ro\s       rocCs|ttjdt}|ddddtdfttjdttt dt t t j _ tj}|d}||||tdS) Nspushrr$s bundle-stores/force push to go to bundle store (EXPERIMENTAL)pulls checkheadss changeset)r wrapcommandr table_pushappendr_pullrr _checkheadslistkeyspatternsrwirepeerrb2partsgenorderr_insertpopscratchbranchparttype)r6entry partorderr_r+r+r-rpxs   rpcCs|jttdr dS||Sr*)r6rh experimentalconfigscratchpush)origpushopr+r+r-rsrcCs,t|}t|t||}t|Srk) r decodelistr iteritemslistkeysrtolocalr encodekeys)raproto namespacepatternsdr+r+r-rrs  rrc Cs|dkrI|rI|jj}i}|||}|D]3}||||dr,d|ddd}t|\}} } t|D] \} } | | rE| || <q9q|S|||S)N bookmarks*sre:^s.*) r[r_update getbookmarksendswithr rsrr) rr`rrr_results bookmarkspatternrvrwmatcherbookmarknoder+r+r-rs    rcsJds idfSjdfdd}tt|d|fS)Nrxspreparing listkeys for "%s" cs"jdt|ft|S)Ns$received listkey for "%s": %i bytes )r6debugrWr decodekeys)rrr`r+r-decodes z listkeyspatterns..decode) namespacespatterns)capabler6rr fromlocalr encodelist)r`rrrr+rr-rs rcCst|dS)Nsbundle())listrevs) bundlerepor+r+r-_readbundlerevssrc Cst}|j}|D] }|||dq|s|Sd|}g}d}|p%gD]} | dr:|d| |fd}q&|| q&|sI|d||S)zTells remotefilelog to include all changed files to the changegroup By default remotefilelog doesn't include file content to the changegroup. But we need to include it if we are fetching from bundlestore. Fsexcludepattern=T)set changelogrreadjoinrVr) bundlecapsr bundlerevsr6 changedfilesclrnewcapsappendedcapr+r+r-_includefilelogstobundles"    rcCs`g}d}tj|||gd}t|||d}t|}tjd|d}| d|| ||S)z Bundle may include more revision then user requested. For example, if user asks for revision but bundle also consists its descendants. This function will filter out all revision that user is not requested. s02) commonheads ancestorsofr changegroupdataversion) routgoingr makestreamr chunkbufferrr bundlepartaddparamr)r bundleroots unknownheadpartsversionrcgstreamcgpartr+r+r- _rebundles  rc CsH|j}g}|D]}||}||}|D] }||vr ||qq|Srk)rrparentsr) oldreporrrrrevrrparentr+r+r-_getbundlerootss   rcCs0t|d}t|dko||d|k S)Nheads(bundle())rr)rrrWr)headr bundleheadsr+r+r-_needsrebundlingsrc Cs>g}t||st|d}t|j||}t|tjr2t j d|j d}| dd||nKt|t jrxd}|D])}|jdkrGd}t j |j| d} t|jD] \} } | | | qW|| q>|swtjdd t|d d n td Wd |SWd |S1swY|St|||}|S)zmgenerates bundle that will be send to the user returns tuple with raw bundle string and bundle type rbrrr01FTs,unexpected bundle without changegroup part, shead: %ssreport to administrator)hintsunknown bundle typeN)rr posixfiler readbundler6rOr cg1unpackerr r_streamrrr unbundle20 iterpartstyperrrrr3rr) rrr bundlefilerf unbundlerparthaschangegroupnewpartkeyvaluer+r+r-_generateoutputparts sL              rc sf|pg}igg}g}i}g} z|D]p} |jj| s| |vrlt|| } d|j| f} t|j| } | | | ft t | }t || ||j}| j}t || |}|D]}||}ttjt|<| || f||<qT t| g|| R||| | qW| D]\} }| zt|WqttfyYqwn| D]\} }| zt|WqttfyYqwwtd}d}tjdzOfdd}|tjd<d}fdd}ttj d |d}t!t |t |Bt |}|||f||d |}W|rtjd<|rt"tj d ||S|r&tjd<|r2t"tj d |ww) N bundle:%s+%sFrcs8|g|Ri|}D] }|D]}||qq |Srk)addpart)bundlerr{r|resultbundler)oldchangegrouppartscratchbundlesr+r-_changegrouppartjs  z)getbundlechunks.._changegrouppartTcs8|||}|dkrr|ddkr|d=||S)Nsphasess publishingTrue)getr)rr`r origvalues) newphasespullfrombundlestorer+r- _listkeysws   z"getbundlechunks.._listkeysr)headsr)#rr_has_noderdownloadbundlerootr repositoryr6rrrrrrstrrdraftrrextendcloseosunlinkIOErrorOSErrorboolrgetbundle2partsmappingrrrrrunwrapfunction)rrasourcer rr|newheads scratchheads nodestobundleallbundlestocleanupr newbundlefile bundlepathrrrrrrrwrappedchangegrouppartwrappedlistkeysrr rr+)rrr rr-r4s               rcCs dd}|S)Nc Sst|}t|tr%t|r%|jj|}|rdd|fSddd|fSzt| |}dd|fWSt yc}z$|jj |rNdd|fWYd}~St |}dd|fWYd}~Sd}~ww)Ns%d %s rrsscratch branch %s not found)rrrOrrtr[r_getnoderlookup Exception getbundler forcebytestr)rarrlocalkey scratchnoderinstr+r+r-_lookups&   z_lookupwrap.._lookupr+)rr,r+r+r-rsrdefaultc Kst|}tjd||||dd\}}i}|}g}|dgD] } | |vr-|| q"|drg} |dp;g} |dD]} t| rLd|| <qA| | qA|rt |||} z*| j d|d}|D]} | |vrrt d | || || <| || qeW| n| w| |d<| |d<|s|rttd tz5t|||}||||fit|}t|r||t|||nt|||W|rttd SS|rttd ww) Nsinfinite-push's pullsbranch)default_branchessrevbookmarks REVTOFETCHrrsremote bookmark %s not found!sfindcommonincoming)r byteskwargsr get_unique_pull_pathr unfilteredrrtrpeerrrr3rrrr_findcommonincoming_readscratchremotebookmarks strkwargsrrUr_saveremotebookmarks_savelocalbookmarksr)rr6raroptsbranchesscratchbookmarksunfi unknownnodesrrrrotherfetchedbookmarksremotescratchbookmarksrr+r+r-rsp           rc Cst|rGtd}||j|}i}|j|jd |D]$}| |\}}||krDt |rD|jd ||} | rDt | d||<q |SiS)NrSsremotebookmarksr)rrUrfind activepathr6 _remotenames clearnamesnames listnamessplitremotenamertnodesr) r6rar?remotenamesext remotepathrrXpathbooknamerIr+r+r-r6s    r6c Cstd}||j|}tt}i}||}|D](\}} }} ||kr%q| dkr5| |vr0|| =||| <q| dkrB|| t |qt |D]\} }||| <qH| ||||dS)NrSrbranches) rrBrCr6 collections defaultdictrreadremotenamesrrrrsaveremotenames) ra newbookmarksremoterJrKr;r remotenameshexnodenametypernamerr+r+r-r8 s$     r8c Cs|sdS|[|>|d)}g}t|D]\}}||}|||fq|j|||Wdn1s?wYWdn1sNwYWddSWddS1sfwYdS)Nr/) wlocklock transactionrrrr _bookmarks applychanges)rartrchanges scratchbookr changectxr+r+r-r9#s Pr9cOs||i|\}}}|d|fS)NTr+)rr{r|rinc remoteheadsr+r+r-r5.s r5c Ost|}|d}t|dkr|d}nd}d}ttf|i}||d|d}t|r8d}|tdd|rLg|d<|tt dt t d t }tt|||} t| dkrdtd } t| | d} | jpm| j} t||| } |||g|Rit|}t|r|r|rt||| }z|jd |gd }| |W|n|wt|| | Wdn1swY|r|t _ |S) Nr/rrr$r s bundle_storeTsbundle2.pushbacks_localphasemoves.cannot push to multiple path with infinitepushrr0)!rr1rrWrconfigbookmarkconfigoverridert setconfigrrrr _phasemoverr get_push_pathsrrr3pushloclocr6r7rrUrr4rrrr8_localphasemove)rr6radestsr:r oldphasemove overrides scratchpushpathsmsgrLdestpathrArr?r@r+r+r-r3sR             'rc std}dddfdd||D}fdd|D}|D]}||vr5ttd|q$i}tt } |D] \} } } }| d krR||vrR| ||<q?| d kr_| | t | q?| || |d S) zqPrune remote names by removing the bookmarks we don't want anymore, then writing the result back to disk rSrrcsg|] }|kr|qSr+r+.0 remotename)rL remote_idxr+r- vs  z0_deleteinfinitepushbookmarks..cs g|] }|dkr|qS)rr+rt)name_idx nametype_idxr+r-rx{s  s6infinitepush bookmark '{}' does not exist in path '{}'rrNN) rrBrQrr3rformatrOrPrrrrR) r6rarLrFrJrUremote_bm_namesnamerr;rrWrTr+)ryrzrLrwr-_deleteinfinitepushbookmarksks8     r~cCs|tjkr ||||dSdS)zprevent commits from being marked public Since these are going to a scratch branch, they aren't really being published.N)rpublic)rrrIphaser+r+r-rgs rgcsjtt}jtt}djvs|sdStt j vr!dSj dj j s8jtdd_dS|ddtjj j j|}|D]}||qMfdd}|S)Ns changesetssno changes found rr rcs d_dS)Nr)cgresult)oprr+r- handlereplys zpartgen..handlereply)r6r2rrdrhr stepsdonerr bundle2capsrTaddrmissingstatusrrrrgetscratchbranchpartsrar)rrrro scratchparts scratchpartrr+rr-partgens&    rr+csF fdddD}|dur|S|vr!td|r!|S|S)Ncg|]}|qSr+r+)rurrr+r-rxz_getrevs..ssort(bundle())sbundle() & %s::)rrr)roldnodeforcerrr+rr-_getrevssrc kst}||fddi|zdV||fdt|dd|WdStyH}z||fdt|dt|d|d}~ww)N eventtypestartsuccessr elapsedmsfailurerrerrormsg)timer&r r()loggerservicer|starter+r+r-logservicecalls4 rc Cs|jd}|sD|jj}zt}Wn tyd}Ynwttdd}t }t j |j d||||ddd}|jd||S|d}|S)Nsinfinitepushloggersunknownri5wr r')rH requestidhostnamerJ)recordsrar6rgetuserr&randomseedrandintsocket gethostname functoolspartiallogr2r)rrr6usernamerrr+r+r-_getorcreateinfinitepushloggers.     rcCs|jtdt|j}t|||m}|D]b}d}|jdkr(t||nAtj|j| d}t |j D] \}} | || q8|jdvri|jduri|jd|j} | j dd|jd d | j d d d d |j|jd d i|rz||qWdn1swYt|} t \} } z3zt| d}|| W|n|wt|i| Wzt| WdStyYdSwzt| WwtyYww)zmstores the incoming bundle coming from push command to the bundlestore instead of applying on the revlogss&storing changesets on the bundlestore N replycapsr)rxrsreply:%s in-reply-tos%dF mandatoryreturn1rwb)r6rrr bundle20 partiteratorr _processpartrrrrrrreplyridrrrrr getchunksmkstemprfdopenwriter storebundlerr&)rrarrrrrrrrrpartbuffdrfpr+r+r-storetobundlestore sZ          rcCs|jdkr ||||S|jddrt||||S|jddkr'||||S|jdd}t|j}d}t|||}|D]}d} |j dkrPt ||n{|j t j kr|jdd} tj d |d } | d | |j}|s|jt d d |jt dd nG|rtj |j |d } t|jD] \} } | | | q|j dkr|jdur|jd} | jdt|jdd| jddddnt |||r|j|j ddi| r|| q@Wdn1swY|rEt|}t\}}z4zt|d}||W| n| wt!|||Wzt"|WdSt#y2YdSwzt"|Wwt#yDYwwdS)Nsunbundler r)rr&r cgversionrrrr _skippushkeyT_skipphaseheadsrx reply:pushkeyrFrrrrr)$rr6rhrrrr rrrrrrrrrrrrrrrrrrrrrrrrrrrrr&)rrarrhandleallpartsrcgparamsrrr cgversionrrrrrrrr+r+r-r@s             =  rc st|}t}|tdd|jjj|jjj}|jtdddzzd|jj |f}t |jj || d}| dd}| d } |rO|} nd} d } |ret| d krettd t| | |} t| d krtd nd} |jj tdt| | fd}tdtt| |D] }| |dddd}|jj d| ||fqt| |d kr|jj d| ddddd}|jj d| d|ffdd| D}fdd| rtfdd| D}nd}|r|dnd}d}|r]t|d@}|}t|dt|d"d}t||kr8d }t|t|f||}Wdn 1sHwYWdn 1sXwYt|d!|d"5"|rq |||r!||t"|||||Wdn 1swYWdn 1swY|td#t|d$d%Wnt#y}z|td&t|d$t$%|d'd}~wwWrڈ&dSdSr&ww)(Nr)rrTrr/ bookprevnoder$forcerrs2cannot push more than one head to a scratch branchsspushing %d commit%s: r 2s %s %s s ... rcrr+r+rurrr+r-rxrzstorebundle..cst|Srk)rr'r)r)rr_r+r-r.szstorebundle..c3s|]}| VqdSrkr+r)inindexr+r- szstorebundle..rs bundlestore) bundlesizei@s7bundle is too big: %d bytes. max allowed size is 100 MBsindex) newheadscountrrrrr)'rrrrar[r_r0rrrrrr6rr$rrWrr3rrwarnrangemin descriptionrAsumrrrrr addbundle addbookmark_maybeaddpushbackpartr&r r(r)rrrrparthandlerstartr0r!r bookprevnoderrrrplural maxoutputi firstlinenodesctxr bookmarknoderr bundledatabundlesizelimit error_msgrr+)rr_rr-rs        ""         r)r/rrpushbackbookmarksrc Cs*t|jj}|jdd}tjd|d}|d|| |t | }t \}}zDzt|d}||W|n|wt||j|Wzt|WdStys} z| jtjkrhWYd} ~ dSd} ~ wwzt|Wwty} z | jtjkrWYd} ~ wd} ~ ww) z9unbundle a bundle2 part containing a changegroup to storerrrrrrNr)r rrar6rrrrrrrrrrrrrrrrrrerrnoENOENT) rrrrrrrrrrr+r+r-bundle2scratchbranchs>          rcCsP|dr"|jr$d|jjvr&d|||d}|jjdt|ddSdSdSdS)Nrspushbackr)rrsnewsoldrx)mandatoryparams)rr capabilitiesrrr)rrnewnoderrr+r+r-r&s   rcCsX|jtdr'|jdur%|jd}|jdt|jdd|jddddd S|||S) zWrapper of bundle2.handlepushkey() The only goal is to skip calling the original function if flag is set. It's set if infinitepush push is happening. rNrrFrrrr)rrrrrrr)rrrrr+r+r-ry4s   rycCs|jtdr dS|||S)zWrapper of bundle2.handlephases() The only goal is to skip calling the original function if flag is set. It's set if infinitepush push is happening. rN)rr)rrrr+r+r-rDs rc Csd}t||kr dSg}|D] }|d||qttjd#}tdd|g|}tjt t j |dd|||d WddS1sGwYdS) zstarts a separate process that fills metadata for the nodes This function creates a separate process and doesn't wait for it's completion. This was done to avoid slowing down pushes rNs--nodesw+bsdebugfillinfinitepushmetadatas-RTF) close_fdsshellstdinstdoutstderr) rWrrrdevnullr hgexecutable subprocessPopenrrapplyr tonativestr)rrImaxnodes nodesargsrrcmdliner+r+r-_asyncsavemetadataQs0    "rrk)NN)r-)o__doc__ __future__rrO contextlibrrrMrrrerrrmercurial.noderrmercurial.i18nrmercurial.pycompatrrmercurial.utilsrr r mercurialr r r rrrrrrrrrrrrrrrr4rr testedwith configtable configitemrrdrrrevsetpredicatetemplatekeywordrtcompilesearch _maybehashr;rLrCrZobjectr[rjrmrqrnrorprrrr batchablerrrrrrrrrrr6r8r9r5rr~rrgb2partsgeneratorrrrcontextmanagerrrrrr parthandlerrrryrrr+r+r+r-s S P       '^ F 8+  " 7``