o ]Lb?@sddlmZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z ddlmZmZmZddlmZdd lmZmZGd d d eZGd d d eZdS))absolute_importN)_)binhex)open)errorpycompatutil)hashutil) constants shallowutilc@seZdZd#ddZddZd$ddZd d Zd d Zd dZddZ ddZ ddZ ddZ ddZ ddZddZddZdd Zd!d"ZdS)% basestoreFcCs||_|j|_||_||_||_tjstnd|_ |j dd|_ |j ddd|_ |j dvr3d|_ |j dkr;d|_ |rFt |j|dSdS) aCreates a remotefilelog store object for the given repo name. `path` - The file path where this store keeps its data `reponame` - The name of the repo. This is used to partition data from many repos. `shared` - True if this store is a shared cache of data from the central server, for many repos on this machine. False means this store is for the local data for one repo. N remotefilelogsvalidatecachelogs validatecacheon)rstrictoffrF)repoui_path _reponame_sharedr iswindowsosgetuid_uidconfig_validatecachelog_validatecacher mkstickygroupdir)selfrpathreponamesharedr$?/usr/lib/python3/dist-packages/hgext/remotefilelog/basestore.py__init__s&   zbasestore.__init__cCs^g}|D](\}}|||}tj|}|r#|jdkr#||ds#d}|s,|||fq|S)NrscontainsF) _getfilepathrr!existsr _validatekeyappend)r keysmissingnamenodefilepathr(r$r$r% getmissing8s     zbasestore.getmissingNcCsZ|r |tjr dS|jr)|D]\}}|D]}||||||||qqdSdSN)getr OPTION_PACKSONLYr _getfiles markdataentrymarkhistoryentry)r ledgeroptionsfilenamenodesr.r$r$r% markledgerJszbasestore.markledgercCs|j}|j|g}d}|jtddt|d}|D]!}|js%|jr7|jr7| || |j |j }t ||d7}q|||dS)Nrs cleaning upfilesunittotalr )rsourcesr2 makeprogressrlengced datarepackedhistoryrepackedupdater'r9r.r tryunlinkcomplete_cleanupdirectory_getrepocachepath)r r7rentriescountprogressentryr!r$r$r%cleanupSs   zbasestore.cleanupc Cst}t}tj|D]?\}}t|r2tj||}| |zt |Wq t y1Yq wt |rK| drF||ddq ||q ||D]}tj||d}t|qPdS)zRemoves the empty directories and unnecessary files within the root directory recursively. Note that this method does not remove the root directory itself._oldN)setr osutillistdirstatS_ISDIRrr!joinrIrmdirOSErrorS_ISREGendswithaddrG) r rootdiroldfiles otherfilesr-modedirpathr9r/r$r$r%rIgs(        zbasestore._cleanupdirectoryccs^i}|D] \}}||g|q||}t|D] \}}|||fVq!dS)a8Return a list of (filename, [node,...]) for all the revisions that exist in the store. This is useful for obtaining a list of all the contents of the store when performing a repack to another store, since the store API requires name+node keys and not namehash+node keys. N) _listkeys setdefaultr*_resolvefilenamesr+r iteritems)r existing filenamehashr. filenamemapr9shar$r$r%r4szbasestore._getfilesc Cs|siSi}t|}|jdD]}t|}||vr'|||<||q|jj}t t |dddD](}|s@|S| | |}|D]}t|}||vr`|||<||qJq9|S)zGiven a list of filename hashes that are present in the remotefilelog store, return a mapping from filename->hash. This is useful when converting remotefilelog blobs into other storage formats. stipr )rRrmanifestr sha1digestdiscard unfiltered changelogrxrangerB readfilesr.) r hashes filenamesmissingfilenamer9riclrevfilesr$r$r%rds0   zbasestore._resolvefilenamescCs|jr tj|j|jS|jSr1)rrr!rWrr)r r$r$r%rJs zbasestore._getrepocachepathccszt|D]2\}}}|D]*}t|dkrq|}|jr*|dd|dd}n|dd}t|t|fVqqdS)zList all the remotefilelog keys that exist in the store. Returns a iterator of (filename hash, filecontent hash) tuples. (iiiNi)rwalkrJrBrr)r rootdirsrxr9r.rgr$r$r%rbs  zbasestore._listkeyscCs<t|}|jrt|j||}nt||}tj|j |Sr1) rrr getcachekeyr getlocalkeyrr!rWr)r r-r.keyr$r$r%r's  zbasestore._getfilepathc Cs|||}zBt|}|jrC|||sF|jr5t|jd}|d|Wdn1s0wYt ||dt d|W|SW|St yZt d||t |fw)Nab+scorrupt %s during read .corruptscorrupt local cache file %ssno file found at %s for %s:%s) r'r readfiler _validatedatarrwriterrenameKeyErrorIOErrorr)r r-r.r/datafr$r$r%_getdatas&     zbasestore._getdatac Cs|||}td}zPtj|r'|d}tj|r!t|t||t |j tj |tj ||dd|j rL||dsTttd|Wt|dSWt|dSt|w)NrPT)readonlyswrites"local cache write was corrupted %s)r'rumaskr!r(r unlinkfileshutilcopyrrdirname writefilerr)rAbortr)r r-r.rr/oldumask newfilenamer$r$r%addremotefilelognodes&        zbasestore.addremotefilelognodecCsztj|jd}t|d}|tj|dWdn1s#wYt|}|j|j kr;t |ddSdS)aCall this to add the given repo path to the store's list of repositories that are using it. This is useful later when doing garbage collection, since it allows us to insecpt the repos to see what nodes they want to be kept alive in the store. repossab Ni) rr!rWrrrrrUst_uidrchmod)r r! repospath reposfile repospathstatr$r$r%markrepo s   zbasestore.markrepocCst|d }|}Wdn1swY|||r!dS|jrCt|jd}|d||fWdn1s>wYt||ddS)NsrbTrscorrupt %s during %s rF)rreadrrrrr)r r!actionrrr$r$r%r)s   zbasestore._validatekeyc Csz6t|dkr4t|\}}}t||krWdS||7}|||d}tj|t|kr1WdSWdSWdSttjfyCYdSw)NrFT) rBr parsesizeflagsrr!basenamer ValueErrorBadRemotefilelogHeader)r rr!offsetsizeflagsdatanoder$r$r%r%s   zbasestore._validatedatac Cs|j}|j}tj}d}d}d}d}td} |jtddd} | dt |D]\} } } | D]}|dkr:q3d| vr?q3| |t j | |}t j ||}|d7}zt |}Wn$ty}z|jtjkrltd }|||WYd}~q3d}~ww||j7}||vs|j| kr||j||f||j7}q3zt|Wn$ty}z|jtjkrtd }|||WYd}~q3d}~ww|d7}q3q,| |d d } || krU|| }|jtd d |d} d}|rQ|| krQ|dkrQ| ||\}}}zt|Wn&ty6}z|jtjkr!td }|||WYd}~nd}~ww||j8}|d7}||j7}|rQ|| krQ|dks| |td||t|dddt|dddfdS)NriQsremoving unnecessary filesr<)r>rs/packs/r s0warning: file %s was removed by another process rs cachelimitsenforcing cache limitsbytesr=s8finished: removed %d of %d files (%0.2f GB to %0.2f GB) g@)rrrqueue PriorityQueuetimerArrFrrzr!rWrelpathrUrYerrnoENOENTwarnst_sizest_atimeputr rrH configbytesr2statusfloat)r keepkeysr cachepathr originalsizerrLremovedlimitrMr{r|rxfiler!rpathstatemsgexcess removedexcessatimeoldpath oldpathstatr$r$r%gc<s           +      z basestore.gc)Fr1)__name__ __module__ __qualname__r&r0r;rOrIr4rdrJrbr'rrrr)rrr$r$r$r%rs"   ""  rc@s(eZdZddZddZeddZdS)baseunionstorecOs$|ddd|_|dd|_dS)N numretriesrr retrylog)r2 numattemptsr)r argskwargsr$r$r%r&szbaseunionstore.__init__cCs$|jD] }t|dr|qdS)Nsmarkforrefresh)storesr safehasattrmarkforrefresh)r storer$r$r%rs  zbaseunionstore.markforrefreshcsddfdd}|S)NcWsdSr1r$)rr$r$r%noopsz&baseunionstore.retriable..noopcs|jp}j}d}||jkrR|dkr"|d|t|f||d7}z |g|Ri|WStyJ||jkrH|dt|Ynw||jksdSdS)Nrsre-attempting (n=%d) %s r s*retries exhausted in %s, raising KeyError )rrrrsysbytesrr)r rrrfuncnameifnrr$r%wrappeds2     z)baseunionstore.retriable..wrappedr$)rrr$rr% retriableszbaseunionstore.retriableN)rrrr&r staticmethodrr$r$r$r%rs  r) __future__rrrrrUrmercurial.i18nrmercurial.noderrmercurial.pycompatr mercurialrrr mercurial.utilsr r r objectrrr$r$r$r%s"