o ]Lb @sldZddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZdd lm Z m!Z!z dd lm"Z"e"j#Wn e$ypdZ"YnwiZ%e&e%Z&e&d d e"rd nddddZ'dZ(dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dddddd d!d"e-gZ4dGd#d$Z5d%d&Z6Gd'd(d(ej7Z8ej9Gd)d*d*e:Z;eeZJd?d@ZKdAdBZLdCdDZMdEdFZNdS)Hamstore repository data in SQLite (EXPERIMENTAL) The sqlitestore extension enables the storage of repository data in SQLite. This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY GUARANTEES. This means that repositories created with this extension may only be usable with the exact version of this extension/Mercurial that was used. The extension attempts to enforce this in order to prevent repository corruption. In addition, several features are not yet supported or have known bugs: * Only some data is stored in SQLite. Changeset, manifest, and other repository data is not yet stored in SQLite. * Transactions are not robust. If the process is aborted at the right time during transaction close/rollback, the repository could be in an inconsistent state. This problem will diminish once all repository data is tracked by SQLite. * Bundle repositories do not work (the ability to use e.g. `hg -R log` to automatically overlay a bundle on top of the existing repository). * Various other features don't work. This extension should work for basic clone/pull, update, and commit workflows. Some history rewriting operations may fail due to lack of support for bundle repositories. To use, activate the extension and set the ``storage.new-repo-backend`` config option to ``sqlite`` to enable new repositories to use SQLite for storage. )absolute_importN_)nullrevsha1nodeconstantsshort)attr) ancestordagopencodingerror extensions localrepomdiffpycompat registrar requirementsutilverify) repositoryr)hashutil storageutil)zstdstoragesqlite.compressionzstdzlibT)default experimentalsships-with-hg-coresexp-sqlite-001sexp-sqlite-comp-001=zstdsexp-sqlite-comp-001=zlibsexp-sqlite-comp-001=nonesexp-sqlite-shallow-fileszCREATE TABLE delta ( id INTEGER PRIMARY KEY, compression INTEGER NOT NULL, hash BLOB UNIQUE ON CONFLICT ABORT, delta BLOB NOT NULL )zKCREATE TABLE filepath ( id INTEGER PRIMARY KEY, path BLOB NOT NULL )z4CREATE UNIQUE INDEX filepath_path ON filepath (path)acCREATE TABLE fileindex ( id INTEGER PRIMARY KEY, pathid INTEGER REFERENCES filepath(id), revnum INTEGER NOT NULL, p1rev INTEGER NOT NULL, p2rev INTEGER NOT NULL, linkrev INTEGER NOT NULL, flags INTEGER NOT NULL, deltaid INTEGER REFERENCES delta(id), deltabaseid INTEGER REFERENCES fileindex(id), node BLOB NOT NULL )zJCREATE UNIQUE INDEX fileindex_pathrevnum ON fileindex (pathid, revnum)zBCREATE UNIQUE INDEX fileindex_pathnode ON fileindex (pathid, node)aCREATE VIEW filedata AS SELECT fileindex.id AS id, filepath.id AS pathid, filepath.path AS path, fileindex.revnum AS revnum, fileindex.node AS node, fileindex.p1rev AS p1rev, fileindex.p2rev AS p2rev, fileindex.linkrev AS linkrev, fileindex.flags AS flags, fileindex.deltaid AS deltaid, fileindex.deltabaseid AS deltabaseid FROM filepath, fileindex WHERE fileindex.pathid=filepath.idzPRAGMA user_version=%dcCs|djddgt|dt||gt|}g}d}|D]-\} } } | }| tkr3|| } n| t kr:| } n| t krDt | } nt d| | | q"||vr[|||} n|} |t| |} t| tsrt| } | S)z&Resolve a delta chain for a file node.aWITH RECURSIVE deltachain(deltaid, baseid) AS ( SELECT deltaid, deltabaseid FROM fileindex WHERE pathid=? AND node=? UNION ALL SELECT fileindex.deltaid, deltabaseid FROM fileindex, deltachain WHERE fileindex.id=deltachain.baseid AND deltachain.baseid IS NOT NULL AND fileindex.id NOT IN ({stops}) ) SELECT deltachain.baseid, compression, delta FROM deltachain, delta WHERE delta.id=deltachain.deltaid,?)stopsNsunhandled compression type: %d)executeformatjoinlentuplelistkeysCOMPRESSION_ZSTD decompressCOMPRESSION_NONECOMPRESSION_ZLIBzlibSQLiteStoreErrorappendpopreverserpatches isinstancebytes)dbpathidnode revisioncachestopridszstddctxresdeltaslastdeltabaseid deltabaseid compressiondeltabasetextfulltextrG3/usr/lib/python3/dist-packages/hgext/sqlitestore.pyresolvedeltachains:     rIcCsBz |d|||fjWStjy |d|fdYSw)Nz=INSERT INTO delta (compression, hash, delta) VALUES (?, ?, ?)z!SELECT id FROM delta WHERE hash=?r)r& lastrowidsqlite3IntegrityErrorfetchone)r9rChashrDrGrGrH insertdeltasrOc@s eZdZdS)r2N)__name__ __module__ __qualname__rGrGrGrHr2sr2c@sTeZdZeZeZeZeZeZ eZ eZ eZ eZ dS) revisionentryN)rPrQrRribridrevr;p1revp2revp1nodep2nodelinkrevflagsrGrGrGrHrS s rS)slotsc@sheZdZeZeZeZeZeZ eZ eZ eZ eZ eZejddZdS)sqliterevisiondeltaNr)rPrQrRrrTr;rYrZbasenoder\baserevisionsizerevisionrDsidedataprotocol_flagslinknoderGrGrGrHr^sr^)frozenc@s0eZdZejddZejddZejddZdS) sqliteproblemNr_)rPrQrRrrTwarningr r;rGrGrGrHrg(s  rgc@sleZdZdZddZddZddZdd Zd d ZdOddZ ddZ ddZ ddZ ddZ ddZddZddZddZd d!ZdPd"d#Zd$d%Zd&d'ZdQd*d+Zd,d-Zd.d/Zd0d1Zd2d3Zd d(d(ejd fd4d5Zd6d7Z dRd8d9Z (dSd:d;Z dTd=d>Z!d?d@Z"dAdBZ#dCdDZ$dUdEdFZ% ( ( ( ( (dVdGdHZ&dIdJZ'dPdKdLZ( dWdMdNZ)d S)Xsqlitefilestorez2Implements storage for an individual tracked path.cCsxtj|_||_||_d|_i|_i|_i|_t d|_ ||_ |dkr0t j dd|_t |_nd|_d|_|dS)N rr!)level)rnullid_db_path_pathid _revtonode _nodetorev _revisionsr lrucachedict_revisioncache _compenginerZstdCompressor_cctxZstdDecompressor_dctx _refreshindex)selfr9pathrCrGrGrH__init__4s   zsqlitefilestore.__init__c Csi|_i|_i|_t|jd|jf}|sd|_dS|dd|_|jd|jf}t|D]M\}}|\}}}}}} } ||krFt t d|t krNt j } n|j|} |t kr[t j } n|j|} t|||||| | | | d } ||j|<||j|<| |j|<q/dS)Nz$SELECT id FROM filepath WHERE path=?rzgSELECT id, revnum, node, p1rev, p2rev, linkrev, flags FROM fileindex WHERE pathid=? ORDER BY revnum ASCs1sqlite database has inconsistent revision numbers rUrVr;rWrXrYrZr[r\)rprqrrr+rmr&rnro enumerater2rrrrlrS)r{r?irowrUrVr;rWrXr[r\rYrZentryrGrGrHrzOsV    zsqlitefilestore._refreshindexcCs t|jSN)r)rrr{rGrGrH__len__s zsqlitefilestore.__len__cCsttt|jSr)iterrxranger)rrrrGrGrH__iter__szsqlitefilestore.__iter__cCs|tjkrdS||jvSNF)rrlrqr{r;rGrGrHhasnodes  zsqlitefilestore.hasnoderNcCstjt|j||dS)N)startstop)riterrevsr)rr)r{rrrGrGrHrevss zsqlitefilestore.revscCsJ|tjkr tjtjfS||jvrt||jtd|j|}|j|jfSNno node) rrlrrr LookupErrorrnrrYrZ)r{r;rrGrGrHparentss     zsqlitefilestore.parentscCs>|tkrttfS||jvrt||j|j|}|j|jfSr)rrp IndexErrorrrrWrXr{rVrrGrGrH parentrevss   zsqlitefilestore.parentrevscCs6|tjkrtS||jvrt||jtd|j|Sr)rrlrrqr rrnrrrGrGrHrVs   zsqlitefilestore.revcCs*|tkrtjS||jvrt||j|Sr)rrrlrprr{rVrGrGrHr;s   zsqlitefilestore.nodecCst|||jSr)r fileidlookuprnrrGrGrHlookupzsqlitefilestore.lookupcCs4|tkrtS||jvrt||j|j|}|jSr)rrprrrr[rrGrGrHr[s  zsqlitefilestore.linkrevcCs4|tkrdS||jvrt||j|j|jt@Sr)rrprrrr\ FLAG_CENSOREDrrGrGrH iscensoreds  zsqlitefilestore.iscensoredcCs2||}||}t|j||}t|j|Sr)rVr commonancestorsheadsrrmaplistr;)r{node1node2rev1rev2 ancestorsrGrGrHrs  z$sqlitefilestore.commonancestorsheadscCst||j|jSr)r descendantrevsrr)r{rrGrGrH descendantsszsqlitefilestore.descendantscst|dur|durtstjgS|dur|nt}fdd|p#gD}tjjj||d}fdd|DS)Ncsh|]}|qSrGrV.0nrrGrH z(sqlitefilestore.heads..)startrevstoprevscg|]}|qSrG)r;)rrVrrGrH rz)sqlitefilestore.heads..) r)rrlrVrr headrevssubsetrr)r{rrrrrrGrrHheadss zsqlitefilestore.headscCs.||}|jd|j||f}dd|DS)NzZSELECT node FROM filedata WHERE path=? AND (p1rev=? OR p2rev=?) ORDER BY revnum ASCcSsg|]}|dqS)rrG)rrrGrGrHrsz,sqlitefilestore.children..)rVrmr&rn)r{r;rVr?rGrGrHchildrens   zsqlitefilestore.childrencCsN|tkrdS||jvrt||j|}||r t||St||SNr)rrprrenamedr)readrb)r{rVr;rGrGrHsizes   zsqlitefilestore.sizeFTcs|tjtfvr dSt|tr|}|jvr"t|j t d|j vr,j |Sfddj D}|s$z,sqlitefilestore.revision..r>F)rrlrr7intr;rqr rrnrrtrIrmroryrrr\FLAG_MISSING_P1FLAG_MISSING_P2 _checkhash)r{r;raw _verifyhashr=rFrGrrHrbs2        zsqlitefilestore.revisioncOs|j|i|Sr)rb)r{argskwargsrGrGrHrawdata=rzsqlitefilestore.rawdatacCst||Sr)rfiltermetadatarbrrGrGrHr@rzsqlitefilestore.readcCs t||Sr)rfilerevisioncopiedrrGrGrHrCs zsqlitefilestore.renamedcCst||| Sr)rfiledataequivalent)r{r;rFrGrGrHcmpFrzsqlitefilestore.cmpc cs|dvr td|dd|D}|sdS|jdddgt|t|jg|}i}|D]\} } |jd|j| f}|d || <q2t j |||t |j ||||d D]} | VqXdS) N)snodesrslinearNs"unhandled value for nodesorder: %scSsg|] }|tjkr|qSrG)rrlrrGrGrHrWrz1sqlitefilestore.emitrevisions..zISELECT revnum, deltaid FROM fileindex WHERE pathid=? AND node in (%s)r#r$z9SELECT revnum from fileindex WHERE pathid=? AND deltaid=?r) deltaparentfn revisiondataassumehaveparentrevisions deltamodesidedata_helpers) r ProgrammingErrorrmr&r(r)r*rorMr emitrevisionsr^ __getitem__) r{nodes nodesorderrrrrr? deltabasesrVdeltaidrDrGrGrHrIsD    zsqlitefilestore.emitrevisionscCs6|s|dr t||}||||||}||S)Ns ) startswithrpackmeta addrevisionr;)r{filedatameta transactionr[p1p2rVrGrGrHadds  zsqlitefilestore.addc Csx|rttd|du} |pt|||}| r||||||j|} | dur+| S|||||||} ||j|<| S)Ns flags not supported on revisions) r2rrhashrevisionsha1rrqget_addrawrevisionrt) r{rrr[rrr;r\ cachedelta validatehashrVrGrGrHrs    zsqlitefilestore.addrevisionc s^d}|D]&\}} } } } } }}d}|tj@r|tO}|tj@r$td|rH| tjkr7| s7tj} |tO}| tjkrH| sHtj} |tO} | }|t kr} |r}t d}tj| dd}t| |}| d|t||kr}tj| |t@st| |fddr|tO}|| }|jvrj|}|jt@r| tjkr| |_j| |_|jtM_jd j| |j|jf|jt@r| tjkr| |_j| |_ |jtM_jd j| |j|jf|r| |d}q| tjkrt!d | }d}nd}| | f}j"||||| | ||d }|r)||d}q| S) NTrsunhandled revision flags>lllF)rcst|Sr)r)r)xrrGrHsz*sqlitefilestore.addgroup..z0UPDATE fileindex SET p1rev=?, flags=? WHERE id=?z0UPDATE fileindex SET p2rev=?, flags=? WHERE id=?r) storedeltar\)#rREVISION_FLAG_CENSOREDrr2rrlrrrrVrrstructcalcsizer)rrreplacediffheaderr CensoredBaseErrorrnrdeltaiscensoredrrr\rYrqrWrmr&rUrZrXpatchr)r{r@ linkmapperr addrevisioncbduplicaterevisioncbmaybemissingparentsemptyr;rrre deltabaserD wireflagsrc storeflagsbaserevhlenoldlennewlenr[rtextrrVrGrrHaddgroups                 zsqlitefilestore.addgrouprc Cstd|id}t|t||krttd|jd|j |j f d}t |jd||f}|D]f}|\}}} t |j|| iddi|jd} t| } |jd kra|j| } t} n|jd krnt| } t} n|jd krx| } t} ntd |jt| t| kr| } t} t|j| | | }|jd ||fq7t|} t|jt| |}|j |j}|tO}|jd|||j|f|jd|f||j dS)Nscensoredrs5censor tombstone must be no longer than censored dataz(SELECT deltaid FROM fileindex WHERE id=?rzGSELECT id, pathid, node FROM fileindex WHERE deltabaseid=? OR deltaid=?rrrrnone unhandled compression engine: %sz;UPDATE fileindex SET deltaid=?, deltabaseid=NULL WHERE id=?zSUPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL WHERE pathid=? AND node=?zDELETE FROM delta WHERE id=?)!rrr)rr Abortrrmr&rrrUrMr+rIryrsha1digestrurwcompressr-r1r0r/rrOr\rrorzrtclear)r{tr censornode tombstonecensoreddeltaidrowsrrUr:r;rF deltahash deltablobrCrtombstonedeltaidr\rGrGrHcensorrevisionsp          zsqlitefilestore.censorrevisioncs0t|tdfddDjjS)NrcrrGrrrrGrHr{rz1sqlitefilestore.getstrippoint..)rresolvestripinfor)rr[r)r{minlinkrGrrH getstrippointws zsqlitefilestore.getstrippointcCs`t|sdS||\}}|t|krdS||D]}|jd|j||fq|dS)Nz/DELETE FROM fileindex WHERE pathid=? AND node=?)r)r rrmr&ror;rz)r{r rrV_ignoredrGrGrHstrips  zsqlitefilestore.stripcCsgSrrGrrGrGrHfileszsqlitefilestore.filescCsiSrrG)r{ nodeorrev_dfrGrGrHrcszsqlitefilestore.sidedatacs\i}|rg|d<|rg|d<|rt|d<|r&tfddjD|d<|r,d|d<|S)Nsexclusivefiless sharedfilessrevisionscountc3s|] }t|VqdSr)r)rb)rr;rrGrH s z.sqlitefilestore.storageinfo..s trackedsizes storedsize)r)sumrq)r{exclusivefiles sharedfilesrevisionscount trackedsize storedsizedrGrrH storageinfos   zsqlitefilestore.storageinfoc cst|d<|D]6}||}z||Wqty>}zttdt||f|dV|d|WYd}~qd}~wwdS)Nsskipreadsunpacking %s: %s)r r;)setr;rb Exceptionrgrrr)r{staterVr;erGrGrHverifyintegritys  zsqlitefilestore.verifyintegritycCs|dur|dur||\}}|t|||krdSz|j|=Wn ty)Ynwt|r7t|j||t t d|j)Nsintegrity check failed on %s) rrrrtKeyErroriscensoredtextr CensoredNodeErrorrnr2r)r{rFr;rrrGrGrHrs   zsqlitefilestore._checkhashc  Cs|jdur|jd|jf} | j|_|r#|\} } t| tr"|| } n|dus)J|} | tj kr3|} n t | | | |} | tj krGd} n|j| j} t| } |jdkrb|j| }t}n|jdkrot| }t}n|jdkry| }t}ntd|jt|t| kr| }t}t|j|| |}t|}|tj krt}n|j|}|tj krt}n|j|}|jd|j|||||||| f j}t |||||||||d }||j|<||j!|<||j|<|S)Nz&INSERT INTO filepath (path) VALUES (?)rrrrzINSERT INTO fileindex ( pathid, revnum, node, p1rev, p2rev, linkrev, flags, deltaid, deltabaseid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)r~)"rormr&rnrJr7rr;rrlrtextdiffrbrVrrrUrrrrurwrr-r1r0r/r rr)rOrrqrSrp)r{r;rrr[rrrr\r?rrDbaseidrrrCrrVrWrXrUrrGrGrHrs                  zsqlitefilestore._addrawrevision)rN)NN)FT)NrN)NNF)rr)FFFFFr)*rPrQrR__doc__r}rzrrrrrrrVr;rr[rrrrrrrbrrrrrCG_DELTAMODE_STDrrrrr r rrrcrr!rrrGrGrGrHri0sp=         * 7 $ t]    rics0eZdZddZfddZeddZZS)sqliterepositorycCsdSrrGrrGrGrHcancopyXrzsqliterepository.cancopycsN}ttj|i|}|r|Sjdfdd}|d||S)NzBEGIN TRANSACTIONcsjdSr)_dbconncommitrrrGrHcommittransactionesz7sqliterepository.transaction..committransactions sqlitestore)currenttransactionsuperr)rr+r& addfinalize)r{rrcurrentrr- __class__rrHr[s   zsqliterepository.transactioncCsFtj}|jr|jd|kr|jdSt|jd}||f|_|S)Nrrs db.sqlite) threadingcurrent_threadidentrmmakedbsvfsr()r{tidr9rGrGrHr+ls   zsqliterepository._dbconn)rPrQrRr*rpropertyr+ __classcell__rGrGr2rHr)Ws  r)cCsttt|}t|_|dd}|dkr'tD]}||q| n |t kr,nt t d|d|S)z3Construct a database handle for a database at path.zPRAGMA user_versionrs(sqlite database has unrecognized versionzPRAGMA journal_mode=WAL)rKconnectr strfromlocalr8 text_factoryr&rM CREATE_SCHEMAr,CURRENT_SCHEMA_VERSIONr rr)r|r9r? statementrGrGrHr7}s   r7cCsF|ttr |t|t|t|t|tjdSr) r REQUIREMENTrREQUIREMENT_ZSTDREQUIREMENT_ZLIBREQUIREMENT_NONEREQUIREMENT_SHALLOW_FILESrNARROW_REQUIREMENT)ui supportedrGrGrH featuresetups     rJcCs|ddkr |||Sd|vrttdhd}t||}|r0ttddt|d|d<|||}|t|d d }|d krQt sQttd |d kr[|t n|d kre|t n|dkro|t n ttd|| dr|t|S)Nbackendssqlites sharedrepos3shared repositories not supported with SQLite store> narrowfilesrKshallowfilestores6SQLite store does not support repo creation option: %ss, srevlogv1rrrsjstorage.sqlite.compression set to "zstd" but zstandard compression not available to this Mercurial installrrsDunknown compression engine defined in storage.sqlite.compression: %srM)r rrrr(sortedrrBconfigrrCrDrErrF)origrH createoptsknown unsupportedrrCrGrGrHnewreporequirementssP             rTc@seZdZdZddZdS)sqlitefilestoragez)Repository file storage backed by SQLite.cCsd|ddkr |dd}t|jvrd}nt|jvrd}nt|jvr$d}nttdt|j||S)Nr/rrrrsEunable to determine what compression engine to use for SQLite storage) rCrrDrEr rrrir+)r{r|rCrGrGrHfiles     zsqlitefilestorage.fileN)rPrQrRr'rWrGrGrGrHrUs rUcKs4t|vrt|vr|tjtS|d||d|S)z=Produce a type conforming to ``ilocalrepositoryfilestorage``.)rfeaturesNrG)rBrFrr!REPO_FEATURE_SHALLOW_FILE_STORAGErU)rPrrXrrGrGrHmakefilestorages  rZcKs8t|vrt|vrtsttdtS|dd|i|S)NsWrepository uses zstandard compression, which is not available to this Mercurial installrrG)rBrCrr rrr))rPrHrrrGrGrHmakemains r[cOs ||g|Ri|d|_dSr)warnorphanstorefiles)rPr{rrrGrGrH verifierinits r]cCsJtjtttdtttdtttdttt j dt dS)Nsnewreporequirementssmakefilestoragesmakemains__init__) rfeaturesetupfuncsrrJr wrapfunctionrTrZr[rverifierr])rHrGrGrHextsetups racCst|tr d|_dSdSr)r7r)rm)rHreporGrGrH reposetup)s  rcr)Or' __future__rrKrr4r1mercurial.i18nrmercurial.noderrrmercurial.thirdpartyr mercurialr r r r r rrrrrrrmercurial.interfacesr interfaceutilmercurial.utilsrrr __version__ ImportError configtable configitem testedwithrBrCrDrErFr@r/r-r0rrrr?rIrO StorageErrorr2sobjectrS implementerirevisiondeltar^iverifyproblemrg ifilestoragerilocalrepositoryr)r7rJrTilocalrepositoryfilestoragerUrZr[r]rarcrGrGrGrHs &  8         ?A     ,& A