Skip to content

Commit

Permalink
Enhancements to deserialize: (1) Add the SQLITE_FCNTL_SIZE_LIMIT file…
Browse files Browse the repository at this point in the history 
… control

to set a maximum size for an in-memory database, defaulting to 
SQLITE_MEMDB_DEFAULT_MAXSIZE or 1GiB.  (2) Honor the SQLITE_DESERIALIZE_READONLY
flag. (3) Enhance the TCL interface to support -maxsize N and -readonly BOOLEAN.
(4) Add the --maxsize option to the ".open" command and on the command-line for
the CLI.
  • Loading branch information
D. Richard Hipp committed Jan 22, 2019
1 parent 2da9d73 commit 7a68967
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 19 deletions.
38 changes: 34 additions & 4 deletions src/memdb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,19 @@ typedef struct MemFile MemFile;
struct MemFile {
sqlite3_file base; /* IO methods */
sqlite3_int64 sz; /* Size of the file */
sqlite3_int64 szMax; /* Space allocated to aData */
sqlite3_int64 szAlloc; /* Space allocated to aData */
sqlite3_int64 szMax; /* Maximum allowed size of the file */
unsigned char *aData; /* content of the file */
int nMmap; /* Number of memory mapped pages */
unsigned mFlags; /* Flags */
int eLock; /* Most recent lock against this file */
};

/* The default maximum size of an in-memory database */
#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
#endif

/*
** Methods for MemFile
*/
Expand Down Expand Up @@ -160,10 +166,15 @@ static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){
if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){
return SQLITE_FULL;
}
if( newSz>p->szMax ){
return SQLITE_FULL;
}
newSz *= 2;
if( newSz>p->szMax ) newSz = p->szMax;
pNew = sqlite3_realloc64(p->aData, newSz);
if( pNew==0 ) return SQLITE_NOMEM;
p->aData = pNew;
p->szMax = newSz;
p->szAlloc = newSz;
return SQLITE_OK;
}

Expand All @@ -177,10 +188,11 @@ static int memdbWrite(
sqlite_int64 iOfst
){
MemFile *p = (MemFile *)pFile;
if( p->mFlags & SQLITE_DESERIALIZE_READONLY ) return SQLITE_READONLY;
if( iOfst+iAmt>p->sz ){
int rc;
if( iOfst+iAmt>p->szMax
&& (rc = memdbEnlarge(p, (iOfst+iAmt)*2))!=SQLITE_OK
if( iOfst+iAmt>p->szAlloc
&& (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK
){
return rc;
}
Expand Down Expand Up @@ -250,6 +262,19 @@ static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){
*(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz);
rc = SQLITE_OK;
}
if( op==SQLITE_FCNTL_SIZE_LIMIT ){
sqlite3_int64 iLimit = *(sqlite3_int64*)pArg;
if( iLimit<p->sz ){
if( iLimit<0 ){
iLimit = p->szMax;
}else{
iLimit = p->sz;
}
}
p->szMax = iLimit;
*(sqlite3_int64*)pArg = iLimit;
rc = SQLITE_OK;
}
return rc;
}

Expand Down Expand Up @@ -311,6 +336,7 @@ static int memdbOpen(
assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */
*pOutFlags = flags | SQLITE_OPEN_MEMORY;
p->base.pMethods = &memdb_io_methods;
p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
return SQLITE_OK;
}

Expand Down Expand Up @@ -560,7 +586,11 @@ int sqlite3_deserialize(
}else{
p->aData = pData;
p->sz = szDb;
p->szAlloc = szBuf;
p->szMax = szBuf;
if( p->szMax<SQLITE_MEMDB_DEFAULT_MAXSIZE ){
p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
}
p->mFlags = mFlags;
rc = SQLITE_OK;
}
Expand Down
18 changes: 18 additions & 0 deletions src/shell.c.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1025,6 +1025,7 @@ struct ShellState {
int showHeader; /* True to show column names in List or Column mode */
int nCheck; /* Number of ".check" commands run */
unsigned shellFlgs; /* Various flags */
sqlite3_int64 szMax; /* --maxsize argument to .open */
char *zDestTable; /* Name of destination table when MODE_Insert */
char *zTempFile; /* Temporary file that might need deleting */
char zTestcase[30]; /* Name of current test case */
Expand Down Expand Up @@ -3449,6 +3450,7 @@ static const char *(azHelp[]) = {
#ifdef SQLITE_ENABLE_DESERIALIZE
" --deserialize Load into memory useing sqlite3_deserialize()",
" --hexdb Load the output of \"dbtotxt\" as an in-memory database",
" --maxsize N Maximum size for --hexdb or --deserialized database",
#endif
" --new Initialize FILE to an empty database",
" --readonly Open FILE readonly",
Expand Down Expand Up @@ -3927,6 +3929,9 @@ static void open_db(ShellState *p, int openFlags){
if( rc ){
utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
}
if( p->szMax>0 ){
sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
}
}
#endif
}
Expand Down Expand Up @@ -6841,6 +6846,7 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(p->zFreeOnClose);
p->zFreeOnClose = 0;
p->openMode = SHELL_OPEN_UNSPEC;
p->szMax = 0;
/* Check for command-line arguments */
for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
const char *z = azArg[iName];
Expand All @@ -6859,6 +6865,8 @@ static int do_meta_command(char *zLine, ShellState *p){
p->openMode = SHELL_OPEN_DESERIALIZE;
}else if( optionMatch(z, "hexdb") ){
p->openMode = SHELL_OPEN_HEXDB;
}else if( optionMatch(z, "maxsize") && iName+1<nArg ){
p->szMax = integerValue(azArg[++iName]);
#endif /* SQLITE_ENABLE_DESERIALIZE */
}else if( z[0]=='-' ){
utf8_printf(stderr, "unknown option: %s\n", z);
Expand Down Expand Up @@ -8549,6 +8557,9 @@ static const char zOptions[] =
" -column set output mode to 'column'\n"
" -cmd COMMAND run \"COMMAND\" before reading stdin\n"
" -csv set output mode to 'csv'\n"
#if defined(SQLITE_ENABLE_DESERIALIZE)
" -deserialize open the database using sqlite3_deserialize()\n"
#endif
" -echo print commands before execution\n"
" -init FILENAME read/process named file\n"
" -[no]header turn headers on or off\n"
Expand All @@ -8561,6 +8572,9 @@ static const char zOptions[] =
" -line set output mode to 'line'\n"
" -list set output mode to 'list'\n"
" -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n"
#if defined(SQLITE_ENABLE_DESERIALIZE)
" -maxsize N maximum size for a --deserialize database\n"
#endif
" -mmap N default mmap size set to N\n"
#ifdef SQLITE_ENABLE_MULTIPLEX
" -multiplex enable the multiplexor VFS\n"
Expand Down Expand Up @@ -8871,6 +8885,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
#ifdef SQLITE_ENABLE_DESERIALIZE
}else if( strcmp(z,"-deserialize")==0 ){
data.openMode = SHELL_OPEN_DESERIALIZE;
}else if( strcmp(z,"-maxsize")==0 && i+1<argc ){
data.szMax = integerValue(argv[++i]);
#endif
}else if( strcmp(z,"-readonly")==0 ){
data.openMode = SHELL_OPEN_READONLY;
Expand Down Expand Up @@ -8972,6 +8988,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
#ifdef SQLITE_ENABLE_DESERIALIZE
}else if( strcmp(z,"-deserialize")==0 ){
data.openMode = SHELL_OPEN_DESERIALIZE;
}else if( strcmp(z,"-maxsize")==0 && i+1<argc ){
data.szMax = integerValue(argv[++i]);
#endif
}else if( strcmp(z,"-readonly")==0 ){
data.openMode = SHELL_OPEN_READONLY;
Expand Down
10 changes: 10 additions & 0 deletions src/sqlite.h.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,15 @@ struct sqlite3_io_methods {
** file space based on this hint in order to help writes to the database
** file run faster.
**
** <li>[[SQLITE_FCNTL_SIZE_LIMIT]]
** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that
** implements [sqlite3_deserialize()] to set an upper bound on the size
** of the in-memory database. The argument is a pointer to a [sqlite3_int64].
** If the integer pointed to is negative, then it is filled in with the
** current limit. Otherwise the limit is set to the larger of the value
** of the integer pointed to and the current database size. The integer
** pointed to is set to the new limit.
**
** <li>[[SQLITE_FCNTL_CHUNK_SIZE]]
** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS
** extends and truncates the database file in chunks of a size specified
Expand Down Expand Up @@ -1131,6 +1140,7 @@ struct sqlite3_io_methods {
#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33
#define SQLITE_FCNTL_LOCK_TIMEOUT 34
#define SQLITE_FCNTL_DATA_VERSION 35
#define SQLITE_FCNTL_SIZE_LIMIT 36

/* deprecated names */
#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE
Expand Down
55 changes: 41 additions & 14 deletions src/tclsqlite.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2418,7 +2418,7 @@ static int SQLITE_TCLAPI DbObjCmd(
}

/*
** $db deserialize ?DATABASE? VALUE
** $db deserialize ?-maxsize N? ?-readonly BOOL? ?DATABASE? VALUE
**
** Reopen DATABASE (default "main") using the content in $VALUE
*/
Expand All @@ -2428,38 +2428,65 @@ static int SQLITE_TCLAPI DbObjCmd(
(char*)0);
rc = TCL_ERROR;
#else
const char *zSchema;
Tcl_Obj *pValue;
const char *zSchema = 0;
Tcl_Obj *pValue = 0;
unsigned char *pBA;
unsigned char *pData;
int len, xrc;

if( objc==3 ){
zSchema = 0;
pValue = objv[2];
}else if( objc==4 ){
zSchema = Tcl_GetString(objv[2]);
pValue = objv[3];
}else{
sqlite3_int64 mxSize = 0;
int i;
int isReadonly = 0;


if( objc<3 ){
Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? VALUE");
rc = TCL_ERROR;
break;
}
for(i=2; i<objc-1; i++){
const char *z = Tcl_GetString(objv[i]);
if( strcmp(z,"-maxsize")==0 && i<objc-2 ){
rc = Tcl_GetWideIntFromObj(interp, objv[++i], &mxSize);
if( rc ) goto deserialize_error;
continue;
}
if( strcmp(z,"-readonly")==0 && i<objc-2 ){
rc = Tcl_GetBooleanFromObj(interp, objv[++i], &isReadonly);
if( rc ) goto deserialize_error;
continue;
}
if( zSchema==0 && i==objc-2 && z[0]!='-' ){
zSchema = z;
continue;
}
Tcl_AppendResult(interp, "unknown option: ", z, (char*)0);
rc = TCL_ERROR;
goto deserialize_error;
}
pValue = objv[objc-1];
pBA = Tcl_GetByteArrayFromObj(pValue, &len);
pData = sqlite3_malloc64( len );
if( pData==0 && len>0 ){
Tcl_AppendResult(interp, "out of memory", (char*)0);
rc = TCL_ERROR;
}else{
int flags;
if( len>0 ) memcpy(pData, pBA, len);
xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len,
SQLITE_DESERIALIZE_FREEONCLOSE |
SQLITE_DESERIALIZE_RESIZEABLE);
if( isReadonly ){
flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY;
}else{
flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE;
}
xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len, flags);
if( xrc ){
Tcl_AppendResult(interp, "unable to set MEMDB content", (char*)0);
rc = TCL_ERROR;
}
if( mxSize>0 ){
sqlite3_file_control(pDb->db, zSchema,SQLITE_FCNTL_SIZE_LIMIT,&mxSize);
}
}
deserialize_error:
#endif
break;
}
Expand Down
27 changes: 26 additions & 1 deletion test/memdb1.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,31 @@ do_execsql_test 140 {
PRAGMA page_count;
} {2}

do_test 150 {
catch {db deserialize -unknown 1 $db1} msg
set msg
} {unknown option: -unknown}
do_test 151 {
db deserialize -readonly 1 $db1
db eval {SELECT * FROM t1}
} {1 2}
do_test 152 {
catchsql {INSERT INTO t1 VALUES(3,4);}
} {1 {attempt to write a readonly database}}

breakpoint
do_test 160 {
db deserialize -maxsize 32768 $db1
db eval {SELECT * FROM t1}
} {1 2}
do_test 161 {
db eval {INSERT INTO t1 VALUES(3,4); SELECT * FROM t1}
} {1 2 3 4}
do_test 162 {
catchsql {INSERT INTO t1 VALUES(5,randomblob(100000))}
} {1 {database or disk is full}}


# Build a largish on-disk database and serialize it. Verify that the
# serialization works.
#
Expand Down Expand Up @@ -154,7 +179,7 @@ do_test 600 {
do_test 610 {
set rc [catch {db deserialize a b c} msg]
lappend rc $msg
} {1 {wrong # args: should be "db deserialize ?DATABASE? VALUE"}}
} {1 {unknown option: a}}
do_test 620 {
set rc [catch {db serialize a b} msg]
lappend rc $msg
Expand Down

0 comments on commit 7a68967

Please sign in to comment.