This source file includes following definitions.
- fts3isspace
- fts3MallocZero
- getNextToken
- fts3ReallocOrFree
- getNextString
- getNextNode
- opPrecedence
- insertBinaryOperator
- fts3ExprParse
- sqlite3Fts3ExprParse
- sqlite3Fts3ExprFree
- queryTestTokenizer
- exprToString
- fts3ExprTest
- sqlite3Fts3ExprInitTestInterface
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#ifdef SQLITE_TEST
int sqlite3_fts3_enable_parentheses = 0;
#else
# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
# define sqlite3_fts3_enable_parentheses 1
# else
# define sqlite3_fts3_enable_parentheses 0
# endif
#endif
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
#include "fts3Int.h"
#include <string.h>
#include <assert.h>
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer;
const char **azCol;
int nCol;
int iDefaultCol;
sqlite3_context *pCtx;
int nNest;
};
static int fts3isspace(char c){
return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
}
static void *fts3MallocZero(int nByte){
void *pRet = sqlite3_malloc(nByte);
if( pRet ) memset(pRet, 0, nByte);
return pRet;
}
static int getNextToken(
ParseContext *pParse,
int iCol,
const char *z, int n,
Fts3Expr **ppExpr,
int *pnConsumed
){
sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
int rc;
sqlite3_tokenizer_cursor *pCursor;
Fts3Expr *pRet = 0;
int nConsumed = 0;
rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken, iStart, iEnd, iPosition;
int nByte;
pCursor->pTokenizer = pTokenizer;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
pRet = (Fts3Expr *)fts3MallocZero(nByte);
if( !pRet ){
rc = SQLITE_NOMEM;
}else{
pRet->eType = FTSQUERY_PHRASE;
pRet->pPhrase = (Fts3Phrase *)&pRet[1];
pRet->pPhrase->nToken = 1;
pRet->pPhrase->iColumn = iCol;
pRet->pPhrase->aToken[0].n = nToken;
pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
if( iEnd<n && z[iEnd]=='*' ){
pRet->pPhrase->aToken[0].isPrefix = 1;
iEnd++;
}
if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
pRet->pPhrase->isNot = 1;
}
}
nConsumed = iEnd;
}
pModule->xClose(pCursor);
}
*pnConsumed = nConsumed;
*ppExpr = pRet;
return rc;
}
static void *fts3ReallocOrFree(void *pOrig, int nNew){
void *pRet = sqlite3_realloc(pOrig, nNew);
if( !pRet ){
sqlite3_free(pOrig);
}
return pRet;
}
static int getNextString(
ParseContext *pParse,
const char *zInput, int nInput,
Fts3Expr **ppExpr
){
sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
int rc;
Fts3Expr *p = 0;
sqlite3_tokenizer_cursor *pCursor = 0;
char *zTemp = 0;
int nTemp = 0;
rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
if( rc==SQLITE_OK ){
int ii;
pCursor->pTokenizer = pTokenizer;
for(ii=0; rc==SQLITE_OK; ii++){
const char *zToken;
int nToken, iBegin, iEnd, iPos;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
if( rc==SQLITE_OK ){
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
if( !p || !zTemp ){
goto no_mem;
}
if( ii==0 ){
memset(p, 0, nByte);
p->pPhrase = (Fts3Phrase *)&p[1];
}
p->pPhrase = (Fts3Phrase *)&p[1];
memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
p->pPhrase->nToken = ii+1;
p->pPhrase->aToken[ii].n = nToken;
memcpy(&zTemp[nTemp], zToken, nToken);
nTemp += nToken;
if( iEnd<nInput && zInput[iEnd]=='*' ){
p->pPhrase->aToken[ii].isPrefix = 1;
}else{
p->pPhrase->aToken[ii].isPrefix = 0;
}
}
}
pModule->xClose(pCursor);
pCursor = 0;
}
if( rc==SQLITE_DONE ){
int jj;
char *zNew = NULL;
int nNew = 0;
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
p = fts3ReallocOrFree(p, nByte + nTemp);
if( !p ){
goto no_mem;
}
if( zTemp ){
zNew = &(((char *)p)[nByte]);
memcpy(zNew, zTemp, nTemp);
}else{
memset(p, 0, nByte+nTemp);
}
p->pPhrase = (Fts3Phrase *)&p[1];
for(jj=0; jj<p->pPhrase->nToken; jj++){
p->pPhrase->aToken[jj].z = &zNew[nNew];
nNew += p->pPhrase->aToken[jj].n;
}
sqlite3_free(zTemp);
p->eType = FTSQUERY_PHRASE;
p->pPhrase->iColumn = pParse->iDefaultCol;
rc = SQLITE_OK;
}
*ppExpr = p;
return rc;
no_mem:
if( pCursor ){
pModule->xClose(pCursor);
}
sqlite3_free(zTemp);
sqlite3_free(p);
*ppExpr = 0;
return SQLITE_NOMEM;
}
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
static int getNextNode(
ParseContext *pParse,
const char *z, int n,
Fts3Expr **ppExpr,
int *pnConsumed
){
static const struct Fts3Keyword {
char *z;
unsigned char n;
unsigned char parenOnly;
unsigned char eType;
} aKeyword[] = {
{ "OR" , 2, 0, FTSQUERY_OR },
{ "AND", 3, 1, FTSQUERY_AND },
{ "NOT", 3, 1, FTSQUERY_NOT },
{ "NEAR", 4, 0, FTSQUERY_NEAR }
};
int ii;
int iCol;
int iColLen;
int rc;
Fts3Expr *pRet = 0;
const char *zInput = z;
int nInput = n;
while( nInput>0 && fts3isspace(*zInput) ){
nInput--;
zInput++;
}
if( nInput==0 ){
return SQLITE_DONE;
}
for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
const struct Fts3Keyword *pKey = &aKeyword[ii];
if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
continue;
}
if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
int nKey = pKey->n;
char cNext;
if( pKey->eType==FTSQUERY_NEAR ){
assert( nKey==4 );
if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
nNear = 0;
for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
nNear = nNear * 10 + (zInput[nKey] - '0');
}
}
}
cNext = zInput[nKey];
if( fts3isspace(cNext)
|| cNext=='"' || cNext=='(' || cNext==')' || cNext==0
){
pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
if( !pRet ){
return SQLITE_NOMEM;
}
pRet->eType = pKey->eType;
pRet->nNear = nNear;
*ppExpr = pRet;
*pnConsumed = (int)((zInput - z) + nKey);
return SQLITE_OK;
}
}
}
if( sqlite3_fts3_enable_parentheses ){
if( *zInput=='(' ){
int nConsumed;
pParse->nNest++;
rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
if( rc==SQLITE_OK && !*ppExpr ){
rc = SQLITE_DONE;
}
*pnConsumed = (int)((zInput - z) + 1 + nConsumed);
return rc;
}
if( *zInput==')' ){
pParse->nNest--;
*pnConsumed = (int)((zInput - z) + 1);
return SQLITE_DONE;
}
}
if( *zInput=='"' ){
for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
*pnConsumed = (int)((zInput - z) + ii + 1);
if( ii==nInput ){
return SQLITE_ERROR;
}
return getNextString(pParse, &zInput[1], ii-1, ppExpr);
}
iCol = pParse->iDefaultCol;
iColLen = 0;
for(ii=0; ii<pParse->nCol; ii++){
const char *zStr = pParse->azCol[ii];
int nStr = (int)strlen(zStr);
if( nInput>nStr && zInput[nStr]==':'
&& sqlite3_strnicmp(zStr, zInput, nStr)==0
){
iCol = ii;
iColLen = (int)((zInput - z) + nStr + 1);
break;
}
}
rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
*pnConsumed += iColLen;
return rc;
}
static int opPrecedence(Fts3Expr *p){
assert( p->eType!=FTSQUERY_PHRASE );
if( sqlite3_fts3_enable_parentheses ){
return p->eType;
}else if( p->eType==FTSQUERY_NEAR ){
return 1;
}else if( p->eType==FTSQUERY_OR ){
return 2;
}
assert( p->eType==FTSQUERY_AND );
return 3;
}
static void insertBinaryOperator(
Fts3Expr **ppHead,
Fts3Expr *pPrev,
Fts3Expr *pNew
){
Fts3Expr *pSplit = pPrev;
while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
pSplit = pSplit->pParent;
}
if( pSplit->pParent ){
assert( pSplit->pParent->pRight==pSplit );
pSplit->pParent->pRight = pNew;
pNew->pParent = pSplit->pParent;
}else{
*ppHead = pNew;
}
pNew->pLeft = pSplit;
pSplit->pParent = pNew;
}
static int fts3ExprParse(
ParseContext *pParse,
const char *z, int n,
Fts3Expr **ppExpr,
int *pnConsumed
){
Fts3Expr *pRet = 0;
Fts3Expr *pPrev = 0;
Fts3Expr *pNotBranch = 0;
int nIn = n;
const char *zIn = z;
int rc = SQLITE_OK;
int isRequirePhrase = 1;
while( rc==SQLITE_OK ){
Fts3Expr *p = 0;
int nByte = 0;
rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
if( rc==SQLITE_OK ){
int isPhrase;
if( !sqlite3_fts3_enable_parentheses
&& p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
){
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pNot->eType = FTSQUERY_NOT;
pNot->pRight = p;
if( pNotBranch ){
pNot->pLeft = pNotBranch;
}
pNotBranch = pNot;
p = pPrev;
}else{
int eType = p->eType;
assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
if( !isPhrase && isRequirePhrase ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase && !isRequirePhrase ){
Fts3Expr *pAnd;
assert( pRet && pPrev );
pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
pAnd->eType = FTSQUERY_AND;
insertBinaryOperator(&pRet, pPrev, pAnd);
pPrev = pAnd;
}
if( pPrev && (
(eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
|| (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
)){
sqlite3Fts3ExprFree(p);
rc = SQLITE_ERROR;
goto exprparse_out;
}
if( isPhrase ){
if( pRet ){
assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
pPrev->pRight = p;
p->pParent = pPrev;
}else{
pRet = p;
}
}else{
insertBinaryOperator(&pRet, pPrev, p);
}
isRequirePhrase = !isPhrase;
}
assert( nByte>0 );
}
assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
nIn -= nByte;
zIn += nByte;
pPrev = p;
}
if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
rc = SQLITE_ERROR;
}
if( rc==SQLITE_DONE ){
rc = SQLITE_OK;
if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
if( !pRet ){
rc = SQLITE_ERROR;
}else{
Fts3Expr *pIter = pNotBranch;
while( pIter->pLeft ){
pIter = pIter->pLeft;
}
pIter->pLeft = pRet;
pRet = pNotBranch;
}
}
}
*pnConsumed = n - nIn;
exprparse_out:
if( rc!=SQLITE_OK ){
sqlite3Fts3ExprFree(pRet);
sqlite3Fts3ExprFree(pNotBranch);
pRet = 0;
}
*ppExpr = pRet;
return rc;
}
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer,
char **azCol,
int nCol,
int iDefaultCol,
const char *z, int n,
Fts3Expr **ppExpr
){
int nParsed;
int rc;
ParseContext sParse;
sParse.pTokenizer = pTokenizer;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.nNest = 0;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
}
if( n<0 ){
n = (int)strlen(z);
}
rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
if( rc==SQLITE_OK && sParse.nNest ){
rc = SQLITE_ERROR;
sqlite3Fts3ExprFree(*ppExpr);
*ppExpr = 0;
}
return rc;
}
void sqlite3Fts3ExprFree(Fts3Expr *p){
if( p ){
sqlite3Fts3ExprFree(p->pLeft);
sqlite3Fts3ExprFree(p->pRight);
sqlite3_free(p->aDoclist);
sqlite3_free(p);
}
}
#ifdef SQLITE_TEST
#include <stdio.h>
static int queryTestTokenizer(
sqlite3 *db,
const char *zName,
const sqlite3_tokenizer_module **pp
){
int rc;
sqlite3_stmt *pStmt;
const char zSql[] = "SELECT fts3_tokenizer(?)";
*pp = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
return sqlite3_finalize(pStmt);
}
static char *exprToString(Fts3Expr *pExpr, char *zBuf){
switch( pExpr->eType ){
case FTSQUERY_PHRASE: {
Fts3Phrase *pPhrase = pExpr->pPhrase;
int i;
zBuf = sqlite3_mprintf(
"%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
for(i=0; zBuf && i<pPhrase->nToken; i++){
zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
pPhrase->aToken[i].n, pPhrase->aToken[i].z,
(pPhrase->aToken[i].isPrefix?"+":"")
);
}
return zBuf;
}
case FTSQUERY_NEAR:
zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear);
break;
case FTSQUERY_NOT:
zBuf = sqlite3_mprintf("%zNOT ", zBuf);
break;
case FTSQUERY_AND:
zBuf = sqlite3_mprintf("%zAND ", zBuf);
break;
case FTSQUERY_OR:
zBuf = sqlite3_mprintf("%zOR ", zBuf);
break;
}
if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf);
if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf);
if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf);
if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf);
if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf);
return zBuf;
}
static void fts3ExprTest(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
sqlite3_tokenizer_module const *pModule = 0;
sqlite3_tokenizer *pTokenizer = 0;
int rc;
char **azCol = 0;
const char *zExpr;
int nExpr;
int nCol;
int ii;
Fts3Expr *pExpr;
char *zBuf = 0;
sqlite3 *db = sqlite3_context_db_handle(context);
if( argc<3 ){
sqlite3_result_error(context,
"Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
);
return;
}
rc = queryTestTokenizer(db,
(const char *)sqlite3_value_text(argv[0]), &pModule);
if( rc==SQLITE_NOMEM ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
}else if( !pModule ){
sqlite3_result_error(context, "No such tokenizer module", -1);
goto exprtest_out;
}
rc = pModule->xCreate(0, 0, &pTokenizer);
assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
if( rc==SQLITE_NOMEM ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
}
pTokenizer->pModule = pModule;
zExpr = (const char *)sqlite3_value_text(argv[1]);
nExpr = sqlite3_value_bytes(argv[1]);
nCol = argc-2;
azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
if( !azCol ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
}
for(ii=0; ii<nCol; ii++){
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
}else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
sqlite3_result_error_nomem(context);
}else{
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
sqlite3_free(zBuf);
}
sqlite3Fts3ExprFree(pExpr);
exprtest_out:
if( pModule && pTokenizer ){
rc = pModule->xDestroy(pTokenizer);
}
sqlite3_free(azCol);
}
int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
return sqlite3_create_function(
db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
);
}
#endif
#endif