The fts3_tokenizer() function returns NULL if the

SQLITE_DBCONFIG_ENABLE_FTS_TOKENIZER setting is disabled, which is is
by default.

ext/fts3/README.tokenizers

@@ -52,8 +52,10 @@
 
   SECURITY: If the fts3 extension is used in an environment where potentially
     malicious users may execute arbitrary SQL (i.e. gears), they should be
-    prevented from invoking the fts3_tokenizer() function, possibly using the
-    authorisation callback.
+    prevented from invoking the fts3_tokenizer() function.  The
+    fts3_tokenizer() function is disabled by default. It is only enabled
+    by SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER. Do not enable it in
+    security sensitive environments.
 
   See "Sample code" below for an example of calling the fts3_tokenizer()
   function from C code.

ext/fts3/fts3_tokenizer.c

@@ -106,7 +106,9 @@ static void fts3TokenizerFunc(
       return;
     }
   }
-  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+  if( fts3TokenizerEnabled(context) ){
+    sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+  }
 }
 
 int sqlite3Fts3IsIdChar(char c){

src/sqlite.h.in

@@ -2086,8 +2086,8 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt>
-** <dd> ^This option is used to enable or disable the two-argument
-** version of the [fts3_tokenizer()] function which is part of the
+** <dd> ^This option is used to enable or disable the
+** [fts3_tokenizer()] function which is part of the
 ** [FTS3] full-text search engine extension.
 ** There should be two additional arguments.
 ** The first argument is an integer which is 0 to disable fts3_tokenizer() or

test/fts3atoken.test

@@ -107,6 +107,7 @@ do_test fts3atoken-2.1 {
 # simple input string via the built-in test function. This is as much
 # to test the test function as the tokenizer implementations.
 #
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
 do_test fts3atoken-3.1 {
   execsql {
     SELECT fts3_tokenizer_test('simple', 'I don''t see how');