prepare for enabling phishing code in non-experimental builds

git-svn: trunk@3042

ChangeLog

@@ -1,3 +1,8 @@
+Tue May  1 17:20:53 CEST 2007 (tk)
+----------------------------------
+  * prepare for enabling phishing code in non-experimental builds - apply
+    patches from Edwin
+
 Tue May  1 13:13:56 CEST 2007 (tk)
 ----------------------------------
   * configure: remove libcurl checks

clamd/clamd.c

@@ -302,12 +302,10 @@ int main(int argc, char **argv)
     else
 	logg("Not loading phishing signatures.\n");
 
-#ifdef CL_EXPERIMENTAL
     if(cfgopt(copt,"PhishingScanURLs")->enabled)
 	dboptions |= CL_DB_PHISHING_URLS;
     else
 	logg("Disabling URL based phishing detection.\n");
-#endif
 
     if(cfgopt(copt, "NodalCoreAcceleration")->enabled) {
 #ifdef HAVE_NCORE

clamd/server-th.c

@@ -431,7 +431,6 @@ int acceptloop_th(int *socketds, int nsockets, struct cl_engine *engine, unsigne
 	logg("HTML support disabled.\n");
     }
 
-#ifdef CL_EXPERIMENTAL
     if(cfgopt(copt,"PhishingScanURLs")->enabled) {
 
 	if(cfgopt(copt,"PhishingRestrictedScan")->enabled) {
@@ -457,7 +456,6 @@ int acceptloop_th(int *socketds, int nsockets, struct cl_engine *engine, unsigne
 	    logg("Phishing: Always checking for ssl mismatches\n");
 	}
     }
-#endif /* CL_EXPERIMENTAL */
 
     selfchk = cfgopt(copt, "SelfCheck")->numarg;
     if(!selfchk) {

clamscan/clamscan.c

@@ -300,12 +300,10 @@ void help(void)
     mprintf("\n");
     mprintf("    --no-mail                            Disable mail file support\n");
     mprintf("    --no-phishing-sigs                   Disable signature-based phishing detection\n");
-#ifdef CL_EXPERIMENTAL
     mprintf("    --no-phishing-scan-urls              Disable url-based phishing detection\n");
     mprintf("    --no-phishing-restrictedscan         Enable phishing detection for all domains (might lead to false positives!)\n");
     mprintf("    --phishing-ssl                       Always block SSL mismatches in URLs (phishing module)\n");
     mprintf("    --phishing-cloak                     Always block cloaked URLs (phishing module)\n");
-#endif
     mprintf("    --no-algorithmic                     Disable algorithmic detection\n");
     mprintf("    --no-pe                              Disable PE analysis\n");
     mprintf("    --no-elf                             Disable ELF support\n");

clamscan/clamscan_opt.h

@@ -75,12 +75,10 @@ static struct option clamscan_longopt[] = {
     {"no-mail", 0, 0, 0},
     {"mail-follow-urls", 0, 0, 0},
     {"no-phishing-sigs", 0, 0, 0},
-#ifdef CL_EXPERIMENTAL
     {"no-phishing-scan-urls", 0, 0, 0},
     {"no-phishing-restrictedscan", 0, 0, 0},
     {"phishing-ssl", 0, 0, 0},
     {"phishing-cloak", 0, 0, 0},
-#endif
     {"no-algorithmic", 0, 0, 0},
     {"unzip", 2, 0, 0},
     {"unrar", 2, 0, 0},

clamscan/manager.c

@@ -171,7 +171,6 @@ int scanmanager(const struct optstruct *opt)
     if(!opt_check(opt, "no-phishing-sigs"))
 	dboptions |= CL_DB_PHISHING;
 
-#ifdef CL_EXPERIMENTAL
     if(!opt_check(opt,"no-phishing-scan-urls"))
 	dboptions |= CL_DB_PHISHING_URLS;
     if(!opt_check(opt,"no-phishing-restrictedscan")) {
@@ -184,7 +183,6 @@ int scanmanager(const struct optstruct *opt)
     if(opt_check(opt,"phishing-cloak")) {
 	options |= CL_SCAN_PHISHING_BLOCKCLOAK;
     }
-#endif
 
     if(opt_check(opt, "dev-ac-only"))
 	dboptions |= CL_DB_ACONLY;

libclamav/clamav.h

@@ -75,7 +75,7 @@ extern "C"
 #define CL_DB_PHISHING_URLS 0x8
 
 /* recommended db settings */
-#define CL_DB_STDOPT	    CL_DB_PHISHING
+#define CL_DB_STDOPT	    (CL_DB_PHISHING | CL_DB_PHISHING_URLS)
 
 /* scan options */
 #define CL_SCAN_RAW		    0x0

libclamav/dconf.c

@@ -39,6 +39,12 @@ struct dconf_module {
     uint8_t	state;	    /* default state (on/off) */
 };
 
+#ifdef CL_EXPERIMENTAL
+#define DCONF_ENABLE_EXPERIMENTAL 1
+#else
+#define DCONF_ENABLE_EXPERIMENTAL 0
+#endif
+
 static struct dconf_module modules[] = {
 
     { "PE",	    "PARITE",	    PE_CONF_PARITE,	    1 },
@@ -88,6 +94,9 @@ static struct dconf_module modules[] = {
     { "OTHER",	    "JPEG",	    OTHER_CONF_JPEG,	    1 },
     { "OTHER",	    "CRYPTFF",	    OTHER_CONF_CRYPTFF,	    1 },
 
+    { "PHISHING",   "ENGINE",       PHISHING_CONF_ENGINE,   1 },
+    { "PHISHING",   "ENTCONV",      PHISHING_CONF_ENTCONV,  DCONF_ENABLE_EXPERIMENTAL }, /* exp */
+
     { NULL,	    NULL,	    0,			    0 }
 };
 
@@ -125,6 +134,9 @@ struct cli_dconf *cli_dconf_init(void)
 	} else if(!strcmp(modules[i].mname, "OTHER")) {
 	    if(modules[i].state)
 		dconf->other |= modules[i].bflag;
+	} else if(!strcmp(modules[i].mname, "PHISHING")) {
+	    if(modules[i].state)
+		dconf->phishing |= modules[i].bflag;
 	}
     }
 
@@ -133,7 +145,7 @@ struct cli_dconf *cli_dconf_init(void)
 
 void cli_dconf_print(struct cli_dconf *dconf)
 {
-	uint8_t pe = 0, elf = 0, arch = 0, doc = 0, mail = 0, other = 0;
+	uint8_t pe = 0, elf = 0, arch = 0, doc = 0, mail = 0, other = 0, phishing=0;
 	unsigned int i;
 
 
@@ -196,6 +208,15 @@ void cli_dconf_print(struct cli_dconf *dconf)
 		cli_dbgmsg("   * Submodule %10s:\t%s\n", modules[i].sname, (dconf->other & modules[i].bflag) ? "On" : "** Off **");
 	    else
 		continue;
+	} else if(!strcmp(modules[i].mname, "PHISHING")) {
+	    if(!phishing) {
+		cli_dbgmsg("Module PHISHING %s\n", dconf->phishing ? "On" : "Off");
+		phishing = 1;
+	    }
+	    if(dconf->phishing)
+		cli_dbgmsg("   * Submodule %10s:\t%s\n", modules[i].sname, (dconf->phishing & modules[i].bflag) ? "On" : "** Off **");
+	    else
+		continue;
 	}
     }
 }
@@ -309,6 +330,15 @@ int cli_dconf_load(FILE *fd, struct cl_engine **engine, unsigned int options)
 		break;
 	    }
 	}
+
+	if(!strncmp(buffer, "PHISHING:", 9) && chkflevel(buffer, 2)) {
+	    if(sscanf(buffer + 9, "0x%x", &val) == 1) {
+		dconf->phishing = val;
+	    } else {
+		ret = CL_EMALFDB;
+		break;
+	    }
+	}
     }
 
     if(ret) {

libclamav/dconf.h

@@ -31,6 +31,7 @@ struct cli_dconf {
     uint32_t doc;
     uint32_t mail;
     uint32_t other;
+    uint32_t phishing;
 };
 
 /* PE flags */
@@ -80,6 +81,9 @@ struct cli_dconf {
 #define OTHER_CONF_JPEG	    0x8
 #define OTHER_CONF_CRYPTFF  0x10
 
+/* Phishing flags */
+#define PHISHING_CONF_ENGINE   0x1
+#define PHISHING_CONF_ENTCONV  0x2
 
 struct cli_dconf *cli_dconf_init(void);
 void cli_dconf_print(struct cli_dconf *dconf);

libclamav/filetypes.c

@@ -37,10 +37,8 @@
 #include "matcher-ac.h"
 #include "str.h"
 
-#ifdef CL_EXPERIMENTAL
 #include "htmlnorm.h"
 #include "entconv.h"
-#endif
 
 struct cli_magic_s {
     size_t offset;
@@ -287,8 +285,7 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)
 	    }
 	    cli_ac_freedata(&mdata);
 
-#ifdef CL_EXPERIMENTAL
-	    if(ret != CL_TYPE_HTML_UTF16) {
+	    if((((struct cli_dconf*) engine->dconf)->phishing & PHISHING_CONF_ENTCONV) && ret != CL_TYPE_HTML_UTF16) {
 		    struct entity_conv conv;
 		    const size_t conv_size = 2*bread < 256 ? 256 : 2*bread;
 
@@ -324,7 +321,6 @@ cli_file_t cli_filetype2(int desc, const struct cl_engine *engine)
 		    cli_warnmsg("cli_filetype2: Error initializing entity converter\n");
 		}
 	    }
-#endif /* CL_EXPERIMENTAL */
 	}
     }
 

libclamav/readdb.c

@@ -55,12 +55,10 @@
 #include "lockdb.h"
 #include "readdb.h"
 
-#ifdef CL_EXPERIMENTAL
 #include "phishcheck.h"
 #include "phish_whitelist.h"
 #include "phish_domaincheck_db.h"
 #include "regex_list.h"
-#endif
 
 #if defined(HAVE_READDIR_R_3) || defined(HAVE_READDIR_R_2)
 #include <limits.h>
@@ -275,9 +273,7 @@ int cli_parse_add(struct cli_matcher *root, const char *virname, const char *hex
 
 int cli_initengine(struct cl_engine **engine, unsigned int options)
 {
-#ifdef CL_EXPERIMENTAL
 	int ret;
-#endif
 
 
     if(!*engine) {
@@ -309,11 +305,9 @@ int cli_initengine(struct cl_engine **engine, unsigned int options)
 	}
     }
 
-#ifdef CL_EXPERIMENTAL
-    if(options & CL_DB_PHISHING_URLS)
+    if((options & CL_DB_PHISHING_URLS) && (((struct cli_dconf*) (*engine)->dconf)->phishing & PHISHING_CONF_ENGINE))
 	if((ret = phishing_init(*engine)))
 	    return ret;
-#endif
 
     return CL_SUCCESS;
 }
@@ -418,7 +412,6 @@ static int cli_loaddb(FILE *fd, struct cl_engine **engine, unsigned int *signo,
     return CL_SUCCESS;
 }
 
-#ifdef CL_EXPERIMENTAL
 static int cli_loadwdb(FILE *fd, struct cl_engine **engine, unsigned int options)
 {
 	int ret = 0;
@@ -429,6 +422,9 @@ static int cli_loadwdb(FILE *fd, struct cl_engine **engine, unsigned int options
 	return ret;
     }
 
+    if(!(((struct cli_dconf *) (*engine)->dconf)->phishing & PHISHING_CONF_ENGINE))
+	return CL_SUCCESS;
+
     if(!(*engine)->whitelist_matcher) {
 	if((ret = init_whitelist(*engine))) {
 	    phishing_done(*engine);
@@ -456,6 +452,9 @@ static int cli_loadpdb(FILE *fd, struct cl_engine **engine, unsigned int options
 	return ret;
     }
 
+    if(!(((struct cli_dconf *) (*engine)->dconf)->phishing & PHISHING_CONF_ENGINE))
+	return CL_SUCCESS;
+
     if(!(*engine)->domainlist_matcher) {
 	if((ret = init_domainlist(*engine))) {
 	    phishing_done(*engine);
@@ -472,7 +471,6 @@ static int cli_loadpdb(FILE *fd, struct cl_engine **engine, unsigned int options
 
     return CL_SUCCESS;
 }
-#endif
 
 #define NDB_TOKENS 6
 static int cli_loadndb(FILE *fd, struct cl_engine **engine, unsigned int *signo, unsigned short sdb, unsigned int options)
@@ -1017,7 +1015,6 @@ static int cli_load(const char *filename, struct cl_engine **engine, unsigned in
 	else
 #endif
 	    skipped = 1;
-#ifdef CL_EXPERIMENTAL
     } else if(cli_strbcasestr(filename, ".wdb")) {
 	if(options & CL_DB_PHISHING_URLS)
 	    ret = cli_loadwdb(fd, engine, options);
@@ -1028,7 +1025,6 @@ static int cli_load(const char *filename, struct cl_engine **engine, unsigned in
 	    ret = cli_loadpdb(fd, engine, options);
 	else
 	    skipped = 1;
-#endif
     } else {
 	cli_dbgmsg("cli_load: unknown extension - assuming old database format\n");
 	ret = cli_loaddb(fd, engine, signo, options);
@@ -1108,10 +1104,8 @@ static int cli_loaddbdir_l(const char *dirname, struct cl_engine **engine, unsig
 	     cli_strbcasestr(dent->d_name, ".sdb")  ||
 	     cli_strbcasestr(dent->d_name, ".zmd")  ||
 	     cli_strbcasestr(dent->d_name, ".rmd")  ||
-#ifdef CL_EXPERIMENTAL
 	     cli_strbcasestr(dent->d_name, ".pdb")  ||
 	     cli_strbcasestr(dent->d_name, ".wdb")  ||
-#endif
 	     cli_strbcasestr(dent->d_name, ".ncdb") ||
 	     cli_strbcasestr(dent->d_name, ".inc")  ||
 	     cli_strbcasestr(dent->d_name, ".cvd"))) {
@@ -1265,10 +1259,8 @@ int cl_statinidir(const char *dirname, struct cl_stat *dbstat)
 	    cli_strbcasestr(dent->d_name, ".zmd")  || 
 	    cli_strbcasestr(dent->d_name, ".rmd")  || 
 	    cli_strbcasestr(dent->d_name, ".cfg")  ||
-#ifdef CL_EXPERIMENTAL
 	    cli_strbcasestr(dent->d_name, ".pdb")  ||
 	    cli_strbcasestr(dent->d_name, ".wdb")  ||
-#endif
 	    cli_strbcasestr(dent->d_name, ".ncdb")  ||
 	    cli_strbcasestr(dent->d_name, ".inc")   ||
 	    cli_strbcasestr(dent->d_name, ".cvd"))) {
@@ -1379,10 +1371,8 @@ int cl_statchkdir(const struct cl_stat *dbstat)
 	    cli_strbcasestr(dent->d_name, ".zmd")  || 
 	    cli_strbcasestr(dent->d_name, ".rmd")  || 
 	    cli_strbcasestr(dent->d_name, ".cfg")  ||
-#ifdef CL_EXPERIMENTAL
 	    cli_strbcasestr(dent->d_name, ".pdb")  ||
 	    cli_strbcasestr(dent->d_name, ".wdb")  ||
-#endif
 	    cli_strbcasestr(dent->d_name, ".ncdb")  ||
 	    cli_strbcasestr(dent->d_name, ".inc")   ||
 	    cli_strbcasestr(dent->d_name, ".cvd"))) {
@@ -1553,9 +1543,8 @@ void cl_free(struct cl_engine *engine)
 	free(metah);
     }
 
-#ifdef CL_EXPERIMENTAL
-    phishing_done(engine);
-#endif
+    if(((struct cli_dconf *) engine->dconf)->phishing & PHISHING_CONF_ENGINE)
+	phishing_done(engine);
 
     if(engine->dconf)
 	free(engine->dconf);

shared/cfgparser.c

@@ -49,14 +49,12 @@ struct cfgoption cfg_options[] = {
     {"MailFollowURLs", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},
     {"MailMaxRecursion", OPT_NUM, 64, NULL, 0, OPT_CLAMD},
     {"PhishingSignatures", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
-#ifdef CL_EXPERIMENTAL
     {"PhishingScanURLs",OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     /* these are FP prone options, if default isn't used */
     {"PhishingAlwaysBlockCloak", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},
     {"PhishingAlwaysBlockSSLMismatch", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},
     {"PhishingRestrictedScan", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     /* end of FP prone options */
-#endif
     {"AlgorithmicDetection", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     {"ScanHTML", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},
     {"ScanOLE2", OPT_BOOL, 1, NULL, 0, OPT_CLAMD},

sigtool/sigtool.c

@@ -140,7 +140,7 @@ static int htmlnorm(struct optstruct *opt)
 	return -1;
     }
 
-    html_normalise_fd(fd, ".", NULL);
+    html_normalise_fd(fd, ".", NULL, NULL);
     close(fd);
 
     return 0;