add htmlnorm unit tests and fix jsnorm space normalization

git-svn: trunk@4201

contrib/test/clam.exe.html

@@ -0,0 +1 @@
+<a href="data:application/octet-stream;base64,TVpQAAIAAAAEAA8A//8AALgAAAAhAAAAQAAaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAALtxEEAAM8BQUIvzU1NQsClAMARmrHn5ujEAeA2tUP9mcA4fvjEA6eX/tAnNIbRMzSFiDAoBAnB2FwIeTgwEL9rMEAAAAAAAAAAAAAAAAAAAwBAAAIAQAAAAAAAAAAAAAAAAAADaEAAA9BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAS0VSTkVMMzIuRExMAABFeGl0UHJvY2VzcwBVU0VSMzIuRExMAENMQU1lc3NhZ2VCb3hBAOYQAAAAAAAAPz8/P1BFAABMAQEAYUNhQgAAAAAAAAAA4ACOgQsBAhkABAAAAAYAAAAAAABAEAAAABAAAEAAAAAAAEAAABAAAAACAAABAAAAAAAAAAMACgAAAAAAACAAAAAEAAAAAAAAAgAAAAAAEAAAIAAAAAAQAAAQAAAAAAAAEAAAAAAAAAAAAAAAhBAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAW0NMQU1BVl0AEAAAABAAAAACAAABAAAAAAAAAAAAAAAAAAAAAAAAwA==">t</a>

libclamav/jsparse/js-norm.c

@@ -337,7 +337,11 @@ static inline int buf_outs(const char *s, struct buf *buf)
 	i = buf->pos;
 	while(*s) {
 		while(i < buf_len && *s) {
-			buf->buf[i++] = tolower((unsigned char)(*s++));
+			if(isspace(*s))
+				buf->buf[i++] = ' ';
+			else
+				buf->buf[i++] = tolower((unsigned char)(*s));
+			++s;
 		}
 		if(i == buf_len) {
 			if(write(buf->outfd, buf->buf, buf_len) < 0)
@@ -882,11 +886,15 @@ void cli_js_output(struct parser_state *state, const char *tempdir)
 		/* separate multiple scripts with \n */
 		buf_outc('\n', &buf);
 	}
+	buf_outs("<script>", &buf);
 	state->current = state->global;
 	for(i = 0; i < state->tokens.cnt; i++) {
 		if(state_update_scope(state, &state->tokens.data[i]))
 			lastchar = output_token(&state->tokens.data[i], state->current, &buf, lastchar);
 	}
+	/* add /script if not already there */
+	if(buf.pos < 9 || memcmp(buf.buf + buf.pos - 9, "</script>", 9))
+		buf_outs("</script>", &buf);
 	if(write(buf.outfd, buf.buf, buf.pos) < 0) {
 		cli_dbgmsg(MODULE "I/O error");
 	}
@@ -1512,7 +1520,9 @@ static int yylex(YYSTYPE *lvalp, yyscan_t  scanner)
 				return parseNumber(lvalp, scanner);
 			case SinglelineComment:
 				while(scanner->pos < scanner->insize) {
-					if(in[scanner->pos] == '\n')
+					/* htmlnorm converts \n to space, so
+					 * stop on space too */
+					if(in[scanner->pos] == '\n' || in[scanner->pos] == ' ')
 						break;
 					scanner->pos++;
 				}

test/Makefile.am

@@ -4,7 +4,7 @@ FILES = clam-v2.rar clam-v3.rar clam.cab clam.exe.bz2 clam.exe clam.zip \
 	clam-nsis.exe clam-petite.exe clam-upack.exe clam-wwpack.exe clam.pdf\
 	clam.mail clam.ppt clam.tnef clam.ea05.exe clam.ea06.exe clam.d64.zip\
 	clam.exe.mbox.base64 clam.exe.mbox.uu clam.exe.binhex clam.ole.doc \
-	clam.impl.zip
+	clam.impl.zip clam.exe.html
 
 SPLIT_DIR=$(top_srcdir)/test/.split
 

test/Makefile.in

@@ -197,7 +197,7 @@ FILES = clam-v2.rar clam-v3.rar clam.cab clam.exe.bz2 clam.exe clam.zip \
 	clam-nsis.exe clam-petite.exe clam-upack.exe clam-wwpack.exe clam.pdf\
 	clam.mail clam.ppt clam.tnef clam.ea05.exe clam.ea06.exe clam.d64.zip\
 	clam.exe.mbox.base64 clam.exe.mbox.uu clam.exe.binhex clam.ole.doc \
-	clam.impl.zip
+	clam.impl.zip clam.exe.html
 
 SPLIT_DIR = $(top_srcdir)/test/.split
 EXTRA_DIST = .split

unit_tests/Makefile.am

@@ -15,7 +15,10 @@ check_PROGRAMS = $(programs)
 check_SCRIPTS = $(scripts)
 
 if HAVE_LIBCHECK
-check_clamav_SOURCES = check_clamav.c check_jsnorm.c check_str.c check_regex.c checks.h $(top_builddir)/libclamav/clamav.h check_disasm.c check_uniq.c check_matchers.c
+check_clamav_SOURCES = check_clamav.c checks.h $(top_builddir)/libclamav/clamav.h\
+		       check_jsnorm.c check_str.c check_regex.c\
+		       check_disasm.c check_uniq.c check_matchers.c\
+		       check_htmlnorm.c
 check_clamav_CPPFLAGS = @CHECK_CPPFLAGS@ -DSRCDIR=\"$(abs_srcdir)\"
 check_clamav_LDADD = $(top_builddir)/libclamav/libclamav.la @THREAD_LIBS@ @CHECK_LIBS@
 else
@@ -41,7 +44,7 @@ lcov: $(LCOV_HTML)
 DIRECTORIES=--directory . --directory ../libclamav --directory ../clamd --directory ../freshclam --directory ../sigtool --directory ../clamscan --directory ../clamdscan
 .libs/check_clamav.gcda: $(TESTS)
 	$(LCOV_LCOV) $(DIRECTORIES) --zerocounters
-	@$(MAKE) check VALGRIND=
+	@$(MAKE) check VALGRIND= LIBDUMA=no LIBEFENCE=no
 
 $(LCOV_OUTPUT): .libs/check_clamav.gcda
 	$(LCOV_LCOV) --capture $(DIRECTORIES) --output-file $@

unit_tests/Makefile.in

@@ -51,9 +51,9 @@ CONFIG_HEADER = $(top_builddir)/clamav-config.h
 CONFIG_CLEAN_FILES =
 am__EXEEXT_1 = check_clamav$(EXEEXT)
 am__check_clamav_SOURCES_DIST = check_clamav_skip.c check_clamav.c \
-	check_jsnorm.c check_str.c check_regex.c checks.h \
-	$(top_builddir)/libclamav/clamav.h check_disasm.c check_uniq.c \
-	check_matchers.c
+	checks.h $(top_builddir)/libclamav/clamav.h check_jsnorm.c \
+	check_str.c check_regex.c check_disasm.c check_uniq.c \
+	check_matchers.c check_htmlnorm.c
 @HAVE_LIBCHECK_FALSE@am_check_clamav_OBJECTS =  \
 @HAVE_LIBCHECK_FALSE@	check_clamav-check_clamav_skip.$(OBJEXT)
 @HAVE_LIBCHECK_TRUE@am_check_clamav_OBJECTS =  \
@@ -63,7 +63,8 @@ am__check_clamav_SOURCES_DIST = check_clamav_skip.c check_clamav.c \
 @HAVE_LIBCHECK_TRUE@	check_clamav-check_regex.$(OBJEXT) \
 @HAVE_LIBCHECK_TRUE@	check_clamav-check_disasm.$(OBJEXT) \
 @HAVE_LIBCHECK_TRUE@	check_clamav-check_uniq.$(OBJEXT) \
-@HAVE_LIBCHECK_TRUE@	check_clamav-check_matchers.$(OBJEXT)
+@HAVE_LIBCHECK_TRUE@	check_clamav-check_matchers.$(OBJEXT) \
+@HAVE_LIBCHECK_TRUE@	check_clamav-check_htmlnorm.$(OBJEXT)
 check_clamav_OBJECTS = $(am_check_clamav_OBJECTS)
 @HAVE_LIBCHECK_TRUE@check_clamav_DEPENDENCIES =  \
 @HAVE_LIBCHECK_TRUE@	$(top_builddir)/libclamav/libclamav.la
@@ -234,7 +235,11 @@ scripts = check_clamd.sh check_freshclam.sh check_sigtool.sh check_clamscan.sh\
 TESTS_ENVIRONMENT = export abs_srcdir=$(abs_srcdir) AWK=$(AWK);
 check_SCRIPTS = $(scripts)
 @HAVE_LIBCHECK_FALSE@check_clamav_SOURCES = check_clamav_skip.c
-@HAVE_LIBCHECK_TRUE@check_clamav_SOURCES = check_clamav.c check_jsnorm.c check_str.c check_regex.c checks.h $(top_builddir)/libclamav/clamav.h check_disasm.c check_uniq.c check_matchers.c
+@HAVE_LIBCHECK_TRUE@check_clamav_SOURCES = check_clamav.c checks.h $(top_builddir)/libclamav/clamav.h\
+@HAVE_LIBCHECK_TRUE@		       check_jsnorm.c check_str.c check_regex.c\
+@HAVE_LIBCHECK_TRUE@		       check_disasm.c check_uniq.c check_matchers.c\
+@HAVE_LIBCHECK_TRUE@		       check_htmlnorm.c
+
 @HAVE_LIBCHECK_TRUE@check_clamav_CPPFLAGS = @CHECK_CPPFLAGS@ -DSRCDIR=\"$(abs_srcdir)\"
 @HAVE_LIBCHECK_TRUE@check_clamav_LDADD = $(top_builddir)/libclamav/libclamav.la @THREAD_LIBS@ @CHECK_LIBS@
 CLEANFILES = lcov.out *.gcno *.gcda *.log $(FILES) test-stderr.log clamscan.log valgrind-*.log duma.log duma2.log
@@ -298,6 +303,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_clamav.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_clamav_skip.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_disasm.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_htmlnorm.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_jsnorm.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_matchers.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/check_clamav-check_regex.Po@am__quote@
@@ -437,6 +443,20 @@ check_clamav-check_matchers.obj: check_matchers.c
 @AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCC_FALSE@	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(check_clamav_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o check_clamav-check_matchers.obj `if test -f 'check_matchers.c'; then $(CYGPATH_W) 'check_matchers.c'; else $(CYGPATH_W) '$(srcdir)/check_matchers.c'; fi`
 
+check_clamav-check_htmlnorm.o: check_htmlnorm.c
+@am__fastdepCC_TRUE@	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(check_clamav_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT check_clamav-check_htmlnorm.o -MD -MP -MF $(DEPDIR)/check_clamav-check_htmlnorm.Tpo -c -o check_clamav-check_htmlnorm.o `test -f 'check_htmlnorm.c' || echo '$(srcdir)/'`check_htmlnorm.c
+@am__fastdepCC_TRUE@	mv -f $(DEPDIR)/check_clamav-check_htmlnorm.Tpo $(DEPDIR)/check_clamav-check_htmlnorm.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	source='check_htmlnorm.c' object='check_clamav-check_htmlnorm.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(check_clamav_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o check_clamav-check_htmlnorm.o `test -f 'check_htmlnorm.c' || echo '$(srcdir)/'`check_htmlnorm.c
+
+check_clamav-check_htmlnorm.obj: check_htmlnorm.c
+@am__fastdepCC_TRUE@	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(check_clamav_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT check_clamav-check_htmlnorm.obj -MD -MP -MF $(DEPDIR)/check_clamav-check_htmlnorm.Tpo -c -o check_clamav-check_htmlnorm.obj `if test -f 'check_htmlnorm.c'; then $(CYGPATH_W) 'check_htmlnorm.c'; else $(CYGPATH_W) '$(srcdir)/check_htmlnorm.c'; fi`
+@am__fastdepCC_TRUE@	mv -f $(DEPDIR)/check_clamav-check_htmlnorm.Tpo $(DEPDIR)/check_clamav-check_htmlnorm.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	source='check_htmlnorm.c' object='check_clamav-check_htmlnorm.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(check_clamav_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o check_clamav-check_htmlnorm.obj `if test -f 'check_htmlnorm.c'; then $(CYGPATH_W) 'check_htmlnorm.c'; else $(CYGPATH_W) '$(srcdir)/check_htmlnorm.c'; fi`
+
 mostlyclean-libtool:
 	-rm -f *.lo
 
@@ -710,7 +730,7 @@ $(top_builddir)/test/clam.exe:
 @ENABLE_COVERAGE_TRUE@lcov: $(LCOV_HTML)
 @[email protected]/check_clamav.gcda: $(TESTS)
 @ENABLE_COVERAGE_TRUE@	$(LCOV_LCOV) $(DIRECTORIES) --zerocounters
-@ENABLE_COVERAGE_TRUE@	@$(MAKE) check VALGRIND=
+@ENABLE_COVERAGE_TRUE@	@$(MAKE) check VALGRIND= LIBDUMA=no LIBEFENCE=no
 
 @ENABLE_COVERAGE_TRUE@$(LCOV_OUTPUT): .libs/check_clamav.gcda
 @ENABLE_COVERAGE_TRUE@	$(LCOV_LCOV) --capture $(DIRECTORIES) --output-file $@