Merge tag 'upstream/1.6.54'

Upstream version 1.6.54
EngEhabFahmi · Jun 15, 2015 · dbb0291 · dbb0291
2 parents 5e674d2 + ca081ba
commit dbb0291
Show file tree

Hide file tree

Showing 399 changed files with 143,211 additions and 16,498 deletions.
diff --git a/.gitignore b/.gitignore
@@ -92,3 +92,5 @@ debian/w3af.substvars
 debian/w3af/
 docker/kali-debootstrap
 *.deb
+
+w3af/core/data/parsers/tests/data/huge.html
diff --git a/doc/sphinx/advanced-tips-tricks.rst b/doc/sphinx/advanced-tips-tricks.rst
@@ -0,0 +1,33 @@
+Advanced tips and tricks
+========================
+
+Memory usage and caches
+-----------------------
+
+``w3af`` uses various types of caches to speed-up the scan process, one of the
+most important ones is an in-memory cache which holds the result of parsing an
+HTTP response body. Parsing HTTP response bodies in a CPU intensive process, and
+different ``w3af`` plugins might want to parse the same response so it makes a
+lot of sense to use a cache in this situation.
+
+The `ParserCache <https://github.com/andresriancho/w3af/blob/master/w3af/core/data/parsers/parser_cache.py>`_
+is a LRU cache which holds the items in memory to provide fast access. Some
+advanced users might note that the cache size is set to a constant (10 at the
+time of writing this documentation), which has these side effects:
+
+ * ``w3af`` will consume ~250MB of RAM, most of it allocated by the cache.
+
+ * When run on a system with low free RAM using ~250MB is good, since we want to
+   avoid operating system swapping pages to disk.
+
+ * When run on a system with 8GB of free RAM ``w3af`` could be adding more items
+   to the cache and, increase the cache hit-rate, reduce the CPU usage and
+   overall scan time.
+
+Most users won't even notice all this and use ``w3af`` without this advanced
+tweak, but feel free to adjust the ``CACHE_SIZE = 10`` to any value that fits
+your needs.
+
+In order to debug the cache hit-rate (which should increase with the CACHE_SIZE)
+run ``w3af`` with the ``W3AF_CORE_PROFILING`` environment variable set to ``1``
+and inspect the JSON files at ``/tmp/w3af-*.core``
diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
@@ -78,3 +78,10 @@ GUI documentation
    :maxdepth: 3
 
    gui/index
+
+Advanced tips and tricks
+------------------------
+.. toctree::
+   :maxdepth: 2
+
+   advanced-tips-tricks
diff --git a/doc/sphinx/install.rst b/doc/sphinx/install.rst
@@ -41,7 +41,7 @@ in various Linux distributions, Mac OSX, FreeBSD and OpenBSD.
 
 .. note::
 
-   The platform used for development and running our continuous integration tests
+   The platform used for development is Ubuntu 14.04 and running our continuous integration tests
    is Ubuntu 12.04 LTS.
 
 .. warning::

diff --git a/extras/docker/.dockerignore b/extras/docker/.dockerignore
@@ -1,6 +1,19 @@
 .git
 .dockerignore
 
+### CircleCI specific ###
+circle.yml
+nohup.out
+parsetab.py
+parsetab.pyc
+php-moth
+pico-wavsep
+sqlmap-testenv
+venv
+wivet
+xpresser
+
+
 ### Python ###
 
 # Byte-compiled / optimized / DLL files

diff --git a/extras/docker/scripts/common/docker_helpers.py b/extras/docker/scripts/common/docker_helpers.py
@@ -19,7 +19,7 @@ def start_container(tag):
     if tag is not None:
         docker_run = DOCKER_RUN + ':%s' % tag
     else:
-        docker_run = DOCKER_RUN
+        docker_run = DOCKER_RUN + ':stable'
 
     try:
         container_id = subprocess.check_output(docker_run, shell=True)

diff --git a/extras/docker/scripts/w3af_console_docker b/extras/docker/scripts/w3af_console_docker
@@ -11,7 +11,7 @@ def parse_args():
     parser.add_argument('-d', '--debug', help='Use bash to connect',
                         action='store_true')
     parser.add_argument('-t', '--tag', help='Docker image tag to run. Default'
-                                            'is: latest.',
+                                            'is: stable.',
                         required=False)
     args = parser.parse_args()
     return args

diff --git a/extras/docker/scripts/w3af_gui_docker b/extras/docker/scripts/w3af_gui_docker
@@ -9,7 +9,7 @@ from common.docker_helpers import (check_root, create_volumes, start_container,
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('-t', '--tag', help='Docker image tag to run. Default'
-                                            'is: latest.',
+                                            'is: stable.',
                         required=False)
     args = parser.parse_args()
     return args

diff --git a/profiles/OWASP_TOP10.pw3af b/profiles/OWASP_TOP10.pw3af
@@ -40,14 +40,9 @@ execOneTime = False
 [grep.strange_headers]
 
 [misc-settings]
-maxThreads = 15
 form_fuzzing_mode = tmb
 fuzzed_files_extension = gif
-autoDependencies = True
-demo = False
 fuzzable_headers = 
-showProgressBar = True
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 max_discovery_time = 120
@@ -204,7 +199,6 @@ proxy_port = 8080
 url_parameter = 
 never_404 = 
 headers_file = 
-maxFileSize = 400000
 proxy_address = 
 basic_auth_domain = 
 always_404 = 

diff --git a/profiles/audit_high_risk.pw3af b/profiles/audit_high_risk.pw3af
@@ -14,13 +14,9 @@ listen_address =
 listen_port = 44449
 
 [misc-settings]
-maxThreads = 15
 form_fuzzing_mode = tmb
 fuzzed_files_extension = gif
-autoDependencies = True
-demo = False
 fuzzable_headers = 
-fuzzCookie = False
 fuzz_url_filenames = False
 fuzz_form_files = True
 max_discovery_time = 120
@@ -40,7 +36,6 @@ proxy_port = 8080
 url_parameter = 
 never_404 = 
 basic_auth_domain = 
-maxFileSize = 400000
 always_404 = 
 headers_file = 
 proxy_address = 
@@ -52,7 +47,6 @@ timeout = 0
 user_agent = w3af.org
 basic_auth_user = 
 basic_auth_passwd = 
-byDirectoryAndExtension404 = False
 string_match_404 = 
 cookie_jar_file = 
 ntlm_auth_domain = 

diff --git a/profiles/bruteforce.pw3af b/profiles/bruteforce.pw3af
@@ -10,19 +10,16 @@ verbose = True
 [bruteforce.form_auth]
 
 [misc-settings]
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 fuzz_url_parts = False
 fuzzed_files_extension = gif
 fuzzable_headers = 
 form_fuzzing_mode = tmb
-autoDependencies = True
 stop_on_first_exception = False
 max_discovery_time = 120
 interface = ppp0
 local_ip_address = 10.5.6.18
-demo = False
 non_targets = 
 msf_location = /opt/metasploit3/bin/
 
@@ -41,7 +38,6 @@ ignore_session_cookies = False
 proxy_port = 8080
 proxy_address = 
 user_agent = w3af.org
-maxFileSize = 400000
 max_http_retries = 2
 always_404 = 
 never_404 = 

diff --git a/profiles/empty_profile.pw3af b/profiles/empty_profile.pw3af
@@ -1,11 +1,7 @@
 [misc-settings]
-maxThreads = 15
 form_fuzzing_mode = tmb
 fuzzed_files_extension = gif
-autoDependencies = True
-demo = False
 fuzzable_headers = 
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 max_discovery_time = 120
@@ -27,7 +23,6 @@ name = empty_profile
 proxy_port = 8080
 url_parameter = 
 basic_auth_domain = 
-maxFileSize = 400000
 proxy_address = 
 headers_file = 
 always_404 = 

diff --git a/profiles/fast_scan.pw3af b/profiles/fast_scan.pw3af
@@ -7,15 +7,10 @@
 [grep.blank_body]
 
 [misc-settings]
-maxThreads = 15
 form_fuzzing_mode = tmb
 fuzzed_files_extension = gif
-autoDependencies = True
-demo = False
 max_discovery_time = 120
 fuzzable_headers = 
-showProgressBar = True
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 interface = ppp0
@@ -97,7 +92,6 @@ proxy_port = 8080
 url_parameter = 
 never_404 = 
 basic_auth_domain = 
-maxFileSize = 400000
 proxy_address = 
 headers_file = 
 always_404 = 

diff --git a/profiles/full_audit.pw3af b/profiles/full_audit.pw3af
@@ -213,19 +213,16 @@ reportDavOnly = True
 gen_fingerprint = False
 
 [misc-settings]
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 fuzz_url_parts = False
 fuzzed_files_extension = gif
 fuzzable_headers = 
 form_fuzzing_mode = tmb
-autoDependencies = True
 stop_on_first_exception = False
 max_discovery_time = 120
 interface = ppp0
 local_ip_address = 10.5.6.13
-demo = False
 non_targets = 
 msf_location = /opt/metasploit3/bin/
 fuzz_cookies = False
@@ -245,7 +242,6 @@ ignore_session_cookies = False
 proxy_port = 8080
 proxy_address = 
 user_agent = w3af.org
-maxFileSize = 400000
 max_http_retries = 2
 always_404 = 
 never_404 = 

diff --git a/profiles/full_audit_spider_man.pw3af b/profiles/full_audit_spider_man.pw3af
@@ -160,19 +160,16 @@ only_target_domain = True
 [grep.directory_indexing]
 
 [misc-settings]
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 fuzz_url_parts = False
 fuzzed_files_extension = gif
 fuzzable_headers = 
 form_fuzzing_mode = tmb
-autoDependencies = True
 stop_on_first_exception = False
 max_discovery_time = 120
 interface = ppp0
 local_ip_address = 10.5.6.18
-demo = False
 non_targets = 
 msf_location = /opt/metasploit3/bin/
 
@@ -191,7 +188,6 @@ ignore_session_cookies = False
 proxy_port = 8080
 proxy_address = 
 user_agent = w3af.org
-maxFileSize = 400000
 max_http_retries = 2
 always_404 = 
 never_404 = 

diff --git a/profiles/sitemap.pw3af b/profiles/sitemap.pw3af
@@ -5,12 +5,8 @@ name = sitemap
 [crawl.sitemap_xml]
 
 [misc-settings]
-maxThreads = 15
 fuzzed_files_extension = gif
-autoDependencies = True
-demo = False
 fuzzable_headers = 
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 max_discovery_time = 120
@@ -29,7 +25,6 @@ proxy_port = 8080
 url_parameter = 
 never_404 = 
 headers_file = 
-maxFileSize = 400000
 proxy_address = 
 basic_auth_domain = 
 always_404 = 

diff --git a/profiles/web_infrastructure.pw3af b/profiles/web_infrastructure.pw3af
@@ -6,13 +6,9 @@ name = web_infrastructure
 result_limit = 300
 
 [misc-settings]
-maxThreads = 15
 form_fuzzing_mode = tmb
 fuzzed_files_extension = gif
-autoDependencies = True
-demo = False
 fuzzable_headers = 
-fuzzCookie = False
 fuzz_form_files = True
 fuzz_url_filenames = False
 max_discovery_time = 120
@@ -37,7 +33,6 @@ proxy_port = 8080
 url_parameter = 
 never_404 = 
 headers_file = 
-maxFileSize = 400000
 proxy_address = 
 basic_auth_domain = 
 always_404 = 

diff --git a/scripts/web_spider.w3af b/scripts/web_spider.w3af
@@ -17,7 +17,7 @@ back
 back
 
 target
-set target http://moth/w3af/crawl/web_spider/follow_links/1.html
+set target http://127.0.0.1:8000/crawl/web_spider/test_case_01/
 back
 
 start

diff --git a/w3af/core/controllers/bruteforce/tests/test_bruteforcer.py b/w3af/core/controllers/bruteforce/tests/test_bruteforcer.py
@@ -27,7 +27,7 @@
 from w3af.core.controllers.misc.temp_dir import create_temp_dir
 from w3af.core.controllers.bruteforce.bruteforcer import (password_bruteforcer,
                                                      user_password_bruteforcer)
-from w3af.core.data.parsers.url import URL
+from w3af.core.data.parsers.doc.url import URL
 
 
 class test_password_bruteforcer(unittest.TestCase):

diff --git a/w3af/core/controllers/ci/nosetests_wrapper/constants.py b/w3af/core/controllers/ci/nosetests_wrapper/constants.py
@@ -22,8 +22,7 @@
 NOSE_TIMEOUT = 60 * 8
 
 # Parameters used to collect the list of tests
-NOSE_COLLECT_PARAMS = '--with-id --collect-only --with-doctest'\
-                      ' --doctest-tests --with-xunit --xunit-file=%%s'\
+NOSE_COLLECT_PARAMS = '--with-id --collect-only --with-xunit --xunit-file=%%s'\
                       ' --id-file=%s' % ID_FILE
 NOSE_COLLECT_IGNORE_PARAMS = '--with-id --collect-only --with-xunit'\
                              ' --xunit-file=%%s --id-file=%s' % ID_FILE
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,7 +17,7 @@ back @@
     back
     target
-    set target http://moth/w3af/crawl/web_spider/follow_links/1.html
+    set target http://127.0.0.1:8000/crawl/web_spider/test_case_01/
     back
     start
@@ Expand Down @@