Split up server/client handling parts

.gitignore

@@ -12,13 +12,17 @@ vendor/*
 !vendor/build-logger
 !vendor/plist_to_html
 !vendor/Makefile
+!vendor/Makefile
+webserver/server/vendor/*
+!webserver/server/vendor/Makefile
+
 
 # build-logger build dir
 vendor/build-logger/build/
 
 # doxygen generated files
 gen-docs/
-www/userguide/gen-docs/
+webserver/server/www/userguide/gen-docs/
 
 # thrift generated files
 thrift_api/gen-js/

.travis.yml

@@ -13,16 +13,19 @@ matrix:
       python: "2.7"
       env: CC_MODULE=analyzer
     - os: linux
+      name: "Server/client test cases (sqlite)"
       sudo: required
       dist: xenial
       python: "2.7"
       env: DATABASE=sqlite
     - os: linux
+      name: "Server/client test cases (psycopg2)"
       sudo: required
       dist: xenial
       python: "2.7"
       env: DATABASE=psql_psycopg2
     - os: linux
+      name: "Server/client test cases (pg8000)"
       sudo: required
       dist: xenial
       python: "2.7"
@@ -35,18 +38,21 @@ matrix:
       env: CC_MODULE=analyzer
     - os: osx
       osx_image: xcode9.4
+      name: "Server/client test cases (sqlite)"
       sudo: false
       language: generic
       env: DATABASE=sqlite
     - os: osx
       osx_image: xcode9.4
+      name: "Server/client test cases (psycopg2)"
       sudo: false
       language: generic
       env:
         - DATABASE=psql_psycopg2
         - POSTGRESQL_VERSION=9.6
     - os: osx
       osx_image: xcode9.4
+      name: "Server/client test cases (pg8000)"
       sudo: false
       language: generic
       env:
@@ -114,7 +120,7 @@ script:
           make -C analyzer test_build_logger
         fi
       elif [[ ! -z "$DATABASE" ]]; then
-        make test_matrix_${DATABASE};
+        make -C webserver test_matrix_${DATABASE};
       else
         make test;
       fi

Doxyfile.in

@@ -654,7 +654,7 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories
 # with spaces.
 
-INPUT                  = api docs docs/checker_docs
+INPUT                  = webserver/api docs docs/checker_docs
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is

Makefile

@@ -15,6 +15,9 @@ CC_BUILD_LIB_DIR = $(CC_BUILD_DIR)/lib/python2.7
 CC_BUILD_LIBCC_DIR = $(CC_BUILD_LIB_DIR)/libcodechecker
 CC_BUILD_GEN_DIR = $(CC_BUILD_LIB_DIR)/gencodechecker
 
+CC_WEB_SERVER = $(CURRENT_DIR)/webserver
+CC_SERVER = $(CC_WEB_SERVER)/server/
+CC_CLIENT = $(CC_WEB_SERVER)/client/
 CC_ANALYZER = $(CURRENT_DIR)/analyzer
 
 # Root of the repository.
@@ -23,16 +26,8 @@ ROOT = $(CURRENT_DIR)
 ACTIVATE_RUNTIME_VENV ?= . venv/bin/activate
 ACTIVATE_DEV_VENV ?= . venv_dev/bin/activate
 
-VENV_REQ_FILE ?= requirements_py/requirements.txt
-VENV_DEV_REQ_FILE ?= requirements_py/dev/requirements.txt
-
-OSX_VENV_REQ_FILE ?= requirements_py/osx/requirements.txt
-
 default: package
 
-# Test running related targets.
-include tests/Makefile
-
 gen-docs: build_dir
 	doxygen ./Doxyfile.in && \
 	cp -a ./gen-docs $(BUILD_DIR)/gen-docs
@@ -43,10 +38,10 @@ thrift: build_dir
 
 	mkdir $(BUILD_DIR)/thrift
 
-	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C api/
+	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_WEB_SERVER)/api/
 
 userguide: build_dir
-	$(MAKE) -C www/userguide
+	$(MAKE) -C $(CC_SERVER)/www/userguide
 
 package: clean_package build_dir gen-docs thrift userguide build_plist_to_html build_tu_collector build_vendor
 	# Create directory structure.
@@ -72,9 +67,14 @@ package: clean_package build_dir gen-docs thrift userguide build_plist_to_html b
 
 	# Copy libraries.
 	mkdir -p $(CC_BUILD_LIBCC_DIR) && \
+	mkdir -p $(CC_BUILD_LIB_DIR)/codechecker && \
 	cp -r $(ROOT)/libcodechecker/* $(CC_BUILD_LIBCC_DIR) && \
 	cp -r $(CC_ANALYZER)/cmd/* $(CC_BUILD_LIBCC_DIR)/libhandlers && \
-	cp -r $(CC_ANALYZER)/codechecker $(CC_BUILD_LIB_DIR)
+	cp -r $(CC_ANALYZER)/codechecker/* $(CC_BUILD_LIB_DIR)/codechecker && \
+	cp -r $(CC_SERVER)/cmd/* $(CC_BUILD_LIBCC_DIR)/libhandlers && \
+	cp -r $(CC_SERVER)/codechecker/* $(CC_BUILD_LIB_DIR)/codechecker && \
+	cp -r $(CC_CLIENT)/cmd/* $(CC_BUILD_LIBCC_DIR)/libhandlers && \
+	cp -r $(CC_CLIENT)/codechecker/* $(CC_BUILD_LIB_DIR)/codechecker
 
 	# Copy sub-commands.
 	cp $(ROOT)/bin/* $(CC_BUILD_DIR)/bin && \
@@ -92,20 +92,21 @@ package: clean_package build_dir gen-docs thrift userguide build_plist_to_html b
 	mkdir -p $(CC_BUILD_DIR)/cc_bin && \
 	./scripts/build/create_commands.py -b $(BUILD_DIR) \
 		$(ROOT)/bin \
+		$(CC_SERVER)/bin \
+		$(CC_CLIENT)/bin \
 		$(CC_ANALYZER)/bin
 
 	# Copy web client files.
 	mkdir -p $(CC_BUILD_WEB_DIR) && \
-	cp -r $(ROOT)/www/* $(CC_BUILD_WEB_DIR) && \
+	cp -r $(CC_SERVER)/www/* $(CC_BUILD_WEB_DIR) && \
 	mv $(CC_BUILD_WEB_DIR)/userguide/images/* $(CC_BUILD_WEB_DIR)/images && \
 	rm -rf $(CC_BUILD_WEB_DIR)/userguide/images
 
 	# Rename gen-docs to doc.
 	mv $(CC_BUILD_WEB_DIR)/userguide/gen-docs $(CC_BUILD_WEB_DIR)/userguide/doc
 
 	# Copy database migration files.
-	cp -r $(ROOT)/config_db_migrate $(CC_BUILD_LIB_DIR)/config_migrate && \
-	cp -r $(ROOT)/db_migrate $(CC_BUILD_LIB_DIR)/run_migrate
+	cp -r $(CC_SERVER)/migrations $(CC_BUILD_LIB_DIR)
 
 	# Copy license file.
 	cp $(ROOT)/LICENSE.TXT $(CC_BUILD_DIR)
@@ -151,7 +152,7 @@ else
 endif
 
 build_vendor:
-	$(MAKE) -C $(VENDOR_DIR) build BUILD_DIR=$(CC_BUILD_WEB_PLUGINS_DIR)
+	$(MAKE) -C $(CC_SERVER)/vendor build BUILD_DIR=$(CC_BUILD_WEB_PLUGINS_DIR)
 
 build_dir:
 	mkdir -p $(BUILD_DIR)
@@ -165,21 +166,28 @@ build_tu_collector:
 venv:
 	# Create a virtual environment which can be used to run the build package.
 	virtualenv -p python2 venv && \
-		$(ACTIVATE_RUNTIME_VENV) && pip install -r $(VENV_REQ_FILE)
+		$(ACTIVATE_RUNTIME_VENV) && \
+		pip install -r $(CC_ANALYZER)/requirements.txt && \
+		pip install -r $(CC_WEB_SERVER)/requirements.txt
 
 venv_osx:
 	# Create a virtual environment which can be used to run the build package.
 	virtualenv -p python2 venv && \
-		$(ACTIVATE_RUNTIME_VENV) && pip install -r $(OSX_VENV_REQ_FILE)
+		$(ACTIVATE_RUNTIME_VENV) && \
+		pip install -r $(CC_ANALYZER)/requirements_py/osx/requirements.txt && \
+		pip install -r $(CC_WEB_SERVER)/requirements_py/osx/requirements.txt
 
 clean_venv:
 	rm -rf venv
 
 venv_dev:
 	# Create a virtual environment for development.
 	virtualenv -p python2 venv_dev && \
-		$(ACTIVATE_DEV_VENV) && pip install -r $(VENV_DEV_REQ_FILE) && \
-		pip install -r requirements_py/test/requirements.txt
+		$(ACTIVATE_DEV_VENV) && \
+		pip install -r $(CC_ANALYZER)/requirements_py/dev/requirements.txt && \
+		pip install -r $(CC_ANALYZER)/requirements_py/test/requirements.txt && \
+		pip install -r $(CC_WEB_SERVER)/requirements_py/dev/requirements.txt && \
+		pip install -r $(CC_WEB_SERVER)/requirements_py/test/requirements.txt
 
 clean_venv_dev:
 	rm -rf venv_dev
@@ -193,6 +201,7 @@ clean_package: clean_userguide clean_plist_to_html clean_tu_collector
 
 clean_vendor:
 	$(MAKE) -C $(VENDOR_DIR) clean_vendor
+	$(MAKE) -C $(CC_SERVER)/vendor clean_vendor
 
 clean_userguide:
 	rm -rf www/userguide/gen-docs
@@ -209,3 +218,6 @@ clean_travis:
 
 test_analyzer:
 	$(MAKE) -C $(CC_ANALYZER) test
+
+test_server:
+	$(MAKE) -C $(CC_WEB_SERVER) test

alembic.ini

@@ -1,6 +1,6 @@
 [config_db]
 # path to migration scripts
-script_location = config_db_migrate
+script_location = webserver/server/migrations/config
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
@@ -31,7 +31,7 @@ sqlalchemy.url = postgres://postgres@localhost:5432/config
 
 [run_db]
 # path to migration scripts
-script_location = db_migrate
+script_location = webserver/server/migrations/report
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s

analyzer/requirements.txt

@@ -0,0 +1,2 @@
+portalocker==1.1.0
+psutil==5.2.2

analyzer/requirements_py/osx/requirements.txt

@@ -0,0 +1,3 @@
+portalocker==1.1.0
+psutil==5.2.2
+scan-build==2.0.14

config/package_layout.json

@@ -24,8 +24,8 @@
     "lib_tu_collector": "lib/python2.7/tu_collector",
     "libcodechecker": "lib/python2.7/libcodechecker",
     "gencodechecker": "lib/python2.7/gencodechecker",
-    "run_db_migrate": "lib/python2.7/run_migrate",
-    "config_db_migrate": "lib/python2.7/config_migrate"
+    "run_db_migrate": "lib/python2.7/migrations/report",
+    "config_db_migrate": "lib/python2.7/migrations/config"
   },
   "runtime": {
     "analyzers": {

webserver/.noserc

@@ -0,0 +1,13 @@
+[nosetests]
+
+# increase verbosity level
+verbosity=3
+
+# more detailed error messages on failed asserts
+detailed-errors=1
+
+# stop running tests on first error
+stop=1
+
+# do not capture stdout
+#nocapture=1

webserver/Makefile

@@ -0,0 +1,33 @@
+-include Makefile.local
+
+CURRENT_DIR = $(shell pwd)
+BUILD_DIR = $(ROOT)/build
+
+# Root of the repository.
+ROOT = $(CURRENT_DIR)/..
+
+ACTIVATE_DEV_VENV ?= . venv_dev/bin/activate
+
+VENV_DEV_REQ_FILE ?= requirements_py/dev/requirements.txt
+VENV_TEST_REQ_FILE ?= requirements_py/test/requirements.txt
+
+include tests/Makefile
+
+thrift: build_dir
+	if [ -d "$(BUILD_DIR)/thrift" ]; then rm -rf $(BUILD_DIR)/thrift; fi
+	mkdir $(BUILD_DIR)/thrift
+	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CURRENT_DIR)/api/
+
+venv_dev:
+	# Create a virtual environment for development.
+	virtualenv -p python2 venv_dev && \
+		$(ACTIVATE_DEV_VENV) && pip install -r $(VENV_DEV_REQ_FILE) && \
+		pip install -r $(VENV_TEST_REQ_FILE)
+
+clean_venv_dev:
+	rm -rf venv_dev
+
+build_dir:
+	mkdir -p $(BUILD_DIR)
+
+# TODO: add targets to build server separately.

libcodechecker/libhandlers/cmd.py → webserver/client/cmd/cmd.py

@@ -16,12 +16,13 @@
 import datetime
 import sys
 
+from codechecker.cmdhelper import cmd_line_client
+from codechecker.cmdhelper import product_client
+from codechecker.cmdhelper import source_component_client, token_client
+
 from libcodechecker import logger
 from libcodechecker import output_formatters
 from libcodechecker import util
-from libcodechecker.cmd import cmd_line_client
-from libcodechecker.cmd import product_client
-from libcodechecker.cmd import source_component_client, token_client
 
 
 DEFAULT_FILTER_VALUES = {

libcodechecker/libhandlers/store.py → webserver/client/cmd/store.py

@@ -24,12 +24,13 @@
 from codeCheckerDBAccess_v6.ttypes import StoreLimitKind
 from shared.ttypes import Permission, RequestFailed, ErrorCode
 
+from codechecker.client import client as libclient
+
 from libcodechecker import logger
 from libcodechecker import package_context
 from libcodechecker import host_check
 from libcodechecker import util
 from libcodechecker import plist_parser
-from libcodechecker.libclient import client as libclient
 from libcodechecker.output_formatters import twodim_to_str
 from libcodechecker.util import sizeof_fmt
 from libcodechecker.util import split_product_url

libcodechecker/libclient/authentication_helper.py → webserver/client/codechecker/client/authentication_helper.py

@@ -15,8 +15,8 @@
 
 from Authentication_v6 import codeCheckerAuthentication
 
+from .thrift_call import ThriftClientCall
 from libcodechecker import util
-from libcodechecker.libclient.thrift_call import ThriftClientCall
 from libcodechecker.logger import get_logger
 
 from .credential_manager import SESSION_COOKIE_NAME

libcodechecker/libclient/product_helper.py → webserver/client/codechecker/client/product_helper.py

@@ -16,9 +16,9 @@
 from ProductManagement_v6 import codeCheckerProductService
 
 from libcodechecker import util
-from libcodechecker.libclient.thrift_call import ThriftClientCall
 from libcodechecker.logger import get_logger
 
+from .thrift_call import ThriftClientCall
 from .credential_manager import SESSION_COOKIE_NAME
 
 LOG = get_logger('system')

libcodechecker/libclient/thrift_helper.py → webserver/client/codechecker/client/thrift_helper.py

@@ -16,9 +16,9 @@
 from codeCheckerDBAccess_v6 import codeCheckerDBAccess
 
 from libcodechecker import util
-from libcodechecker.libclient.thrift_call import ThriftClientCall
 from libcodechecker.logger import get_logger
 
+from .thrift_call import ThriftClientCall
 from .credential_manager import SESSION_COOKIE_NAME
 
 LOG = get_logger('system')

libcodechecker/cmd/cmd_line_client.py → webserver/client/codechecker/cmdhelper/cmd_line_client.py

@@ -26,12 +26,12 @@
 
 from codeCheckerDBAccess_v6 import constants, ttypes
 
+from ..client.client import handle_auth, setup_client
+
 from libcodechecker import package_context
 from libcodechecker import logger
 from libcodechecker import suppress_file_handler
 from libcodechecker import plist_parser
-from libcodechecker.libclient.client import handle_auth
-from libcodechecker.libclient.client import setup_client
 from libcodechecker.output_formatters import twodim_to_str
 from libcodechecker.report import Report, get_report_path_hash
 from libcodechecker.source_code_comment_handler import SourceCodeCommentHandler

libcodechecker/cmd/product_client.py → webserver/client/codechecker/cmdhelper/product_client.py

@@ -15,9 +15,10 @@
 
 from ProductManagement_v6.ttypes import *
 
+from ..client.client import setup_product_client
+
 from libcodechecker import logger
 from libcodechecker import database_status
-from libcodechecker.libclient.client import setup_product_client
 from libcodechecker.output_formatters import twodim_to_str
 from libcodechecker.util import split_server_url