create pypi and npm packages from the generated thrift api files

The new directory structure and build targets enable
to create and publish pypi and npm (jquery, node) packages from the
generated codechecker api thrift IDL's.

- new documentation was added to describe the API change workflow
- the new published packages were added ad external py and js dependecies
- cleanup travis CI thrift deps, docs, make targets
- bump up api package version to v6.25.0, version increment was needed
because publishing the pypi packages with wrong dependencies, with the
newer version it is fixed now. The thrift dependency needs to be a fixed
version (==0.9.2).

.travis.yml

@@ -65,9 +65,6 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update-reset; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install doxygen; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install llvm@9; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install [email protected]; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=/usr/local/Cellar/[email protected]/0.9.3.1/bin:$PATH; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then which thrift; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=/usr/local/opt/llvm@9/bin:$PATH; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PYTHONPATH=~/llvm/tools/scan-build-py/; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=~/llvm/tools/scan-build-py/bin:$PATH; fi

bin/CodeChecker

@@ -39,12 +39,7 @@ def run_codechecker(checker_env, subcommand=None):
                               'lib',
                               'python2.7')
 
-    gen_lib = os.path.join(package_root,
-                           'lib',
-                           'python2.7',
-                           'codechecker_api')
-
-    checker_env['PYTHONPATH'] = common_lib + ':' + gen_lib
+    checker_env['PYTHONPATH'] = common_lib
 
     checker_env['CC_PACKAGE_ROOT'] = \
         os.path.realpath(os.path.join(package_bin, os.pardir))

docs/README.md

@@ -86,7 +86,7 @@ The following commands are used to bootstrap CodeChecker on Ubuntu 18.04 LTS:
 # Install mandatory dependencies for a development and analysis environment.
 # NOTE: clang or clang-tidy can be replaced by any later versions of LLVM/Clang.
 sudo apt-get install clang clang-tidy build-essential curl doxygen gcc-multilib \
-      git python-virtualenv python-dev thrift-compiler
+      git python-virtualenv python-dev
 
 # Check out CodeChecker source code.
 git clone https://github.com/Ericsson/CodeChecker.git --depth 1 ~/codechecker
@@ -156,11 +156,6 @@ pip2 install virtualenv
 # Install the latest clang see: https://formulae.brew.sh/formula/llvm
 brew install llvm@7
 
-# Install Thrift 0.9.3 (Note: the general Thrift version is 0.10.0 on macOS High Sierra 10.13)
-brew unlink thrift
-brew install [email protected]
-# Put the installed `thrift` binary into your PATH; either run `brew link --force [email protected]` or follow the message printed by brew during install
-
 # Fetch source code.
 git clone https://github.com/Ericsson/CodeChecker.git --depth 1 ~/codechecker
 cd ~/codechecker

web/Makefile

@@ -31,11 +31,6 @@ include server/tests/Makefile
 pip_dev_deps:
 	pip install -r $(VENV_DEV_REQ_FILE)
 
-thrift: build_dir
-	if [ -d "$(BUILD_DIR)/thrift" ]; then rm -rf $(BUILD_DIR)/thrift; fi
-	mkdir $(BUILD_DIR)/thrift
-	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CURRENT_DIR)/api/
-
 venv_dev:
 	# Create a virtual environment for development.
 	virtualenv -p python2 venv_dev && \
@@ -52,12 +47,6 @@ package_dir_structure:
 	mkdir -p $(CC_BUILD_DIR)/bin && \
 	mkdir -p $(CC_BUILD_LIB_DIR)
 
-package_thrift: thrift package_dir_structure
-	mkdir -p $(CC_BUILD_API_DIR) && \
-	mkdir -p $(CC_BUILD_GEN_DIR) && \
-	cp -r $(BUILD_DIR)/thrift/v*/gen-py/* $(CC_BUILD_GEN_DIR) && \
-	cp -r $(BUILD_DIR)/thrift/v*/gen-js/* $(CC_BUILD_API_DIR)
-
 gen_docs: build_dir
 	cd $(ROOT) && \
 	doxygen $(ROOT)/Doxyfile.in && \
@@ -91,7 +80,7 @@ package_vendor: package_dir_structure
 # This target should be used from the top level Makefile to build the package
 # together with the analyzer part. This way we will not build plist-to-html
 # multiple times.
-package_web: package_thrift package_userguide package_vendor package_docs
+package_web: package_dir_structure package_userguide package_vendor package_docs
 
 build_plist_to_html:
 	$(MAKE) -C $(ROOT)/tools/plist_to_html build

web/api/Makefile

@@ -1,14 +1,75 @@
-# Check if certain environmental variables are set properly.
-include Makefile.check
+THRIFT_OPTS = -r -o /data
+THRIFT_VERSION = "0.9.2"
+TARGET_PY = --gen py
+TARGET_JS = --gen js:jquery --gen js:node
 
-# Execute the build of each API subfolder.
-SUBDIRS := $(wildcard */.)
+current_dir = $(shell pwd)
+uid = $(shell id -u)
+guid = $(shell id -g)
 
-default: all
+API_DIR := $(current_dir)
 
-all: $(SUBDIRS)
+PY_API_DIR = "$(API_DIR)/py/codechecker_api/codechecker_api"
+PY_API_SHARED_DIR = "$(API_DIR)/py/codechecker_api_shared/codechecker_api_shared"
+JS_API_DIR = "$(API_DIR)/js/codechecker-api-js/lib"
+NODE_JS_API_DIR = "$(API_DIR)/js/codechecker-api-node/lib"
 
-$(SUBDIRS):
-	VERSION=$(dir $@) $(MAKE) -C $@
+default: build
 
-.PHONY: all $(SUBDIRS)
+install_shared_py:
+	cd codechecker_api_shared && python setup.py install
+
+install_api_py:
+	cd codechecker_api && python setup.py install
+
+install_py: install_shared_py install_api_py
+
+target_dirs:
+	mkdir -p $(PY_API_DIR)
+	mkdir -p $(PY_API_SHARED_DIR)
+	mkdir -p $(JS_API_DIR)
+	mkdir -p $(NODE_JS_API_DIR)
+
+build: clean target_dirs
+	docker run \
+	  -u "$(uid):$(guid)" \
+	  -v $(API_DIR):/data \
+	  thrift:$(THRIFT_VERSION) \
+	  /bin/bash -c " \
+		thrift $(THRIFT_OPTS) $(TARGET_PY) $(TARGET_JS) /data/authentication.thrift && \
+		thrift $(THRIFT_OPTS) $(TARGET_PY) $(TARGET_JS) /data/products.thrift && \
+		thrift $(THRIFT_OPTS) $(TARGET_PY) $(TARGET_JS) /data/report_server.thrift && \
+		thrift $(THRIFT_OPTS) $(TARGET_PY) $(TARGET_JS) /data/configuration.thrift"
+
+	cp -r $(API_DIR)/gen-js/* $(JS_API_DIR)
+	cp -r $(API_DIR)/gen-nodejs/* $(NODE_JS_API_DIR)
+
+	cp -r $(API_DIR)/gen-py/codechecker_api_shared/* $(PY_API_SHARED_DIR)
+	rm -rf $(API_DIR)/gen-py/codechecker_api_shared
+
+	cp -r $(API_DIR)/gen-py/* $(PY_API_DIR)
+
+	rm -rf $(API_DIR)/gen-js/
+	rm -rf $(API_DIR)/gen-nodejs/
+	rm -rf $(API_DIR)/gen-py/
+
+publish: build publish_py publish_js
+
+publish_py:
+	cd py/codechecker_api && python setup.py sdist bdist_wheel && twine upload dist/* --verbose
+	cd py/codechecker_api_shared && python setup.py sdist bdist_wheel && twine upload dist/* --verbose
+
+publish_test_py:
+	cd py/codechecker_api && python setup.py sdist bdist_wheel && twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose
+	cd py/codechecker_api_shared && python setup.py sdist bdist_wheel && twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose
+
+publish_js:
+	cd js/codechecker-api-js && npm publish
+	cd js/codechecker-api-node && npm publish
+
+publish_js_dev:
+	cd js/codechecker-api-js && npm publish --tag dev
+	cd js/codechecker-api-node && npm publish --tag dev
+
+clean:
+	git clean -xdf

web/api/README.md

@@ -0,0 +1,57 @@
+# CodeChecker server Thrift API
+
+This directory contains the API IDL files and the generated API stubs for CodeChecker.
+[Apache Thrift](https://thrift.apache.org/) is used to generate the stubs
+for various programming languages. The stubs are published to [pypi](https://pypi.org/) 
+and to [npmjs](https://www.npmjs.com/).
+
+The Thrift compiler is executed inside a [Docker](https://www.docker.com/)
+container so docker needs to be installed to generate the stubs.
+
+## API change workflow:
+
+Assuming the current api version is **6.24.0** and no breaking change was
+introduced.
+
+### 1. Modify the Thrift files.
+
+- Update the versions in the **setup.py** files to **6.24.0-dev1**.
+  - `py/codechecker_api/setup.py`
+  - `py/codechecker_api_shared/setup.py`
+- Update the versions in the **package.json** files to **6.24.0-dev1**.
+  - `js/codechecker-api-js/package.json`
+  - `js/codechecker-api-node/package.json`
+
+- Run the command `make build` to generate the stubs.
+
+- Publish python dev versions:
+  - `make publish_py`
+- Publish javascript dev versions with the `dev` tag:
+  - `make publish_js_dev`
+- In the `setup.py` and `package.json` files use the **6.24.0-dev1** version until the api is accepted.
+
+### 2. Server/client implementation for the API change
+
+- Update the requirements.txt and extend the server with the new
+API functionality with the dev API versions as a dependency.
+- Create a pull request with the changes.
+- Wait while it will be reviewed and approved.
+
+### 3. API change was approved
+
+- After API change approval increment the api versions to **v6.25.0**
+in the setup.py and package.json files.
+- Publish the **v6.25.0** packages to pypi and npm.
+- Update the latest tag in npmjs for the published packages:
+  - `npm dist-tag add [email protected] latest`
+  - `npm dist-tag add [email protected] latest`
+
+### 4. Update the Server/client implementation
+
+- Update the **requirements.txt** files and the **web/server/vendor/Makefile**
+in the pull request to use the new client stubs with version **v6.25.0**.
+- Update the supported api versions to **v6.25** in the server files:
+  - `web/codechecker_web/shared/version.py`
+  - `web/server/www/scripts/version.js`
+
+### 5. Mark the development packages as deprecated on pypi and npmjs

web/api/js/codechecker-api-js/.gitignore

@@ -0,0 +1 @@
+lib