add web-based kitti viewer

Dockerfile

@@ -74,8 +74,8 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \
 
 RUN PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \
     $PIP_INSTALL \
-        shapely fire pybind11 pyqtgraph tensorboardX protobuf \
-        pyopengl pyqt5 matplotlib scikit-image numba pillow
+        shapely fire pybind11 tensorboardX protobuf \
+        scikit-image numba pillow
 
 WORKDIR /root
 RUN wget https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz

README.md

@@ -32,7 +32,7 @@ cd ./second.pytorch/second
 It is recommend to use Anaconda package manager.
 
 ```bash
-pip install shapely fire pybind11 pyqtgraph tensorboardX protobuf
+pip install shapely fire pybind11 tensorboardX protobuf scikit-image numba pillow
 ```
 
 If you don't have Anaconda:
@@ -204,7 +204,23 @@ python ./pytorch/train.py evaluate --config_path=./configs/car.config --model_di
 
 Currently there is a problem that training and evaluating in docker is very slow.
 
-## Try Kitti Viewer (Unstable)
+## Try Kitti Viewer Web
+
+1. run ```python ./kittiviewer/backend.py --port=xxxx``` in your server/local.
+
+2. run ```cd ./kittiviewer/frontend && python -m http.server``` to launch a local web server.
+
+3. open your browser and enter http://127.0.0.1:8000.
+
+4. input backend (http://your_server:your_backend_port)
+
+5. input root path, info path and det path (optional)
+
+6. click load, loadDet (optional), then click plot.
+
+![GuidePic](https://raw.githubusercontent.com/traveller59/second.pytorch/master/images/viewerweb.png)
+
+## Try Kitti Viewer (Deprecated)
 
 You should use kitti viewer based on pyqt and pyqtgraph to check data before training.
 

second/core/box_np_ops.py

@@ -859,4 +859,9 @@ def assign_label_to_voxel(gt_boxes, coors, voxel_size, coors_range):
         axis=2)
     gt_surfaces = corner_to_surfaces_3d(gt_box_corners)
     ret = points_in_convex_polygon_3d_jit(voxel_centers, gt_surfaces)
-    return np.any(ret, axis=1).astype(np.int64)
+    return np.any(ret, axis=1).astype(np.int64)
+
+def change_box3d_center_(box3d, src, dst):
+    dst = np.array(dst, dtype=box3d.dtype)
+    src = np.array(src, dtype=box3d.dtype)
+    box3d[..., :3] += box3d[..., 3:6] * (dst - src)

second/kittiviewer/backend.py

@@ -0,0 +1,183 @@
+import fire 
+import os
+import numpy as np
+import base64
+import json
+import time
+from flask import Flask, jsonify, request
+from flask_cors import CORS
+
+import pickle
+import sys
+from functools import partial
+from pathlib import Path
+
+import second.core.box_np_ops as box_np_ops
+import second.core.preprocess as prep
+from second.core.anchor_generator import AnchorGenerator
+from second.core.box_coders import GroundBox3dCoder
+from second.core.point_cloud.point_cloud_ops import points_to_voxel
+from second.core.region_similarity import (
+    DistanceSimilarity, NearestIouSimilarity, RotateIouSimilarity)
+from second.core.sample_ops import (
+    sample_from_database_v2, sample_from_database_v3, sample_from_database_v4,
+    DataBaseSamplerV2)
+from second.core.target_assigner import TargetAssigner
+from second.data import kitti_common as kitti
+from second.protos import pipeline_pb2
+from second.utils.eval import get_coco_eval_result, get_official_eval_result
+from second.pytorch.inference import TorchInferenceContext
+from second.utils.progress_bar import list_bar
+
+app = Flask("second")
+CORS(app)
+
+class SecondBackend:
+    def __init__(self):
+        self.root_path = None 
+        self.info_path = None 
+        self.kitti_infos = None
+        self.image_idxes = None
+        self.dt_annos = None
+
+
+BACKEND = SecondBackend()
+
+def error_response(msg):
+    response = {}
+    response["status"] = "error"
+    response["message"] = "[ERROR]" + msg
+    print("[ERROR]" + msg)
+    return response
+
+
+@app.route('/api/readinfo', methods=['POST'])
+def readinfo():
+    global BACKEND
+    instance = request.json
+    root_path = Path(instance["root_path"])
+
+
+    response = {"status": "normal"}
+    if not (root_path / "training").exists():
+        response["status"] = "error"
+        response["message"] = "ERROR: your root path is incorrect."
+        print("ERROR: your root path is incorrect.")
+        return response
+    BACKEND.root_path = root_path
+    info_path = Path(instance["info_path"])
+
+    if not info_path.exists():
+        response["status"] = "error"
+        response["message"] = "ERROR: info file not exist."
+        print("ERROR: your root path is incorrect.")
+        return response
+    BACKEND.info_path = info_path
+    with open(info_path, 'rb') as f:
+        kitti_infos = pickle.load(f)
+    BACKEND.kitti_infos = kitti_infos
+    BACKEND.image_idxes = [info["image_idx"] for info in kitti_infos]
+    response["image_indexes"] = BACKEND.image_idxes
+
+    response = jsonify(results=[response])
+    response.headers['Access-Control-Allow-Headers'] = '*'
+    return response
+
+@app.route('/api/read_detection', methods=['POST'])
+def read_detection():
+    global BACKEND
+    instance = request.json
+    det_path = Path(instance["det_path"])
+    response = {"status": "normal"}
+    if BACKEND.root_path is None:
+        return error_response("root path is not set")
+    if BACKEND.kitti_infos is None:
+        return error_response("kitti info is not loaded")
+
+    if Path(det_path).is_file():
+        with open(det_path, "rb") as f:
+            dt_annos = pickle.load(f)
+    else:
+        dt_annos = kitti.get_label_annos(det_path)
+    BACKEND.dt_annos = dt_annos
+    response = jsonify(results=[response])
+    response.headers['Access-Control-Allow-Headers'] = '*'
+    return response
+
+
+@app.route('/api/get_pointcloud', methods=['POST'])
+def get_pointcloud():
+    global BACKEND
+    instance = request.json
+    response = {}
+    if BACKEND.root_path is None:
+        return error_response("root path is not set")
+    if BACKEND.kitti_infos is None:
+        return error_response("kitti info is not loaded")
+    image_idx = instance["image_idx"]
+    idx = BACKEND.image_idxes.index(image_idx)
+    kitti_info = BACKEND.kitti_infos[idx]
+    rect = kitti_info['calib/R0_rect']
+    P2 = kitti_info['calib/P2']
+    Trv2c = kitti_info['calib/Tr_velo_to_cam']
+    if 'annos' in kitti_info:
+        annos = kitti_info['annos']
+        labels = annos['name']
+        num_obj = len([n for n in annos['name'] if n != 'DontCare'])
+        dims = annos['dimensions'][:num_obj]
+        loc = annos['location'][:num_obj]
+        rots = annos['rotation_y'][:num_obj]
+        gt_boxes_camera = np.concatenate(
+            [loc, dims, rots[..., np.newaxis]], axis=1)
+        gt_boxes = box_np_ops.box_camera_to_lidar(
+            gt_boxes_camera, rect, Trv2c)
+        box_np_ops.change_box3d_center_(gt_boxes, src=[0.5, 0.5, 0], dst=[0.5, 0.5, 0.5])
+        locs = gt_boxes[:, :3]
+        dims = gt_boxes[:, 3:6]
+        rots = np.concatenate([np.zeros([num_obj, 2], dtype=np.float32), -gt_boxes[:, 6:7]], axis=1)
+        frontend_annos = {}
+        response["locs"] = locs.tolist()
+        response["dims"] = dims.tolist()
+        response["rots"] = rots.tolist()
+        response["labels"] = labels[:num_obj].tolist()
+
+    v_path = str(Path(BACKEND.root_path) / kitti_info['velodyne_path'])
+    with open(v_path, 'rb') as f:
+        pc_str = base64.encodestring(f.read())
+    response["pointcloud"] = pc_str.decode("utf-8")
+    if "with_det" in instance and instance["with_det"]:
+        if BACKEND.dt_annos is None:
+            return error_response("det anno is not loaded")
+        dt_annos = BACKEND.dt_annos[idx]
+        dims = dt_annos['dimensions']
+        num_obj = dims.shape[0]
+        loc = dt_annos['location']
+        rots = dt_annos['rotation_y']
+        labels = dt_annos['name']
+
+        dt_boxes_camera = np.concatenate(
+            [loc, dims, rots[..., np.newaxis]], axis=1)
+        dt_boxes = box_np_ops.box_camera_to_lidar(
+            dt_boxes_camera, rect, Trv2c)
+        box_np_ops.change_box3d_center_(dt_boxes, src=[0.5, 0.5, 0], dst=[0.5, 0.5, 0.5])
+        locs = dt_boxes[:, :3]
+        dims = dt_boxes[:, 3:6]
+        rots = np.concatenate([np.zeros([num_obj, 2], dtype=np.float32), -dt_boxes[:, 6:7]], axis=1)
+        response["dt_locs"] = locs.tolist()
+        response["dt_dims"] = dims.tolist()
+        response["dt_rots"] = rots.tolist()
+        response["dt_labels"] = labels.tolist()
+        response["dt_scores"] = dt_annos["score"].tolist()
+
+    # if "score" in annos:
+    #     response["score"] = score.tolist()
+    response = jsonify(results=[response])
+    response.headers['Access-Control-Allow-Headers'] = '*'
+    print("send response!")
+    return response
+
+def main(port=16666):
+    app.run(host='0.0.0.0', threaded=True, port=port)
+
+if __name__ == '__main__':
+    fire.Fire()