Merge pull request esp8266#1977 from esp8266/device_tests

First batch of automated tests on the ESP: test library and test runner

.travis.yml

@@ -36,7 +36,7 @@ script:
   - install_libraries
   - echo -e "travis_fold:end:sketch_test_env_prepare"
   - echo -e "travis_fold:start:sketch_test"
-  - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR "-l $HOME/Arduino/libraries"
+  - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries"
   - echo -e "travis_fold:end:sketch_test"
   - echo -e "travis_fold:start:size_report"
   - cat size.log

tests/.gitignore

@@ -1,3 +1,2 @@
 hardware
 tmp
-.env

tests/device/.gitignore

@@ -0,0 +1,2 @@
+.build
+.hardware

tests/device/Makefile

@@ -0,0 +1,98 @@
+SHELL := /bin/bash
+V ?= 0
+TEST_LIST ?= $(wildcard test_*/*.ino)
+ESP8266_CORE_PATH ?= ../..
+BUILD_DIR ?= $(PWD)/.build
+HARDWARE_DIR ?= $(PWD)/.hardware
+ESPTOOL ?= $(ESP8266_CORE_PATH)/tools/esptool/esptool
+UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB)
+UPLOAD_BAUD ?= 921600
+UPLOAD_BOARD ?= nodemcu
+BS_DIR ?= libraries/BSTest
+DEBUG_LEVEL ?= DebugLevel=None____
+FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=DIO,UploadSpeed=115200,FlashSize=4M1M,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL)
+BUILD_TOOL = $(ARDUINO_IDE_PATH)/arduino-builder
+TEST_CONFIG = libraries/test_config/test_config.h
+
+ifeq ("$(UPLOAD_PORT)","")
+$(error "Failed to detect upload port, please export UPLOAD_PORT manually")
+endif
+
+ifeq ("$(ARDUINO_IDE_PATH)","")
+$(error "Please export ARDUINO_IDE_PATH")
+endif
+
+ifneq ("$(V)","1")
+	SILENT = @
+else
+	BUILDER_DEBUG_FLAG = -verbose
+	RUNNER_DEBUG_FLAG = -d
+	UPLOAD_VERBOSE_FLAG = -v
+endif
+
+
+all: count tests
+
+count:
+	@echo Running $(words $(TEST_LIST)) tests
+
+tests: $(BUILD_DIR) $(HARDWARE_DIR) virtualenv $(TEST_CONFIG) $(TEST_LIST)
+
+$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)
+
+$(TEST_LIST):
+	$(SILENT)mkdir -p $(LOCAL_BUILD_DIR)
+ifneq ("$(NO_BUILD)","1")
+	@echo Compiling $(notdir $@)
+	$(SILENT)$(BUILD_TOOL) -compile -logger=human \
+		-libraries "$(PWD)/libraries" \
+		-core-api-version="10608" \
+		-warnings=none \
+		$(BUILDER_DEBUG_FLAG) \
+		-build-path $(LOCAL_BUILD_DIR) \
+		-tools $(ARDUINO_IDE_PATH)/tools-builder \
+		-hardware $(HARDWARE_DIR)\
+		-hardware $(ARDUINO_IDE_PATH)/hardware \
+		-fqbn=$(FQBN) \
+		$@
+endif
+ifneq ("$(NO_UPLOAD)","1")
+	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
+		-cp $(UPLOAD_PORT) \
+		-cb $(UPLOAD_BAUD) \
+		-cd $(UPLOAD_BOARD) \
+		-cf $(LOCAL_BUILD_DIR)/$(notdir $@).bin
+endif
+ifneq ("$(NO_RUN)","1")
+	@echo Running tests
+	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) -cp $(UPLOAD_PORT) -cd $(UPLOAD_BOARD) -cr
+	@source $(BS_DIR)/virtualenv/bin/activate && \
+		python $(BS_DIR)/runner.py \
+			$(RUNNER_DEBUG_FLAG) \
+			-p $(UPLOAD_PORT) \
+			-n $(basename $(notdir $@)) \
+			-o $(LOCAL_BUILD_DIR)/test_result.xml
+endif
+
+$(BUILD_DIR):
+	mkdir -p $(BUILD_DIR)
+
+$(HARDWARE_DIR):
+	mkdir -p $(HARDWARE_DIR)/esp8266com
+	cd $(HARDWARE_DIR)/esp8266com && ln -s $(realpath $(ESP8266_CORE_PATH)) esp8266
+
+virtualenv:
+	make -C $(BS_DIR) virtualenv
+
+clean:
+	rm -rf $(BUILD_DIR)
+	rm -rf $(HARDWARE_DIR)
+
+$(TEST_CONFIG):
+	@echo "******    "
+	@echo "******    libraries/test_config/test_config.h does not exist"
+	@echo "******    Create one from libraries/test_config/test_config.h.template"
+	@echo "******    "
+	false
+
+.PHONY: tests all count venv $(BUILD_DIR) $(TEST_LIST) 

tests/device/libraries/BSTest/.gitignore

@@ -0,0 +1,3 @@
+test/test
+virtualenv
+

tests/device/libraries/BSTest/Makefile

@@ -0,0 +1,22 @@
+PYTHON_ENV_DIR=virtualenv
+TEST_EXECUTABLE=test/test
+
+all: test
+
+install: $(PYTHON_ENV_DIR)
+
+clean:
+	rm -rf $(PYTHON_ENV_DIR)
+	rm -rf $(TEST_EXECUTABLE)
+
+$(PYTHON_ENV_DIR):
+	virtualenv --no-site-packages $(PYTHON_ENV_DIR)
+	source $(PYTHON_ENV_DIR)/bin/activate && pip install -r requirements.txt
+
+test: $(TEST_EXECUTABLE) $(PYTHON_ENV_DIR)
+	source $(PYTHON_ENV_DIR)/bin/activate && python runner.py -e $(TEST_EXECUTABLE)
+
+$(TEST_EXECUTABLE): test/test.cpp
+	g++ -std=c++11 -Isrc -o $@ test/test.cpp
+
+.PHONY: test clean install all

tests/device/libraries/BSTest/library.properties

@@ -0,0 +1,9 @@
+name=BSTest
+version=0.1
+author=Ivan Grokhotkov <[email protected]>
+maintainer=Ivan Grokhotkov <[email protected]>
+sentence=BS Test library
+paragraph=
+category=Uncategorized
+url=
+architectures=esp8266

tests/device/libraries/BSTest/requirements.txt

@@ -0,0 +1,6 @@
+junit-xml==1.6
+pexpect==4.0.1
+ptyprocess==0.5.1
+pyserial==3.0.1
+six==1.10.0
+wheel==0.24.0