Skip to content

Commit

Permalink
add support for environment variables in device tests
Browse files Browse the repository at this point in the history 
Previously device tests included information such as access point SSID/password at compile time. This made it difficult to compile test binaries once and then send them to multiple test runners for execution.

This change adds a command to the test library to set environment variable on the target device: “setenv key value”. C library setenv/getenv facility is used to store variables.

Test runner, tests, and makefile are updated to use this functionality.
  • Loading branch information
@igrr
igrr committed Apr 11, 2018
1 parent 2315ac2 commit 8bd26f2
Show file tree
Hide file tree
Showing 21 changed files with 306 additions and 78 deletions.
1 change: 1 addition & 0 deletions tests/device/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
.hardware
test_report.xml
test_report.html
test_env.cfg
29 changes: 13 additions & 16 deletions tests/device/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,10 @@ BS_DIR ?= libraries/BSTest
DEBUG_LEVEL ?= DebugLevel=None____
FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=dio,UploadSpeed=115200,FlashSize=4M1M,LwIPVariant=v2mss536,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL)
BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder
TEST_CONFIG := libraries/test_config/test_config.h
TEST_CONFIG := test_env.cfg
TEST_REPORT_XML := test_report.xml
TEST_REPORT_HTML := test_report.html

ifeq ("$(UPLOAD_PORT)","")
$(error "Failed to detect upload port, please export UPLOAD_PORT manually")
endif

ifeq ("$(ARDUINO_IDE_PATH)","")
$(error "Please export ARDUINO_IDE_PATH")
endif

ifneq ("$(V)","1")
SILENT = @
Expand All @@ -35,16 +28,17 @@ endif

all: count tests test_report

count:
@echo Running $(words $(TEST_LIST)) tests
$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(BUILD_DIR) $(HARDWARE_DIR)

tests: $(BUILD_DIR) $(HARDWARE_DIR) virtualenv $(TEST_CONFIG) $(TEST_LIST)
tests: $(TEST_LIST)

$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)

$(TEST_LIST):
@echo Running $(words $(TEST_LIST)) tests
$(SILENT)mkdir -p $(LOCAL_BUILD_DIR)
ifneq ("$(NO_BUILD)","1")
@test -n "$(ARDUINO_IDE_PATH)" || (echo "Please export ARDUINO_IDE_PATH" && exit 1)
@echo Compiling $(notdir $@)
$(SILENT)$(BUILD_TOOL) -compile -logger=human \
-libraries "$(PWD)/libraries" \
Expand All @@ -59,6 +53,7 @@ ifneq ("$(NO_BUILD)","1")
$@
endif
ifneq ("$(NO_UPLOAD)","1")
@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
@echo Uploading binary
$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
-cp $(UPLOAD_PORT) \
Expand All @@ -67,6 +62,7 @@ ifneq ("$(NO_UPLOAD)","1")
-cf $(LOCAL_BUILD_DIR)/$(notdir $@).bin
endif
ifneq ("$(NO_RUN)","1")
@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
@echo Running tests
$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) -cp $(UPLOAD_PORT) -cd $(UPLOAD_BOARD) -cr
@source $(BS_DIR)/virtualenv/bin/activate && \
Expand All @@ -75,6 +71,7 @@ ifneq ("$(NO_RUN)","1")
-p $(UPLOAD_PORT) \
-n $(basename $(notdir $@)) \
-o $(LOCAL_BUILD_DIR)/test_result.xml \
--env-file $(TEST_CONFIG) \
`test -f $(addsuffix .py, $(basename $@)) && echo "-m $(addsuffix .py, $(basename $@))" || echo ""`
endif

Expand All @@ -95,7 +92,7 @@ $(HARDWARE_DIR):
cd $(HARDWARE_DIR)/esp8266com && ln -s $(realpath $(ESP8266_CORE_PATH)) esp8266

virtualenv:
make -C $(BS_DIR) virtualenv
@make -C $(BS_DIR) virtualenv

clean:
rm -rf $(BUILD_DIR)
Expand All @@ -104,9 +101,9 @@ clean:

$(TEST_CONFIG):
@echo "****** "
@echo "****** libraries/test_config/test_config.h does not exist"
@echo "****** Create one from libraries/test_config/test_config.h.template"
@echo "****** $(TEST_CONFIG) does not exist"
@echo "****** Create one from $(TEST_CONFIG).template"
@echo "****** "
false
@false

.PHONY: tests all count virtualenv test_report $(BUILD_DIR) $(TEST_LIST)
.PHONY: tests all count virtualenv test_report $(TEST_LIST)
51 changes: 45 additions & 6 deletions tests/device/libraries/BSTest/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@
import serial
import subprocess
import imp
try:
from configparser import ConfigParser
except:
from ConfigParser import ConfigParser
import itertools
from urlparse import urlparse
from junit_xml import TestSuite, TestCase
try:
Expand All @@ -34,12 +39,13 @@ class BSTestRunner(object):
CRASH = 3
BEGINTIMEOUT = 4

def __init__(self, spawn_obj, name, mocks):
def __init__(self, spawn_obj, name, mocks, env_vars):
self.sp = spawn_obj
self.tests = []
self.reset_timeout = 2
self.name = name
self.mocks = mocks
self.env_vars = env_vars

def get_test_list(self):
self.sp.sendline('-1')
Expand Down Expand Up @@ -73,6 +79,7 @@ def get_test_list(self):

def run_tests(self):
test_cases = []
should_update_env = True
for test in self.tests:
desc = test['desc']
name = test['name']
Expand All @@ -84,13 +91,20 @@ def run_tests(self):
else:
test_output = StringIO()
self.sp.logfile = test_output
print('running test "{}"'.format(name))
if should_update_env:
res = self.update_env()
if res != BSTestRunner.SUCCESS:
print('failed to set environment variables')
break;
should_update_env = False
if name in self.mocks:
print('setting up mocks')
debug_print('setting up mocks')
self.mocks[name]['setup']()
t_start = time.time()
result = self.run_test(index)
if name in self.mocks:
print('tearing down mocks')
debug_print('tearing down mocks')
self.mocks[name]['teardown']()
t_stop = time.time()
self.sp.logfile = None
Expand All @@ -103,6 +117,7 @@ def run_tests(self):
else:
print('test "{}" failed'.format(name))
test_case.add_failure_info('Test failed', output=test_output.getvalue())
should_update_env = True
test_output.close()
test_cases += [test_case];
return TestSuite(self.name, test_cases)
Expand Down Expand Up @@ -152,6 +167,22 @@ def run_test(self, index):
if timeout <= 0:
return BSTestRunner.TIMEOUT

def update_env(self):
for env_kv in self.env_vars:
self.sp.sendline('setenv "{}" "{}"'.format(env_kv[0], env_kv[1]))
timeout = 10
while timeout > 0:
res = self.sp.expect(['>>>>>bs_test_setenv ok', EOF, TIMEOUT])
if res == 0:
break
time.sleep(0.1)
timeout -= 0.1
if res == 0:
continue
else:
return BSTestRunner.TIMEOUT
return BSTestRunner.SUCCESS

ser = None

def spawn_port(port_name, baudrate=115200):
Expand All @@ -162,8 +193,8 @@ def spawn_port(port_name, baudrate=115200):
def spawn_exec(name):
return pexpect.spawn(name, timeout=0)

def run_tests(spawn, name, mocks):
tw = BSTestRunner(spawn, name, mocks)
def run_tests(spawn, name, mocks, env_vars):
tw = BSTestRunner(spawn, name, mocks, env_vars)
tw.get_test_list()
return tw.run_tests()

Expand All @@ -175,6 +206,7 @@ def parse_args():
parser.add_argument('-n', '--name', help='Test run name')
parser.add_argument('-o', '--output', help='Output JUnit format test report')
parser.add_argument('-m', '--mock', help='Set python script to use for mocking purposes')
parser.add_argument('--env-file', help='File containing a list of environment variables to set', type=argparse.FileType('r'))
return parser.parse_args()

def main():
Expand All @@ -194,12 +226,19 @@ def main():
if spawn_func is None:
debug_print("Please specify port or executable", file=sys.stderr)
return 1
env_vars = []
if args.env_file is not None:
cfg = ConfigParser()
cfg.optionxform = str
with args.env_file as fp:
cfg.readfp(fp)
env_vars = cfg.items('global')
mocks = {}
if args.mock is not None:
mocks_mod = imp.load_source('mocks', args.mock)
mocks = mock_decorators.env
with spawn_func(spawn_arg) as sp:
ts = run_tests(sp, name, mocks)
ts = run_tests(sp, name, mocks, env_vars)
if args.output:
with open(args.output, "w") as f:
TestSuite.to_file(f, [ts])
Expand Down
24 changes: 20 additions & 4 deletions tests/device/libraries/BSTest/src/BSArduino.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,27 @@ class ArduinoIOHelper
return len;
}

bool read_int(int& result)
size_t read_line(char* dest, size_t dest_size)
{
// TODO: fix this for 0 value
result = m_stream.parseInt();
return result != 0;
size_t len = 0;
// Can't use Stream::readBytesUntil here because it can't tell the
// difference between timing out and receiving the terminator.
while (len < dest_size - 1) {
int c = m_stream.read();
if (c < 0) {
delay(1);
continue;
}
if (c == '\r') {
continue;
}
if (c == '\n') {
dest[len] = 0;
break;
}
dest[len++] = c;
}
return len;
}

protected:
Expand Down
Loading

0 comments on commit 8bd26f2

Please sign in to comment.