sysc release-4.2¡°

.ci/deps_tests/boot.py

@@ -0,0 +1,164 @@
+#!/usr/bin/python
+##
+import os
+import sys
+import re
+from bs4 import BeautifulSoup
+
+##  get all test catalog
+def get_test_all_catalog(name):
+    CatalogList = []
+    PackageNameList = []
+    filepath = os.path.join(os.getcwd(), name)
+    packagenames = os.listdir(name)
+    for packagename in packagenames:
+        tmp_path = os.path.join(filepath, packagename)
+        if  os.path.isdir(tmp_path):
+            CatalogList.append(tmp_path)
+            PackageNameList.append(packagename)
+    return CatalogList, PackageNameList
+
+## Start Galaxy Engine
+## Save index. d by extracting important information from index.html.
+## extract index.html and deal with path information in index.html
+def extract_core_content(path, filename, packagename):
+    filepath = os.path.join(path, filename)
+    if os.path.exists(filepath):
+        fopen = open(filepath)
+        filecontent = fopen.read()
+        original_core_content = re.findall(r"<\s*body[^>]*>(.+?)<div class=\"copyright\">Copyright", filecontent, re.S)
+        content = original_core_content[0]
+        core_content = content.replace('<a href=\"', '<a href=\"'+packagename+"\\")
+        ## change title
+        core_content = core_content.replace('Test Results', packagename + ' Test Results')
+        return core_content
+
+def replace_href(path, filename, packagename):
+    filepath = os.path.join(path, filename)
+    if os.path.exists(filepath):
+        fopen = open(filepath)
+        filecontent = fopen.read()
+        re_data = re.compile(r"<div class=\"copyright\">Copyright(.+?)<\s*/\s*html\s*>", re.S)
+        original_core_content = re_data.sub('', filecontent)
+        replacecontent = original_core_content.replace('<a href=\"', '<a href=\"'+packagename+"\\")
+        replacecontent = replacecontent.replace('<link rel=\"stylesheet\" href=\"', '<link rel=\"stylesheet\" href=\"'+packagename+"\\")
+        replacecontent = replacecontent.replace('<script type=\"text/javascript\" src=\"', '<script type=\"text/javascript\" src=\"'+packagename+"\\")   
+        replacecontent = replacecontent.replace('Test Results', packagename + ' Test Results')
+        ra_title = re.compile("<title>(.+?)</title>", re.S)
+        replacecontent = ra_title.sub('<title> Test Result </title>', replacecontent)
+        return replacecontent
+
+# Splicing content
+def splicing_content(name, filename):
+    CoreContent = []
+    index = 0,
+    CatalogList, PackageNameList = get_test_all_catalog(name)
+    for catalog, packageName in zip(CatalogList, PackageNameList):
+        if(index == (0,)):
+            Content = replace_href(catalog, filename, packageName)
+            CoreContent.append(Content)
+            index = 1
+        else:
+            Content = extract_core_content(catalog, filename, packageName)
+            CoreContent.append(Content)
+    str = ""
+    con =  str.join('%s' % id for id in CoreContent)
+    con = con + "</body></html>"
+    return con
+
+## process data statistics
+def data_statistics(filestream):
+    soup_string = BeautifulSoup(filestream, "html.parser")
+    body_datas = soup_string.find_all("tbody")
+    testpluginList = []
+    testmoudleList = []
+    testsuccessList = []
+    testfailList = []
+    testskipList = []
+    allsuccessValue = allfailValue = allSkipValue = 0
+    for body_data in body_datas:
+        ## test moudle data
+        # testmoudledata = body_data.find_all("a")
+        testplugindata = body_data.find_all("tr")
+        for testplugin in testplugindata:
+            data = testplugin.find_all("td")
+            testpluginname = data[8]
+            con = testpluginname.find("a")['href']
+            re_data = re.compile(r"\\ct_run(.+?).html", re.S)
+            re_data = re_data.sub('', con)
+            testpluginList.append(str(re_data))
+            testmoudledata = testplugin.find_all("a")[0]
+            testmoudleList.append(testmoudledata)
+            successdata = data[3]
+            testsuccessList.append(successdata)
+            faildata = data[4]
+            testfailList.append(faildata)
+            skipdata = data[5]
+            testskipList.append(skipdata)
+    ## get all value         
+    testallvalues = soup_string.find_all("tfoot")
+    for testallvalue in testallvalues:
+        data = testallvalue.find_all("td")
+        allsuccessValue  +=  int(data[3].string)
+        allfailValue += int(data[4].string)
+        datas = str(data[5].string)
+        re_skipdata = re.compile("(.+?)/", re.S)
+        datas = str(re_skipdata.sub('', datas)).replace(")", "")
+        allSkipValue += int(datas)
+    tbody = ""
+    table_head = "<center><h1>Data Outline</h1></center><br /><center><table id=\"SortableTable\"\><thead><tr><th>Test Name</th><th>Success</th><th>Failed</th><th>Skiped</th><th>Plugin Name</th></tr></thead>"
+    for testplugin, testmoudle, testsuccess, testfail, testskip in zip(testpluginList, testmoudleList, testsuccessList, testfailList, testskipList):
+        moudlebody = "<tbody><tr><td>{}</td>".format(testmoudle)
+        successbody = "{}".format(testsuccess)
+        failbody = "{}".format(testfail)
+        skipbody = "{}".format(testskip)
+        pluginbody = "<td>{}</td></tr></tbody>".format(testplugin)
+        tbody += str(moudlebody) + str(successbody) +str(failbody) + str(skipbody) + str(pluginbody)
+    totalbody = "<tfoot><tr><td><b>Total</b></td><td>{}</td><td><font color=\"red\">{}</font></td><td><font color=\"green\">{}</font></td><td>&nbsp;</td></tr></tfoot></table>".format(allsuccessValue, allfailValue, allSkipValue)
+    outline_head = "<center><h1>Executing Plugin Name</h1></center><br /><center><table id=\"SortableTable\"\><thead><tr><th>All Test Plugin Name</th><th>Success extract Log plugin name</th><th>Empty plugin name</th></tr></thead>"
+    catalogList, packageNameList = get_test_all_catalog("logs")
+    successPluginData = ""
+    failedPluginData = ""
+    for catalog, packagename in zip(catalogList, packageNameList):
+        successPlugin, failedPlugin = outline_value(catalog, packagename)
+        successPluginData += successPlugin + " "
+        failedPluginData += failedPlugin +" "
+    outline_body_value =  "<tbody><tfoot><tr><td>{}</td><td>{}</td><td><font color=\"red\">{}<font></td></tr></tfoot></tbody></table>".format(packageNameList, successPluginData, failedPluginData)
+    data_outline_html = outline_head + outline_body_value + table_head + tbody + totalbody
+
+    return  data_outline_html, allfailValue
+
+def outline_value(path, pluginName):
+    filepath = os.path.join(path, "index.html")
+    successPlugin = ""
+    failedPlugin = ""
+    if os.path.exists(filepath):
+        successPlugin = pluginName
+    else:
+        failedPlugin = pluginName
+    return successPlugin, failedPlugin
+
+def produceHtml():
+    htmldata = splicing_content("logs", "index.html")
+    data_outline_html, allfailValue = data_statistics(htmldata)
+    headdatamatch = re.match(r"<!DOCTYPE(.+?)<\s*body[^>]*>", htmldata, re.S)
+    headdata = ""
+    if headdatamatch:
+        headdata = headdatamatch.group()
+    # re_data = re.compile(r"<!DOCTYPE (.+?)<\s*body[^>]*>", re.S)
+    replacedata = headdata + data_outline_html
+    htmldata = htmldata.replace(headdata, replacedata)
+    savepath = os.path.join(os.getcwd(), "logs")
+    savepath = os.path.join(savepath, "index.html")
+    if os.path.exists(savepath):
+        os.remove(savepath)
+    f = open(savepath, 'w')
+    f.write(htmldata)
+    f.close
+
+    ## send exit message when failed
+    if allfailValue >0 :
+        exit(1)
+# htmldata = splicing_content("logs", "index.html")
+# data_statistics(htmldata)
+produceHtml()

.ci/deps_tests/docker-compose.yaml

@@ -0,0 +1,101 @@
+version: '3'
+
+services:
+  nginx:
+    build: ./emqx-nginx
+    image: emqx-nginx:1.15
+    restart: always
+    ports:
+      - "18080:80"
+    networks:
+      - emqx-bridge
+    volumes:
+      - ../../tests/logs:/usr/share/nginx/html
+
+  emqx:
+    build: ./emqx-erlang
+    image: emqx-erlang
+    depends_on:
+      - mysql_server
+      - redis_server
+      - mongo_server
+      - pgsql_server
+      - ldap_server
+    networks:
+      - emqx-bridge
+    volumes:
+      - ../../.:/emqx-rel
+    tty: true
+
+  python:
+    image: python:3.7.2
+    networks:
+      - emqx-bridge
+    volumes:
+      - ../../.:/emqx-rel
+    tty: true
+
+  mysql_server:
+    image: mysql:5.7
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: public 
+      MYSQL_DATABASE: mqtt
+    volumes:
+      - ../../_build/emqx/lib/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem:/etc/certs/ca-cert.pem
+      - ../../_build/emqx/lib/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-cert.pem:/etc/certs/server-cert.pem
+      - ../../_build/emqx/lib/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-key.pem:/etc/certs/server-key.pem
+    networks:
+      - emqx-bridge
+    command:
+      --bind-address 0.0.0.0
+      --default-authentication-plugin=mysql_native_password
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_general_ci
+      --explicit_defaults_for_timestamp=true
+      --lower_case_table_names=1
+      --max_allowed_packet=128M
+      --skip-symbolic-links
+      --ssl-ca=/etc/certs/ca.pem
+      --ssl-cert=/etc/certs/server-cert.pem
+      --ssl-key=/etc/certs/server-key.pem
+
+  redis_server:
+    build: ./emqx-redis
+    image: emqx-redis:5
+    restart: always
+    networks:
+      - emqx-bridge
+
+  mongo_server:
+    image: mongo:4.1
+    restart: always
+    environment:
+      MONGO_INITDB_DATABASE: mqtt
+    networks:
+      - emqx-bridge
+    command: --bind_ip_all
+
+  pgsql_server:
+    image: postgres:11 
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: public
+      POSTGRES_USER: root
+      POSTGRES_DB: mqtt
+    networks:
+      - emqx-bridge
+
+  ldap_server:
+    build: ./emqx-ldap
+    image: emqx-ldap:1.0
+    restart: always
+    networks:
+      - emqx-bridge
+
+networks:
+  emqx-bridge:
+    driver: bridge
+
+volumes:
+  logs-volumes:

.ci/deps_tests/emqx-erlang/Dockerfile

@@ -0,0 +1,15 @@
+FROM erlang:22.3
+
+# RUN curl -L -o /tmp/openjdk-14.0.1_linux-x64_bin.tar.gz  https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/openjdk-14.0.1_linux-x64_bin.tar.gz \
+#     && tar xvf /tmp/openjdk-14.0.1_linux-x64_bin.tar.gz -C /usr/local
+
+# ENV PATH=/usr/local/jdk-14.0.1/bin:$PATH
+
+RUN wget --no-cookies \
+    --no-check-certificate \
+    --header "Cookie: oraclelicense=accept-securebackup-cookie" \
+    https://download.oracle.com/otn-pub/java/jdk/8u251-b08/3d5a2bb8f8d4428bbe94aed7ec7ae784/jdk-8u251-linux-x64.tar.gz \
+    -O /tmp/jdk-8u251-linux-x64.tar.gz \
+    && tar xvf /tmp/jdk-8u251-linux-x64.tar.gz  -C /usr/local
+
+ENV PATH=/usr/local/jdk1.8.0_251/bin:$PATH

.ci/deps_tests/emqx-ldap/Dockerfile

@@ -0,0 +1,22 @@
+FROM buildpack-deps:stretch
+
+ENV VERSION=2.4.47
+
+RUN apt-get update && apt-get install -y groff groff-base
+RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${VERSION}.tgz \
+    && gunzip -c openldap-${VERSION}.tgz | tar xvfB - \
+    && cd openldap-${VERSION} \
+    && ./configure && make depend && make && make install \
+    && cd .. && rm -rf  openldap-${VERSION}
+
+COPY slapd.conf /usr/local/etc/openldap/slapd.conf
+COPY schema/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif
+COPY schema/emqx.schema /usr/local/etc/openldap/schema/emqx.schema
+
+RUN mkdir -p /usr/local/etc/openldap/data \
+    && slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf
+
+WORKDIR /usr/local/etc/
+EXPOSE 389
+
+CMD [ "/usr/local/libexec/slapd", "-d", "3" ]