Feature: fixed TdiVar/TdiCompile lockissue by making them threadsafe&…

…reentrant (MDSplus#1866)

* Tests: fixed Makefile tests-env

* Tests: simplify AllTests configuration and cleanup

* Tests: added suppression for getservbyname's malloc of static memory

* Tests: enable all mdsplus-api test for valgrind

* Tests: disable helgrind for java as java itself has too many issues; TODO find a way to suppress

* Tests: enabled valgrind for java/mdsobjcets; TODO: fix memory leaks in Tree

* Tests: added memcheck suppressions for fc30

* Tests: added -Xint flag to junittest which prevents java helgrind issues

* Tests: TreePutTimstampedSegment, correct dimensions is key

* Build: include JAVAROOT in CLASSPATH

* Build: maven check version script

* Fix: avoid :T tdi calls as freeing may not work

* Fix: cleanup of mdsip; Connection interfaces; ByteBuffer

* Fix: propagate ByteBuffer into API

* Fix: removed unused pubic methods from MdsIp

* Fix: TreeFindTagWild finally cleanup

* Feature: get rid of JavaMdsLib

* Fix: unblocked SSH client and final clean up

* Fix: small cleanup of TreeSegments

* Fix: small cleanup of MdsXdRoutines

* Fix: use calloc for malloc+memset 0 in TreeOpen.c

* Fix: use realloc to trunc allocated memory

* Fix: memoryleak in TreeFindTagWild and cleanup

* Fix: memoryleak in TreeFindTag

* Fix: memory leak in free dbid deleted_nid_list

* Fix: added TreeGetMinimumPathDsc and TreeGetPathDsc

* Fix: use TreeFindTagsDsc in favour of  TreeFindTags

* Fix: optimized TdiHash for speed

* Fix: refactor TdiLex.c to make it threadsafe

* Build: we have git to version and backup old code

* Fix: avoid lock in TdiHash

* Fix: avoid lock in TdiCompile

* Fix: TdiVar reentrant

* Fix: further optimized TdiHash for collisionless table

* Fix: *lastch = text[0] uninitialized

* Fix: range oddity fix

* Fix: re-enabled modulo operator %

* Fix: no need to test next if collisions are resolved

Makefile.in

@@ -23,8 +23,7 @@ JAVA_JAR = \
 	   java/mdsobjects/tests
 
 JAVA_APS = \
-	   javamds \
-	   java/mdsplus-api/src/main/jni/mds/mdslib
+	   javamds
 
 ifeq "@HAVE_MOTIF@" "yes"
 MOTIF_APS = xmdsshr \
@@ -204,12 +203,12 @@ install: $(install_DIRS)
 .PHONY: tests-envdump
 tests-envdump:
 	cd python/MDSobjects/tests&&(\
-	make test-envdump;\
+	make tests-envdump;\
 	cd ../../..;)
 
-test-env:
+tests-env:
 	cd python/MDSplus/tests&&(\
-	make test-env;\
+	make tests-env;\
 	cd ../../..;)
 
 @HAVE_WINE_TRUE@wineconsole:

conf/valgrind-mdsplus.supp

@@ -166,3 +166,14 @@
    fun:dlopen*
    fun:loadLib
 }
+
+{
+   fc26 in getservbyname
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:__nss_database_lookup
+   fun:__nss_services_lookup2
+   fun:getservbyname_r@@GLIBC_2.2.5
+   fun:getservbyname
+}

conf/valgrind-python.supp

@@ -608,3 +608,44 @@
    fun:long_dealloc.lto_priv.328
    fun:frame_dealloc
 }
+
+{
+   fc30 python 3.7m
+   Memcheck:Leak
+   match-leak-kinds: possible
+   fun:*alloc
+   ...
+   fun:PyUFunc_FromFuncAndDataAndSignatureAndIdentity
+   fun:InitOperators
+   fun:PyInit__multiarray_umath
+   fun:_PyImport_LoadDynamicModuleWithSpec
+   obj:*/libpython3.7m.so*
+}
+
+{
+   fc30 python 3.7m
+   Memcheck:Cond
+   fun:PyUnicode_Decode
+   fun:PyUnicode_FromEncodedObject
+   obj:*/libpython3.7m.so*
+}
+
+{
+   fc30 python 3.7m
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:malloc
+   fun:_dl_map_object_deps
+   fun:dl_open_worker
+   fun:_dl_catch_exception
+   fun:_dl_open
+   fun:dlopen_doit
+   fun:_dl_catch_exception
+   fun:_dl_catch_error
+   fun:_dlerror_run
+   fun:dlopen@@GLIBC_2.2.5
+   obj:*/_ctypes.cpython-37m-*.so
+   fun:_PyMethodDef_RawFastCallKeywords
+   fun:_PyCFunction_FastCallKeywords
+   obj:*/libpython3.7m.so*
+}

configure

@@ -18001,7 +18001,7 @@ ac_config_headers="$ac_config_headers include/mdsplus/mdsconfig.h"
 
 
 
-ac_config_files="$ac_config_files Makefile.inc Makefile docs/Makefile envsyms actions/Makefile camshr/Makefile ccl/Makefile d3dshr/Makefile dwscope/Makefile hdf5/Makefile idlmdsevent/Makefile idlmdswidgets/Makefile javamds/Makefile java/Makefile java/mdsplus/Makefile java/mdsplus-api/Makefile java/mdsplus-api/src/main/jni/mds/mdslib/Makefile java/mdsplus-api/tests/Makefile java/mdsobjects/Makefile java/mdsobjects/docs/Makefile java/mdsobjects/tests/Makefile java/jdispatcher/Makefile java/jscope/Makefile java/jtraverser/Makefile java/jtraverser2/Makefile java/jdevices/Makefile LabView/Makefile manpages/Makefile macosx/Makefile math/Makefile mdsdcl/Makefile mdslib/Makefile mdslib/docs/Makefile mdslib/testing/Makefile mdslibidl/Makefile mdsobjects/cpp/Makefile mdsobjects/cpp/docs/Makefile mdsobjects/cpp/testing/Makefile mdsobjects/cpp/testing/testutils/Makefile python/MDSplus/_version.py python/MDSplus/docs/Makefile python/MDSplus/tests/Makefile mdsobjects/labview/Makefile mdsmisc/Makefile mdsshr/Makefile mdsshr/MdsVersionInfo.c mdsshr/testing/Makefile mdsshr/docs/Makefile mdssql/Makefile mdstcpip/Makefile mdstcpip/zlib/Makefile mdstcpip/docs/Makefile mdstcpip/docs/img/Makefile mdsvme/Makefile mitdevices/Makefile pydevices/MitDevices/_version.py pydevices/RfxDevices/_version.py pydevices/W7xDevices/_version.py remcam/Makefile roam/Makefile rpm/Makefile rpm/envsyms rpm/post_install_script scripts/Makefile servershr/Makefile setevent/Makefile tcl/Makefile tdishr/Makefile tdishr/testing/Makefile tdic/Makefile tditest/Makefile tditest/testing/Makefile testing/Makefile testing/backends/check/Makefile testing/selftest/Makefile traverser/Makefile treeshr/Makefile treeshr/testing/Makefile wfevent/Makefile xmdsshr/Makefile xtreeshr/Makefile"
+ac_config_files="$ac_config_files Makefile.inc Makefile docs/Makefile envsyms actions/Makefile camshr/Makefile ccl/Makefile d3dshr/Makefile dwscope/Makefile hdf5/Makefile idlmdsevent/Makefile idlmdswidgets/Makefile javamds/Makefile java/Makefile java/mdsplus/Makefile java/mdsplus-api/Makefile java/mdsplus-api/tests/Makefile java/mdsobjects/Makefile java/mdsobjects/docs/Makefile java/mdsobjects/tests/Makefile java/jdispatcher/Makefile java/jscope/Makefile java/jtraverser/Makefile java/jtraverser2/Makefile java/jdevices/Makefile LabView/Makefile manpages/Makefile macosx/Makefile math/Makefile mdsdcl/Makefile mdslib/Makefile mdslib/docs/Makefile mdslib/testing/Makefile mdslibidl/Makefile mdsobjects/cpp/Makefile mdsobjects/cpp/docs/Makefile mdsobjects/cpp/testing/Makefile mdsobjects/cpp/testing/testutils/Makefile python/MDSplus/_version.py python/MDSplus/docs/Makefile python/MDSplus/tests/Makefile mdsobjects/labview/Makefile mdsmisc/Makefile mdsshr/Makefile mdsshr/MdsVersionInfo.c mdsshr/testing/Makefile mdsshr/docs/Makefile mdssql/Makefile mdstcpip/Makefile mdstcpip/zlib/Makefile mdstcpip/docs/Makefile mdstcpip/docs/img/Makefile mdsvme/Makefile mitdevices/Makefile pydevices/MitDevices/_version.py pydevices/RfxDevices/_version.py pydevices/W7xDevices/_version.py remcam/Makefile roam/Makefile rpm/Makefile rpm/envsyms rpm/post_install_script scripts/Makefile servershr/Makefile setevent/Makefile tcl/Makefile tdishr/Makefile tdishr/testing/Makefile tdic/Makefile tditest/Makefile tditest/testing/Makefile testing/Makefile testing/backends/check/Makefile testing/selftest/Makefile traverser/Makefile treeshr/Makefile treeshr/testing/Makefile wfevent/Makefile xmdsshr/Makefile xtreeshr/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -18875,7 +18875,6 @@ do
     "java/Makefile") CONFIG_FILES="$CONFIG_FILES java/Makefile" ;;
     "java/mdsplus/Makefile") CONFIG_FILES="$CONFIG_FILES java/mdsplus/Makefile" ;;
     "java/mdsplus-api/Makefile") CONFIG_FILES="$CONFIG_FILES java/mdsplus-api/Makefile" ;;
-    "java/mdsplus-api/src/main/jni/mds/mdslib/Makefile") CONFIG_FILES="$CONFIG_FILES java/mdsplus-api/src/main/jni/mds/mdslib/Makefile" ;;
     "java/mdsplus-api/tests/Makefile") CONFIG_FILES="$CONFIG_FILES java/mdsplus-api/tests/Makefile" ;;
     "java/mdsobjects/Makefile") CONFIG_FILES="$CONFIG_FILES java/mdsobjects/Makefile" ;;
     "java/mdsobjects/docs/Makefile") CONFIG_FILES="$CONFIG_FILES java/mdsobjects/docs/Makefile" ;;

configure.ac

@@ -1393,7 +1393,6 @@ AC_OUTPUT(Makefile.inc
           java/Makefile
           java/mdsplus/Makefile
           java/mdsplus-api/Makefile
-          java/mdsplus-api/src/main/jni/mds/mdslib/Makefile
           java/mdsplus-api/tests/Makefile
           java/mdsobjects/Makefile
           java/mdsobjects/docs/Makefile

deploy/packaging/debian/java_bin.amd64

@@ -4,4 +4,3 @@
 ./usr/local/mdsplus/bin/jServer
 ./usr/local/mdsplus/bin/jTraverser
 ./usr/local/mdsplus/lib/libJavaMds.so
-./usr/local/mdsplus/lib/libJavaMdsLib.so

deploy/packaging/debian/java_bin.armhf

@@ -4,4 +4,3 @@
 ./usr/local/mdsplus/bin/jServer
 ./usr/local/mdsplus/bin/jTraverser
 ./usr/local/mdsplus/lib/libJavaMds.so
-./usr/local/mdsplus/lib/libJavaMdsLib.so

deploy/packaging/debian/java_bin.i386

@@ -4,4 +4,3 @@
 ./usr/local/mdsplus/bin/jServer
 ./usr/local/mdsplus/bin/jTraverser
 ./usr/local/mdsplus/lib/libJavaMds.so
-./usr/local/mdsplus/lib/libJavaMdsLib.so

deploy/packaging/redhat/java_bin.i686

@@ -4,4 +4,3 @@
 ./usr/local/mdsplus/bin32/jServer
 ./usr/local/mdsplus/bin32/jTraverser
 ./usr/local/mdsplus/lib32/libJavaMds.so
-./usr/local/mdsplus/lib32/libJavaMdsLib.so

deploy/packaging/redhat/java_bin.x86_64

@@ -4,4 +4,3 @@
 ./usr/local/mdsplus/bin64/jServer
 ./usr/local/mdsplus/bin64/jTraverser
 ./usr/local/mdsplus/lib64/libJavaMds.so
-./usr/local/mdsplus/lib64/libJavaMdsLib.so

deploy/platform/maven/check_version.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-javadir=java/
+cd $(realpath $(dirname $0)/../../../)
 
 get_latest() {
  URL="https://oss.sonatype.org/content/groups/public/org/mdsplus/mdsplus-parent/maven-metadata.xml"
@@ -14,17 +14,17 @@ get_latest() {
 
 get_release_diff() {
  export LATEST=$(get_latest)
- git diff alpha_release-${LATEST//./-} --name-only $javadir
+ git diff alpha_release-${LATEST//./-} --name-only java/
 }
 
 get_modules() {
- echo $(grep -oP '(?<=\<module\>).*(?=\</module\>)' ${javadir}pom.xml)
+ echo $(grep -oP '(?<=\<module\>).*(?=\</module\>)' java/pom.xml)
 }
 
 project_changed() {
  export LATEST=$(get_latest)
  MODULES=$(get_modules)
- if get_release_diff|grep -P "^$javadir(${MODULES// /\|})/.*.java\$" >/dev/null
+ if get_release_diff|grep -P "^java/(${MODULES// /\|})/.*.java\$" >/dev/null
  then
   echo yes
   return 0