Streaming improvements No 2 (netdata#19167)

* fix coverity issue 451681 Read from pointer after free

* coverity 451580 Untrusted loop bound

* move vendored sqlite3 files to its own directory

* move vendored libraries

CMakeLists.txt

@@ -653,40 +653,40 @@ netdata_detect_protobuf()
 #
 
 set(LIBJUDY_PREV_FILES
-        src/libnetdata/libjudy/src/JudyL/JudyLPrev.c
-        src/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLPrev.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLPrevEmpty.c
 )
 
 set(LIBJUDY_NEXT_FILES
-        src/libnetdata/libjudy/src/JudyL/JudyLNext.c
-        src/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLNext.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLNextEmpty.c
 )
 
 set(LIBJUDY_SOURCES
-        src/libnetdata/libjudy/src/Judy.h
-        src/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c
-        src/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h
-        src/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h
-        src/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h
-        src/libnetdata/libjudy/src/JudyL/JudyL.h
-        src/libnetdata/libjudy/src/JudyL/JudyLByCount.c
-        src/libnetdata/libjudy/src/JudyL/JudyLCascade.c
-        src/libnetdata/libjudy/src/JudyL/JudyLCount.c
-        src/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c
-        src/libnetdata/libjudy/src/JudyL/JudyLDecascade.c
-        src/libnetdata/libjudy/src/JudyL/JudyLDel.c
-        src/libnetdata/libjudy/src/JudyL/JudyLFirst.c
-        src/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c
-        src/libnetdata/libjudy/src/JudyL/j__udyLGet.c
-        src/libnetdata/libjudy/src/JudyL/JudyLGet.c
-        src/libnetdata/libjudy/src/JudyL/JudyLInsArray.c
-        src/libnetdata/libjudy/src/JudyL/JudyLIns.c
-        src/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c
-        src/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c
-        src/libnetdata/libjudy/src/JudyL/JudyLMemActive.c
-        src/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c
-        src/libnetdata/libjudy/src/JudyL/JudyLTables.c
-        src/libnetdata/libjudy/src/JudyHS/JudyHS.c
+        src/libnetdata/libjudy/vendored/Judy.h
+        src/libnetdata/libjudy/vendored/JudyCommon/JudyMalloc.c
+        src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivate.h
+        src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivate1L.h
+        src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivateBranch.h
+        src/libnetdata/libjudy/vendored/JudyL/JudyL.h
+        src/libnetdata/libjudy/vendored/JudyL/JudyLByCount.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLCascade.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLCount.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLCreateBranch.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLDecascade.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLDel.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLFirst.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLFreeArray.c
+        src/libnetdata/libjudy/vendored/JudyL/j__udyLGet.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLGet.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLInsArray.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLIns.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLInsertBranch.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLMallocIF.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLMemActive.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLMemUsed.c
+        src/libnetdata/libjudy/vendored/JudyL/JudyLTables.c
+        src/libnetdata/libjudy/vendored/JudyHS/JudyHS.c
         ${LIBJUDY_PREV_FILES}
         ${LIBJUDY_NEXT_FILES}
 )
@@ -733,8 +733,8 @@ set(LIBNETDATA_FILES
         src/libnetdata/inlined.h
         src/libnetdata/json/json.c
         src/libnetdata/json/json.h
-        src/libnetdata/json/jsmn.c
-        src/libnetdata/json/jsmn.h
+        src/libnetdata/json/vendored/jsmn.c
+        src/libnetdata/json/vendored/jsmn.h
         src/libnetdata/libnetdata.c
         src/libnetdata/libnetdata.h
         src/libnetdata/line_splitter/line_splitter.c
@@ -1416,11 +1416,11 @@ set(RRD_PLUGIN_FILES
         src/database/sqlite/sqlite_aclk_node.h
         src/database/sqlite/sqlite_aclk_alert.c
         src/database/sqlite/sqlite_aclk_alert.h
-        src/database/sqlite/sqlite3.c
-        src/database/sqlite/sqlite3.h
-        src/database/sqlite/sqlite3recover.c
-        src/database/sqlite/sqlite3recover.h
-        src/database/sqlite/dbdata.c
+        src/database/sqlite/vendored/sqlite3.c
+        src/database/sqlite/vendored/sqlite3.h
+        src/database/sqlite/vendored/sqlite3recover.c
+        src/database/sqlite/vendored/sqlite3recover.h
+        src/database/sqlite/vendored/dbdata.c
         src/database/KolmogorovSmirnovDist.c
         src/database/KolmogorovSmirnovDist.h
         src/database/rrdfunctions-inflight.c
@@ -1926,14 +1926,14 @@ target_compile_definitions(judy PRIVATE
 )
 
 target_include_directories(judy PUBLIC
-        src/libnetdata/libjudy/src
-        src/libnetdata/libjudy/src/JudyCommon
+        src/libnetdata/libjudy/vendored
+        src/libnetdata/libjudy/vendored/JudyCommon
 )
 
 set_source_files_properties(${LIBJUDY_PREV_FILES} PROPERTIES COMPILE_OPTIONS "-DJUDYPREV")
 set_source_files_properties(${LIBJUDY_NEXT_FILES} PROPERTIES COMPILE_OPTIONS "-DJUDYNEXT")
-set_source_files_properties(src/libnetdata/libjudy/src/JudyL/j__udyLGet.c PROPERTIES COMPILE_OPTIONS "-DJUDYGETINLINE")
-set_source_files_properties(src/libnetdata/libjudy/src/JudyL/JudyLByCount.c PROPERTIES COMPILE_OPTIONS "-DNOSMARTJBB -DNOSMARTJBU -DNOSMARTJLB")
+set_source_files_properties(src/libnetdata/libjudy/vendored/JudyL/j__udyLGet.c PROPERTIES COMPILE_OPTIONS "-DJUDYGETINLINE")
+set_source_files_properties(src/libnetdata/libjudy/vendored/JudyL/JudyLByCount.c PROPERTIES COMPILE_OPTIONS "-DNOSMARTJBB -DNOSMARTJBU -DNOSMARTJLB")
 set_source_files_properties(JudyLTables.c PROPERTIES COMPILE_OPTIONS "-I${CMAKE_SOURCE_DIR}/src/libnetdata/libjudy/src/JudyL")
 
 #

src/daemon/config/netdata-conf-global.c

@@ -13,7 +13,9 @@ static int get_hostname(char *buf, size_t buf_size) {
         }
     }
 
-    return gethostname(buf, buf_size);
+    int rc = gethostname(buf, buf_size);
+    buf[buf_size - 1] = '\0';
+    return rc;
 }
 
 void netdata_conf_section_global(void) {
@@ -26,7 +28,7 @@ void netdata_conf_section_global(void) {
     (void) verify_netdata_host_prefix(true);
 
     char buf[HOSTNAME_MAX + 1];
-    if (get_hostname(buf, HOSTNAME_MAX))
+    if (get_hostname(buf, sizeof(buf)))
         netdata_log_error("Cannot get machine hostname.");
 
     netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", buf);

src/database/sqlite/sqlite_context.h

@@ -4,7 +4,7 @@
 #define NETDATA_SQLITE_CONTEXT_H
 
 #include "daemon/common.h"
-#include "sqlite3.h"
+#include "database/sqlite/vendored/sqlite3.h"
 
 int sql_context_cache_stats(int op);
 typedef struct ctx_chart {

src/database/sqlite/sqlite_db_migration.h

@@ -3,8 +3,7 @@
 #define NETDATA_SQLITE_DB_MIGRATION_H
 
 #include "daemon/common.h"
-#include "sqlite3.h"
-
+#include "database/sqlite/vendored/sqlite3.h"
 
 int perform_database_migration(sqlite3 *database, int target_version);
 int perform_context_database_migration(sqlite3 *database, int target_version);

src/database/sqlite/sqlite_functions.h

@@ -4,7 +4,7 @@
 #define NETDATA_SQLITE_FUNCTIONS_H
 
 #include "daemon/common.h"
-#include "sqlite3.h"
+#include "database/sqlite/vendored/sqlite3.h"
 
 void analytics_set_data_str(char **name, const char *value);
 

src/database/sqlite/sqlite_health.h

@@ -4,7 +4,7 @@
 #define NETDATA_SQLITE_HEALTH_H
 
 #include "daemon/common.h"
-#include "sqlite3.h"
+#include "database/sqlite/vendored/sqlite3.h"
 
 #define ALERT_TRANSITION_DELAY_LONG  (600)
 #define ALERT_TRANSITION_DELAY_SHORT (10)

src/database/sqlite/sqlite_metadata.c

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "sqlite_metadata.h"
-#include "sqlite3recover.h"
+#include "database/sqlite/vendored/sqlite3recover.h"
 //#include "sqlite_db_migration.h"
 
 #define DB_METADATA_VERSION 18

src/database/sqlite/sqlite_metadata.h

@@ -3,7 +3,7 @@
 #ifndef NETDATA_SQLITE_METADATA_H
 #define NETDATA_SQLITE_METADATA_H
 
-#include "sqlite3.h"
+#include "database/sqlite/vendored/sqlite3.h"
 #include "sqlite_functions.h"
 
 typedef enum event_log_type {

src/database/sqlite/vendored/README.md

@@ -0,0 +1 @@
+Vendored copy of SQLITE3

src/libnetdata/inlined.h

@@ -622,9 +622,14 @@ static inline char *strsep_skip_consecutive_separators(char **ptr, char *s) {
 
 // remove leading and trailing spaces; may return NULL
 static inline char *trim(char *s) {
+    char *buf = s;
+
     // skip leading spaces
     while (*s && isspace((uint8_t)*s)) s++;
-    if (!*s) return NULL;
+    if (!*s) {
+        *buf = '\0';
+        return NULL;
+    }
 
     // skip tailing spaces
     // this way is way faster. Writes only one NUL char.
@@ -635,7 +640,10 @@ static inline char *trim(char *s) {
         *++p = '\0';
     }
 
-    if (!*s) return NULL;
+    if (!*s) {
+        *buf = '\0';
+        return NULL;
+    }
 
     return s;
 }

src/libnetdata/json/json.h

@@ -10,7 +10,7 @@
 #endif // error_description
 #endif // ENABLE_JSONC
 
-#include "jsmn.h"
+#include "libnetdata/json/vendored/jsmn.h"
 
 //https://www.ibm.com/support/knowledgecenter/en/SS9H2Y_7.6.0/com.ibm.dp.doc/json_parserlimits.html
 #define JSON_NAME_LEN 256

src/libnetdata/libnetdata.h

@@ -107,7 +107,7 @@ extern const char *netdata_configured_host_prefix;
 
 // safe includes before O/S specific functions
 #include "template-enum.h"
-#include "libjudy/src/Judy.h"
+#include "libjudy/vendored/Judy.h"
 #include "libjudy/judyl-typed.h"
 #include "july/july.h"
 

src/ml/ml.cc

@@ -5,7 +5,7 @@
 #include <array>
 
 #include "ad_charts.h"
-#include "database/sqlite/sqlite3.h"
+#include "database/sqlite/vendored/sqlite3.h"
 
 #define WORKER_TRAIN_QUEUE_POP         0
 #define WORKER_TRAIN_ACQUIRE_DIMENSION 1

src/streaming/stream-receiver.c

@@ -440,6 +440,7 @@ void stream_receiver_move_queue_to_running_unsafe(struct stream_thread *sth) {
 static void stream_receiver_remove(struct stream_thread *sth, struct receiver_state *rpt, const char *why) {
     internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );
 
+    errno_clear();
     nd_log(NDLS_DAEMON, NDLP_ERR,
            "STREAM RECEIVE[%zu] '%s' [from [%s]:%s]: "
            "receiver disconnected: %s"
@@ -700,8 +701,8 @@ void stream_receive_process_poll_events(struct stream_thread *sth, struct receiv
     // we can receive data from this socket
 
     worker_is_busy(WORKER_STREAM_JOB_SOCKET_RECEIVE);
-    while(true) {
-        bool removed = false;
+    bool removed = false;
+    while(!removed) {
         ssize_t rc = stream_receive_and_process(sth, rpt, parser, &removed);
         if (likely(rc > 0)) {
             rpt->last_msg_t = (time_t)(now_ut / USEC_PER_SEC);