Solve the problem of ES writing error caused by truncation of UTF8 ch… (

vesoft-inc#4799) * Solve the problem of ES writing error caused by truncation of UTF8 characters * fix license typo
L-shuai · Oct 28, 2022 · 39ca73b · 39ca73b
1 parent f2b8264
commit 39ca73b
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/src/common/plugin/fulltext/FTUtils.h b/src/common/plugin/fulltext/FTUtils.h
@@ -145,7 +145,15 @@ struct DocIDTraits {
   }
 
   static std::string val(const std::string& v) {
-    return ((v.size() > MAX_INDEX_TYPE_LENGTH) ? v.substr(0, MAX_INDEX_TYPE_LENGTH) : v);
+    if (v.size() <= MAX_INDEX_TYPE_LENGTH) {
+      return v;
+    }
+    size_t len = MAX_INDEX_TYPE_LENGTH;
+    const int utf8Mask = 1 << 7;  // 10000000
+    while (len < v.size() && (v[len - 1] & utf8Mask)) {
+      len++;
+    }
+    return v.substr(0, len);
   }
 
   static std::string normalizedJson(const std::string& v) {

diff --git a/src/storage/query/ScanEdgeProcessor.h b/src/storage/query/ScanEdgeProcessor.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2020 vesoft inc. All rights reserved.:
+/* Copyright (c) 2020 vesoft inc. All rights reserved.
  *
  * This source code is licensed under Apache 2.0 License.
  */

diff --git a/src/storage/query/ScanVertexProcessor.h b/src/storage/query/ScanVertexProcessor.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2020 vesoft inc. All rights reserved.:
+/* Copyright (c) 2020 vesoft inc. All rights reserved.
  *
  * This source code is licensed under Apache 2.0 License.
  */