From 93df776ba71e49471d8f4a9655f40481cf8ab9d7 Mon Sep 17 00:00:00 2001 From: samr46 Date: Fri, 7 Mar 2025 13:48:59 +0200 Subject: [PATCH 1/4] Add zlib compression support --- Shared/mods/deathmatch/logic/Enums.cpp | 15 ++ Shared/mods/deathmatch/logic/Enums.h | 2 + .../logic/luadefs/CLuaCryptDefs.cpp | 152 +++++++++++++++++- Shared/sdk/SharedUtil.Crypto.h | 70 ++++++++ Shared/sdk/SharedUtil.Hash.h | 20 ++- 5 files changed, 256 insertions(+), 3 deletions(-) diff --git a/Shared/mods/deathmatch/logic/Enums.cpp b/Shared/mods/deathmatch/logic/Enums.cpp index 28f0ffe7a20..1bf0befe2cd 100644 --- a/Shared/mods/deathmatch/logic/Enums.cpp +++ b/Shared/mods/deathmatch/logic/Enums.cpp @@ -69,6 +69,7 @@ ADD_ENUM(StringEncodeFunction::AES128, "aes128") ADD_ENUM(StringEncodeFunction::RSA, "rsa") ADD_ENUM(StringEncodeFunction::BASE64, "base64") ADD_ENUM(StringEncodeFunction::BASE32, "base32") +ADD_ENUM(StringEncodeFunction::ZLIB, "zlib") IMPLEMENT_ENUM_CLASS_END("string-encode-function") IMPLEMENT_ENUM_CLASS_BEGIN(KeyPairAlgorithm) @@ -84,6 +85,20 @@ ADD_ENUM(HmacAlgorithm::SHA384, "sha384") ADD_ENUM(HmacAlgorithm::SHA512, "sha512") IMPLEMENT_ENUM_CLASS_END("hmac-algorithm") +IMPLEMENT_ENUM_CLASS_BEGIN(ZLibFormat) +ADD_ENUM(ZLibFormat::ZRAW, "raw") +ADD_ENUM(ZLibFormat::ZLIB, "zlib") +ADD_ENUM(ZLibFormat::GZIP, "gzip") +IMPLEMENT_ENUM_CLASS_END("zlib-format") + +IMPLEMENT_ENUM_CLASS_BEGIN(ZLibStrategy) +ADD_ENUM(ZLibStrategy::DEFAULT, "default") +ADD_ENUM(ZLibStrategy::FILTERED, "filtered") +ADD_ENUM(ZLibStrategy::HUFFMAN_ONLY, "huffman") +ADD_ENUM(ZLibStrategy::RLE, "rle") +ADD_ENUM(ZLibStrategy::FIXED, "fixed") +IMPLEMENT_ENUM_CLASS_END("zlib-strategy") + IMPLEMENT_ENUM_CLASS_BEGIN(WorldSpecialProperty) ADD_ENUM(WorldSpecialProperty::HOVERCARS, "hovercars") ADD_ENUM(WorldSpecialProperty::AIRCARS, "aircars") diff --git a/Shared/mods/deathmatch/logic/Enums.h b/Shared/mods/deathmatch/logic/Enums.h index bf3a9b9fe46..6ba08999848 100644 --- a/Shared/mods/deathmatch/logic/Enums.h +++ b/Shared/mods/deathmatch/logic/Enums.h @@ -72,6 +72,8 @@ DECLARE_ENUM_CLASS(PasswordHashFunction); DECLARE_ENUM_CLASS(StringEncodeFunction); DECLARE_ENUM_CLASS(KeyPairAlgorithm); DECLARE_ENUM_CLASS(HmacAlgorithm); +DECLARE_ENUM_CLASS(ZLibFormat); +DECLARE_ENUM_CLASS(ZLibStrategy); enum class WorldSpecialProperty { diff --git a/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp b/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp index ec63d4ce7fe..b2d04a3523e 100644 --- a/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp +++ b/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp @@ -403,7 +403,8 @@ int CLuaCryptDefs::EncodeString(lua_State* luaVM) argStream.ReadEnumString(algorithm); argStream.ReadString(data); - if ((algorithm != StringEncodeFunction::BASE64 && algorithm != StringEncodeFunction::BASE32) || argStream.NextIsTable()) + if ((algorithm != StringEncodeFunction::BASE64 && algorithm != StringEncodeFunction::BASE32 && algorithm != StringEncodeFunction::ZLIB) || + argStream.NextIsTable()) { argStream.ReadStringMap(options); } @@ -727,6 +728,88 @@ int CLuaCryptDefs::EncodeString(lua_State* luaVM) } return 1; } + case StringEncodeFunction::ZLIB: + { + int compression = 9; + ZLibFormat format = ZLibFormat::GZIP; + ZLibStrategy strategy = ZLibStrategy::DEFAULT; + if (!options["format"].empty() && !StringToEnum(options["format"], format)) + { + m_pScriptDebugging->LogCustom(luaVM, "Invalid value for field 'format'"); + lua::Push(luaVM, false); + return 1; + } + if (!options["strategy"].empty() && !StringToEnum(options["strategy"], strategy)) + { + m_pScriptDebugging->LogCustom(luaVM, "Invalid value for field 'strategy'"); + lua::Push(luaVM, false); + return 1; + } + if (!options["compression"].empty()) + { + compression = atoi(options["compression"].c_str()); + if (compression < 0 || compression > 9) + { + m_pScriptDebugging->LogCustom(luaVM, "Value for field 'compression' is out of range (0-9)"); + lua::Push(luaVM, false); + return 1; + } + } + + // Async + if (VERIFY_FUNCTION(luaFunctionRef)) + { + CLuaMain* pLuaMain = m_pLuaManager->GetVirtualMachine(luaVM); + if (pLuaMain) + { + CLuaShared::GetAsyncTaskScheduler()->PushTask( + [data, format, compression, strategy] + { + // Execute time-consuming task + SString output; + int result = SharedUtil::ZLibCompress(data, &output, format, compression, strategy); + if (result == Z_STREAM_END) + return std::make_pair(output, true); + else + return std::make_pair(SString("zlib error: %i", result), false); + }, + [luaFunctionRef](const std::pair& result) + { + CLuaMain* pLuaMain = m_pLuaManager->GetVirtualMachine(luaFunctionRef.GetLuaVM()); + if (pLuaMain) + { + CLuaArguments arguments; + if (result.second) + { + arguments.PushString(result.first); + arguments.Call(pLuaMain, luaFunctionRef); + } + else + { + m_pScriptDebugging->LogWarning(luaFunctionRef.GetLuaVM(), result.first.c_str()); + arguments.PushBoolean(false); + arguments.Call(pLuaMain, luaFunctionRef); + } + } + }); + + lua_pushboolean(luaVM, true); + } + } + else // Sync + { + SString output; + int result = SharedUtil::ZLibCompress(data, &output, format, compression, strategy); + if (result == Z_STREAM_END) + lua::Push(luaVM, output); + else + { + m_pScriptDebugging->LogWarning(luaVM, "zlib error: %i", result); + lua::Push(luaVM, false); + } + } + return 1; + } default: { m_pScriptDebugging->LogCustom(luaVM, "Unknown encryption algorithm"); @@ -753,7 +836,8 @@ int CLuaCryptDefs::DecodeString(lua_State* luaVM) argStream.ReadEnumString(algorithm); argStream.ReadString(data); - if ((algorithm != StringEncodeFunction::BASE64 && algorithm != StringEncodeFunction::BASE32) || argStream.NextIsTable()) + if ((algorithm != StringEncodeFunction::BASE64 && algorithm != StringEncodeFunction::BASE32 && algorithm != StringEncodeFunction::ZLIB) || + argStream.NextIsTable()) { argStream.ReadStringMap(options); } @@ -1084,6 +1168,70 @@ int CLuaCryptDefs::DecodeString(lua_State* luaVM) } return 1; } + case StringEncodeFunction::ZLIB: + { + ZLibFormat format = ZLibFormat::AUTO; + if (!options["format"].empty() && !StringToEnum(options["format"], format)) + { + m_pScriptDebugging->LogCustom(luaVM, "Not supported value for field 'format'"); + lua::Push(luaVM, false); + return 1; + } + + // Async + if (VERIFY_FUNCTION(luaFunctionRef)) + { + CLuaMain* pLuaMain = m_pLuaManager->GetVirtualMachine(luaVM); + if (pLuaMain) + { + CLuaShared::GetAsyncTaskScheduler()->PushTask( + [data, format] + { + // Execute time-consuming task + SString output; + int result = SharedUtil::ZLibUncompress(data, &output, format); + if (result == Z_STREAM_END) + return std::make_pair(output, true); + else + return std::make_pair(SString("zlib error: %i", result), false); + }, + [luaFunctionRef](const std::pair& result) + { + CLuaMain* pLuaMain = m_pLuaManager->GetVirtualMachine(luaFunctionRef.GetLuaVM()); + if (pLuaMain) + { + CLuaArguments arguments; + if (result.second) + { + arguments.PushString(result.first); + arguments.Call(pLuaMain, luaFunctionRef); + } + else + { + m_pScriptDebugging->LogWarning(luaFunctionRef.GetLuaVM(), result.first.c_str()); + arguments.PushBoolean(false); + arguments.Call(pLuaMain, luaFunctionRef); + } + } + }); + + lua_pushboolean(luaVM, true); + } + } + else // Sync + { + SString output; + int result = SharedUtil::ZLibUncompress(data, &output, format); + if (result == Z_STREAM_END) + lua::Push(luaVM, output); + else + { + m_pScriptDebugging->LogWarning(luaVM, "zlib error: %i", result); + lua::Push(luaVM, false); + } + } + return 1; + } default: { m_pScriptDebugging->LogCustom(luaVM, "Unknown encryption algorithm"); diff --git a/Shared/sdk/SharedUtil.Crypto.h b/Shared/sdk/SharedUtil.Crypto.h index a44cb22c5a0..ccd175b177b 100644 --- a/Shared/sdk/SharedUtil.Crypto.h +++ b/Shared/sdk/SharedUtil.Crypto.h @@ -17,6 +17,7 @@ #include #include #include +#include #include "SString.h" namespace SharedUtil @@ -202,4 +203,73 @@ namespace SharedUtil return result; } + + inline int ZLibCompress(const std::string input, std::string* output, const ZLibFormat format = ZLibFormat::GZIP, const int compression = 9, + const ZLibStrategy strategy = ZLibStrategy::DEFAULT) + { + z_stream stream{}; + + int result = deflateInit2(&stream, compression, Z_DEFLATED, (int)format, MAX_MEM_LEVEL, (int)strategy); + if (result != Z_OK) + return result; + + output->resize(deflateBound(&stream, input.size())); // resize to the upper bound of what the compressed size might be + + stream.next_out = (Bytef*)output->data(); + stream.avail_out = output->size(); + + stream.next_in = (z_const Bytef*)input.data(); + stream.avail_in = input.size(); + + result = deflate(&stream, Z_FINISH); + result |= deflateEnd(&stream); + + if (result == Z_STREAM_END) + output->resize(stream.total_out); // resize to the actual size + + return result; + } + + inline int ZLibUncompress(const std::string& input, std::string* output, const ZLibFormat format = ZLibFormat::AUTO) + { + int windowBits = (int)format; + if (format == ZLibFormat::AUTO) + { + if (input[0] == 0x78) + windowBits = (int)ZLibFormat::ZLIB; + else if (input[0] == 0x1F) + windowBits = (int)ZLibFormat::GZIP; + else + windowBits = (int)ZLibFormat::ZRAW; + } + z_stream stream{}; + + int result = inflateInit2(&stream, windowBits); + if (result != Z_OK) + return result; + + stream.next_in = (z_const Bytef*)input.data(); + stream.avail_in = input.size(); + + // Uncompress in chunks + std::string buffer; + buffer.resize(std::min(stream.avail_in, 128000U)); // use input length for chunk size (capped to 128k bytes which should be efficient enough) + while (true) + { + stream.next_out = (Bytef*)buffer.data(); + stream.avail_out = buffer.size(); + + result = inflate(&stream, Z_NO_FLUSH); + if (result != Z_OK && result != Z_STREAM_END) + break; + + output->append(buffer, 0, stream.total_out - output->size()); // append only what was written to buffer + + if (result == Z_STREAM_END) + break; + } + result |= inflateEnd(&stream); + return result; + } + } // namespace SharedUtil diff --git a/Shared/sdk/SharedUtil.Hash.h b/Shared/sdk/SharedUtil.Hash.h index 1aa28e82ceb..8981b56aef6 100644 --- a/Shared/sdk/SharedUtil.Hash.h +++ b/Shared/sdk/SharedUtil.Hash.h @@ -49,7 +49,8 @@ enum class StringEncodeFunction AES128, RSA, BASE64, - BASE32 + BASE32, + ZLIB, }; enum class KeyPairAlgorithm @@ -67,6 +68,23 @@ enum class HmacAlgorithm SHA512, }; +enum class ZLibFormat +{ + AUTO = 0, + ZRAW = -15, + ZLIB = 15, + GZIP = 31, +}; + +enum class ZLibStrategy +{ + DEFAULT, + FILTERED, + HUFFMAN_ONLY, + RLE, + FIXED, +}; + namespace SharedUtil { struct MD5 From 81160499a1e9e1a3b2c7beabdc59cc302aed70a3 Mon Sep 17 00:00:00 2001 From: samr46 Date: Sun, 9 Mar 2025 15:15:11 +0200 Subject: [PATCH 2/4] Improve signature check and allow numeric zlib wrapper format --- .../logic/luadefs/CLuaCryptDefs.cpp | 10 +++---- Shared/sdk/SharedUtil.Crypto.h | 26 +++++++++++++------ Shared/sdk/SharedUtil.Hash.h | 1 - 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp b/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp index b2d04a3523e..6786cbe485f 100644 --- a/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp +++ b/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp @@ -730,10 +730,10 @@ int CLuaCryptDefs::EncodeString(lua_State* luaVM) } case StringEncodeFunction::ZLIB: { - int compression = 9; - ZLibFormat format = ZLibFormat::GZIP; + int compression = 9; + int format = (int)ZLibFormat::GZIP; ZLibStrategy strategy = ZLibStrategy::DEFAULT; - if (!options["format"].empty() && !StringToEnum(options["format"], format)) + if (!options["format"].empty() && !StringToEnum(options["format"], (ZLibFormat&)format) && !StringToZLibFormat(options["format"], format)) { m_pScriptDebugging->LogCustom(luaVM, "Invalid value for field 'format'"); lua::Push(luaVM, false); @@ -1170,8 +1170,8 @@ int CLuaCryptDefs::DecodeString(lua_State* luaVM) } case StringEncodeFunction::ZLIB: { - ZLibFormat format = ZLibFormat::AUTO; - if (!options["format"].empty() && !StringToEnum(options["format"], format)) + int format = 0; + if (!options["format"].empty() && !StringToEnum(options["format"], (ZLibFormat&)format) && !StringToZLibFormat(options["format"], format)) { m_pScriptDebugging->LogCustom(luaVM, "Not supported value for field 'format'"); lua::Push(luaVM, false); diff --git a/Shared/sdk/SharedUtil.Crypto.h b/Shared/sdk/SharedUtil.Crypto.h index ccd175b177b..3de74ddb49c 100644 --- a/Shared/sdk/SharedUtil.Crypto.h +++ b/Shared/sdk/SharedUtil.Crypto.h @@ -204,12 +204,23 @@ namespace SharedUtil return result; } - inline int ZLibCompress(const std::string input, std::string* output, const ZLibFormat format = ZLibFormat::GZIP, const int compression = 9, + inline bool StringToZLibFormat(std::string format, int &outResult) + { + int value = atoi(format.c_str()); + if ((value >= 9 && value <= 31) || (value >= -15 && value <= -9)) // allowed values: 9..31, -9..-15 + { + outResult = value; + return true; + } + return false; + } + + inline int ZLibCompress(const std::string input, std::string* output, const int windowBits = (int)ZLibFormat::GZIP, const int compression = 9, const ZLibStrategy strategy = ZLibStrategy::DEFAULT) { z_stream stream{}; - int result = deflateInit2(&stream, compression, Z_DEFLATED, (int)format, MAX_MEM_LEVEL, (int)strategy); + int result = deflateInit2(&stream, compression, Z_DEFLATED, windowBits, MAX_MEM_LEVEL, (int)strategy); if (result != Z_OK) return result; @@ -230,15 +241,14 @@ namespace SharedUtil return result; } - inline int ZLibUncompress(const std::string& input, std::string* output, const ZLibFormat format = ZLibFormat::AUTO) + inline int ZLibUncompress(const std::string& input, std::string* output, int windowBits = 0) { - int windowBits = (int)format; - if (format == ZLibFormat::AUTO) + if (windowBits == 0 && input.size() >= 2) // try to determine format automatically { - if (input[0] == 0x78) - windowBits = (int)ZLibFormat::ZLIB; - else if (input[0] == 0x1F) + if (input[0] == '\x1F' && input[1] == '\x8B') windowBits = (int)ZLibFormat::GZIP; + else if (input[0] == '\x78') + windowBits = (int)ZLibFormat::ZLIB; else windowBits = (int)ZLibFormat::ZRAW; } diff --git a/Shared/sdk/SharedUtil.Hash.h b/Shared/sdk/SharedUtil.Hash.h index 8981b56aef6..0e4f6974781 100644 --- a/Shared/sdk/SharedUtil.Hash.h +++ b/Shared/sdk/SharedUtil.Hash.h @@ -70,7 +70,6 @@ enum class HmacAlgorithm enum class ZLibFormat { - AUTO = 0, ZRAW = -15, ZLIB = 15, GZIP = 31, From 839b8c9f9315ad61f2f39ef13c8ca6d718cd90d2 Mon Sep 17 00:00:00 2001 From: samr46 Date: Sun, 9 Mar 2025 15:33:04 +0200 Subject: [PATCH 3/4] Add missing const --- Shared/sdk/SharedUtil.Crypto.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Shared/sdk/SharedUtil.Crypto.h b/Shared/sdk/SharedUtil.Crypto.h index 3de74ddb49c..0a861c4ee4c 100644 --- a/Shared/sdk/SharedUtil.Crypto.h +++ b/Shared/sdk/SharedUtil.Crypto.h @@ -204,7 +204,7 @@ namespace SharedUtil return result; } - inline bool StringToZLibFormat(std::string format, int &outResult) + inline bool StringToZLibFormat(const std::string format, int &outResult) { int value = atoi(format.c_str()); if ((value >= 9 && value <= 31) || (value >= -15 && value <= -9)) // allowed values: 9..31, -9..-15 From 7f3a4b74fabd10eb7ad0aa9872c94d8a187e1ea1 Mon Sep 17 00:00:00 2001 From: samr46 Date: Tue, 18 Mar 2025 21:27:37 +0200 Subject: [PATCH 4/4] Pass variables by references --- .../deathmatch/logic/luadefs/CLuaCryptDefs.cpp | 8 ++++---- Shared/sdk/SharedUtil.Crypto.h | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp b/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp index 6786cbe485f..cd99d75a8f4 100644 --- a/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp +++ b/Shared/mods/deathmatch/logic/luadefs/CLuaCryptDefs.cpp @@ -767,7 +767,7 @@ int CLuaCryptDefs::EncodeString(lua_State* luaVM) { // Execute time-consuming task SString output; - int result = SharedUtil::ZLibCompress(data, &output, format, compression, strategy); + int result = SharedUtil::ZLibCompress(data, output, format, compression, strategy); if (result == Z_STREAM_END) return std::make_pair(output, true); else @@ -799,7 +799,7 @@ int CLuaCryptDefs::EncodeString(lua_State* luaVM) else // Sync { SString output; - int result = SharedUtil::ZLibCompress(data, &output, format, compression, strategy); + int result = SharedUtil::ZLibCompress(data, output, format, compression, strategy); if (result == Z_STREAM_END) lua::Push(luaVM, output); else @@ -1189,7 +1189,7 @@ int CLuaCryptDefs::DecodeString(lua_State* luaVM) { // Execute time-consuming task SString output; - int result = SharedUtil::ZLibUncompress(data, &output, format); + int result = SharedUtil::ZLibUncompress(data, output, format); if (result == Z_STREAM_END) return std::make_pair(output, true); else @@ -1221,7 +1221,7 @@ int CLuaCryptDefs::DecodeString(lua_State* luaVM) else // Sync { SString output; - int result = SharedUtil::ZLibUncompress(data, &output, format); + int result = SharedUtil::ZLibUncompress(data, output, format); if (result == Z_STREAM_END) lua::Push(luaVM, output); else diff --git a/Shared/sdk/SharedUtil.Crypto.h b/Shared/sdk/SharedUtil.Crypto.h index 0a861c4ee4c..a975bbf25e1 100644 --- a/Shared/sdk/SharedUtil.Crypto.h +++ b/Shared/sdk/SharedUtil.Crypto.h @@ -204,7 +204,7 @@ namespace SharedUtil return result; } - inline bool StringToZLibFormat(const std::string format, int &outResult) + inline bool StringToZLibFormat(const std::string& format, int& outResult) { int value = atoi(format.c_str()); if ((value >= 9 && value <= 31) || (value >= -15 && value <= -9)) // allowed values: 9..31, -9..-15 @@ -215,7 +215,7 @@ namespace SharedUtil return false; } - inline int ZLibCompress(const std::string input, std::string* output, const int windowBits = (int)ZLibFormat::GZIP, const int compression = 9, + inline int ZLibCompress(const std::string& input, std::string& output, const int windowBits = (int)ZLibFormat::GZIP, const int compression = 9, const ZLibStrategy strategy = ZLibStrategy::DEFAULT) { z_stream stream{}; @@ -224,10 +224,10 @@ namespace SharedUtil if (result != Z_OK) return result; - output->resize(deflateBound(&stream, input.size())); // resize to the upper bound of what the compressed size might be + output.resize(deflateBound(&stream, input.size())); // resize to the upper bound of what the compressed size might be - stream.next_out = (Bytef*)output->data(); - stream.avail_out = output->size(); + stream.next_out = (Bytef*)output.data(); + stream.avail_out = output.size(); stream.next_in = (z_const Bytef*)input.data(); stream.avail_in = input.size(); @@ -236,12 +236,12 @@ namespace SharedUtil result |= deflateEnd(&stream); if (result == Z_STREAM_END) - output->resize(stream.total_out); // resize to the actual size + output.resize(stream.total_out); // resize to the actual size return result; } - inline int ZLibUncompress(const std::string& input, std::string* output, int windowBits = 0) + inline int ZLibUncompress(const std::string& input, std::string& output, int windowBits = 0) { if (windowBits == 0 && input.size() >= 2) // try to determine format automatically { @@ -273,7 +273,7 @@ namespace SharedUtil if (result != Z_OK && result != Z_STREAM_END) break; - output->append(buffer, 0, stream.total_out - output->size()); // append only what was written to buffer + output.append(buffer, 0, stream.total_out - output.size()); // append only what was written to buffer if (result == Z_STREAM_END) break;