From 1bdf68b905b246ef4eb2959ba5e266005f761a8c Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Tue, 23 Jun 2020 20:48:01 -0700 Subject: [PATCH] Prepare for future release of XXH3 & XXH128. --- NEWS.md | 5 ++- checksum.c | 101 +++++++++++++++++++++++++++++++++++++++++++++-- lib/md-defines.h | 2 + 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index db01a3077..ae283d574 100644 --- a/NEWS.md +++ b/NEWS.md @@ -27,6 +27,9 @@ Protocol: 31 (unchanged) - Improved the man page a bit more. + - Preparing for an upcoming xxHash release that provides new XXH3 & XXH128 + hashing routines (disabled until their code is finalized). + ------------------------------------------------------------------------------ @@ -155,7 +158,7 @@ Protocol: 31 (unchanged) - Various checksum enhancements, including the optional use of openssl's MD4 & MD5 checksum algorithms, some x86-64 optimizations for the rolling checksum, some x86-64 optimizations for the (non-openssl) MD5 checksum, the addition - of xxhash checksum support, and a negotiation heuristic that ensures that it + of xxHash checksum support, and a negotiation heuristic that ensures that it is easier to add new checksum algorithms in the future. The environment variable `RSYNC_CHECKSUM_LIST` can be used to customize the preference order of the negotiation, or use `--checksum-choice` (`--cc`) to force a choice. diff --git a/checksum.c b/checksum.c index 125ee5b0b..824159b02 100644 --- a/checksum.c +++ b/checksum.c @@ -27,8 +27,12 @@ */ #include "rsync.h" + #ifdef SUPPORT_XXHASH #include "xxhash.h" +# if XXH_VERSION_NUMBER >= 800 +# define SUPPORT_XXH3 1 +# endif #endif extern int am_server; @@ -40,6 +44,10 @@ extern const char *checksum_choice; struct name_num_obj valid_checksums = { "checksum", NULL, NULL, 0, 0, { +#ifdef SUPPORT_XXH3 + { CSUM_XXH3_128, "xxh128", NULL }, + { CSUM_XXH3_64, "xxh3", NULL }, +#endif #ifdef SUPPORT_XXHASH { CSUM_XXH64, "xxh64", NULL }, { CSUM_XXH64, "xxhash", NULL }, @@ -135,10 +143,11 @@ int csum_len_for_type(int cst, BOOL flist_csum) return MD4_DIGEST_LEN; case CSUM_MD5: return MD5_DIGEST_LEN; -#ifdef SUPPORT_XXHASH case CSUM_XXH64: + case CSUM_XXH3_64: return 64/8; -#endif + case CSUM_XXH3_128: + return 128/8; default: /* paranoia to prevent missing case values */ exit_cleanup(RERR_UNSUPPORTED); } @@ -160,10 +169,10 @@ int canonical_checksum(int csum_type) case CSUM_MD4: case CSUM_MD5: return -1; -#ifdef SUPPORT_XXHASH case CSUM_XXH64: + case CSUM_XXH3_64: + case CSUM_XXH3_128: return 1; -#endif default: /* paranoia to prevent missing case values */ exit_cleanup(RERR_UNSUPPORTED); } @@ -200,6 +209,17 @@ void get_checksum2(char *buf, int32 len, char *sum) case CSUM_XXH64: SIVAL64(sum, 0, XXH64(buf, len, checksum_seed)); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + SIVAL64(sum, 0, XXH3_64bits_withSeed(buf, len, checksum_seed)); + break; + case CSUM_XXH3_128: { + XXH128_hash_t digest = XXH3_128bits_withSeed(buf, len, checksum_seed); + SIVAL64(sum, 0, digest.low64); + SIVAL64(sum, 8, digest.high64); + break; + } #endif case CSUM_MD5: { MD5_CTX m5; @@ -315,6 +335,45 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) SIVAL64(sum, 0, XXH64_digest(state)); break; } +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: { + static XXH3_state_t* state = NULL; + if (!state && !(state = XXH3_createState())) + out_of_memory("file_checksum"); + + XXH3_64bits_reset(state); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); + + remainder = (int32)(len - i); + if (remainder > 0) + XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); + + SIVAL64(sum, 0, XXH3_64bits_digest(state)); + break; + } + case CSUM_XXH3_128: { + XXH128_hash_t digest; + static XXH3_state_t* state = NULL; + if (!state && !(state = XXH3_createState())) + out_of_memory("file_checksum"); + + XXH3_128bits_reset(state); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); + + remainder = (int32)(len - i); + if (remainder > 0) + XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); + + digest = XXH3_128bits_digest(state); + SIVAL64(sum, 0, digest.low64); + SIVAL64(sum, 8, digest.high64); + break; + } #endif case CSUM_MD5: { MD5_CTX m5; @@ -391,6 +450,9 @@ static union { #ifdef SUPPORT_XXHASH static XXH64_state_t* xxh64_state; #endif +#ifdef SUPPORT_XXH3 +static XXH3_state_t* xxh3_state; +#endif static int cursum_type; void sum_init(int csum_type, int seed) @@ -408,6 +470,18 @@ void sum_init(int csum_type, int seed) out_of_memory("sum_init"); XXH64_reset(xxh64_state, 0); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + if (!xxh3_state && !(xxh3_state = XXH3_createState())) + out_of_memory("sum_init"); + XXH3_64bits_reset(xxh3_state); + break; + case CSUM_XXH3_128: + if (!xxh3_state && !(xxh3_state = XXH3_createState())) + out_of_memory("sum_init"); + XXH3_128bits_reset(xxh3_state); + break; #endif case CSUM_MD5: MD5_Init(&ctx.m5); @@ -450,6 +524,14 @@ void sum_update(const char *p, int32 len) case CSUM_XXH64: XXH64_update(xxh64_state, p, len); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + XXH3_64bits_update(xxh3_state, p, len); + break; + case CSUM_XXH3_128: + XXH3_128bits_update(xxh3_state, p, len); + break; #endif case CSUM_MD5: MD5_Update(&ctx.m5, (uchar *)p, len); @@ -504,6 +586,17 @@ int sum_end(char *sum) case CSUM_XXH64: SIVAL64(sum, 0, XXH64_digest(xxh64_state)); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + SIVAL64(sum, 0, XXH3_64bits_digest(xxh3_state)); + break; + case CSUM_XXH3_128: { + XXH128_hash_t digest = XXH3_128bits_digest(xxh3_state); + SIVAL64(sum, 0, digest.low64); + SIVAL64(sum, 8, digest.high64); + break; + } #endif case CSUM_MD5: MD5_Final((uchar *)sum, &ctx.m5); diff --git a/lib/md-defines.h b/lib/md-defines.h index b92e8c075..1410af5fa 100644 --- a/lib/md-defines.h +++ b/lib/md-defines.h @@ -13,3 +13,5 @@ #define CSUM_MD4 4 #define CSUM_MD5 5 #define CSUM_XXH64 6 +#define CSUM_XXH3_64 7 +#define CSUM_XXH3_128 8