Merge pull request #319 from dynatrace-oss/xxh3-128

added xxh3_128 reference implementation and checksums, fixed use of d…

.gitattributes

@@ -1,5 +1,6 @@
 * text=auto
 *.java text eol=lf
+*.txt text eol=lf
 *.py text eol=lf
 *.md text eol=lf
 *.csv text

.gitmodules

@@ -37,3 +37,6 @@
 [submodule "reference-implementations/komihash_5_10/komihash"]
 	path = reference-implementations/komihash_5_10/komihash
 	url = https://github.com/avaneev/komihash.git
+[submodule "reference-implementations/xxh3_128/xxHash"]
+	path = reference-implementations/xxh3_128/xxHash
+	url = https://github.com/Cyan4973/xxHash.git

reference-implementations/build.sh

@@ -16,6 +16,7 @@ murmur3_32/murmur3_32_checksum_config.cpp \
 murmur3_128/murmur3_128_checksum_config.cpp \
 murmur3_128/smhasher/src/MurmurHash3.cpp \
 xxh3/xxh3_checksum_config.cpp \
+xxh3_128/xxh3_128_checksum_config.cpp \
 xxh3/xxHash/xxhash.c \
 farmna.a \
 farmuo.a \

reference-implementations/calculate_checksums.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2024 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 #include <random>
 #include <fstream>
 
-#include "openssl/sha.h"
+#include <openssl/evp.h>
 
 #include "komihash_4_3/komihash_4_3_checksum_config.hpp"
 #include "komihash_4_5/komihash_4_5_checksum_config.hpp"
@@ -32,6 +32,7 @@
 #include "farmhash_na/farmhash_na_checksum_config.hpp"
 #include "farmhash_uo/farmhash_uo_checksum_config.hpp"
 #include "xxh3/xxh3_checksum_config.hpp"
+#include "xxh3_128/xxh3_128_checksum_config.hpp"
 
 using namespace std;
 
@@ -76,10 +77,10 @@ void computeAndPrintChecksum(
 		if (dataLength > maxSupportedLength)
 			continue;
 
-		uint8_t checkSum[SHA256_DIGEST_LENGTH];
-		SHA256_CTX sha256;
+		uint8_t checkSum[EVP_MAX_MD_SIZE];
+		EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
 
-		SHA256_Init(&sha256);
+		EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL);
 
 		uint64_t seed = rng();
 		uint64_t rngState = seed;
@@ -107,16 +108,18 @@ void computeAndPrintChecksum(
 			hashFunctionConfig.calculateHash(seedBytes, &hashBytes[0],
 					dataBytes, dataLength);
 
-			SHA256_Update(&sha256, &hashBytes[0], hashBytes.size());
+			EVP_DigestUpdate(mdctx, &hashBytes[0], hashBytes.size());
 
 		}
 
-		SHA256_Final(checkSum, &sha256);
+		unsigned int lengthOfHash = 0;
+		EVP_DigestFinal_ex(mdctx, checkSum, &lengthOfHash);
+		EVP_MD_CTX_free(mdctx);
 
 		outputFile << dec << dataLength << ",";
 		outputFile << dec << numCycles << ",";
 		outputFile << hex << setfill('0') << setw(16) << seed << ",";
-		for (uint64_t k = 0; k < SHA256_DIGEST_LENGTH; ++k)
+		for (uint64_t k = 0; k < lengthOfHash; ++k)
 			outputFile << hex << setfill('0') << setw(2)
 					<< static_cast<uint64_t>(checkSum[k]);
 
@@ -142,6 +145,7 @@ int main(int argc, char *argv[]) {
 	computeAndPrintChecksum<FarmHashNaChecksumConfig>();
 	computeAndPrintChecksum<FarmHashUoChecksumConfig>();
 	computeAndPrintChecksum<XXH3ChecksumConfig>();
+	computeAndPrintChecksum<XXH3_128_ChecksumConfig>();
 
 	return 0;
 }

reference-implementations/farmhash_na/farmhash_na_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define FARMHASH_NA_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class FarmHashNaChecksumConfig {
 

reference-implementations/farmhash_uo/farmhash_uo_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define FARMHASH_UO_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class FarmHashUoChecksumConfig {
 

reference-implementations/komihash_4_3/komihash_4_3_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define KOMIHASH_4_3_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class Komihash4_3ChecksumConfig {
 

reference-implementations/komihash_4_5/komihash_4_5_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define KOMIHASH_4_5_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class Komihash4_5ChecksumConfig {
 

reference-implementations/komihash_4_7/komihash_4_7_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define KOMIHASH_4_7_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class Komihash4_7ChecksumConfig {
 

reference-implementations/komihash_5_0/komihash_5_0_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define KOMIHASH_5_0_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class Komihash5_0ChecksumConfig {
 

reference-implementations/komihash_5_10/komihash_5_10_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2024 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define KOMIHASH_5_10_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class Komihash5_10ChecksumConfig {
 

reference-implementations/murmur3_128/murmur3_128_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 #define MURMUR3_128_CHECKSUM_CONFIG_HPP
 
 #include <string>
-#include <cstring>
+#include <cstdint>
 
 class Murmur3_128_ChecksumConfig {
 

reference-implementations/murmur3_32/murmur3_32_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define MURMUR3_32_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class Murmur3_32_ChecksumConfig {
 

reference-implementations/polymur-hash_2_0/polymur-hash_2_0_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define POLYMURHASH_2_0_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class PolymurHash_2_0_ChecksumConfig {
 

reference-implementations/wyhash_final_3/wyhash_final_3_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define WYHASH_FINAL_3_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class WyhashFinal3ChecksumConfig {
 

reference-implementations/wyhash_final_4/wyhash_final_4_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define WYHASH_FINAL_4_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class WyhashFinal4ChecksumConfig {
 

reference-implementations/xxh3/xxh3_checksum_config.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Dynatrace LLC
+ * Copyright 2022-2025 Dynatrace LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 #define XXH3_CHECKSUM_CONFIG_HPP
 
 #include <string>
+#include <cstdint>
 
 class XXH3ChecksumConfig {
 

reference-implementations/xxh3_128/xxh3_128_checksum_config.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2025 Dynatrace LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "xxh3_128_checksum_config.hpp"
+#include "xxhash/xxhash.h"
+#include <cstring>
+
+void XXH3_128_ChecksumConfig::calculateHash(const uint8_t *seedBytes,
+		uint8_t *hashBytes, const uint8_t *dataBytes, uint64_t size) const {
+
+	uint64_t seed;
+	memcpy(&seed, seedBytes, 8);
+
+	XXH128_hash_t hash0 = XXH3_128bits((char*) (&dataBytes[0]), size);
+	XXH128_hash_t hash1 = XXH3_128bits_withSeed((char*) (&dataBytes[0]), size,
+			seed);
+
+	memcpy(hashBytes, &hash0, 16);
+	memcpy(hashBytes + 16, &hash1, 16);
+}

reference-implementations/xxh3_128/xxh3_128_checksum_config.hpp

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2025 Dynatrace LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef XXH3_128_CHECKSUM_CONFIG_HPP
+#define XXH3_128_CHECKSUM_CONFIG_HPP
+
+#include <string>
+#include <cstdint>
+
+class XXH3_128_ChecksumConfig {
+
+public:
+
+	uint64_t getSeedSize() const {
+		return 8;
+	}
+
+	uint64_t getHashSize() const {
+		return 32;
+	}
+
+	std::string getName() const {
+		return "XXH3_128";
+	}
+
+	void calculateHash(const uint8_t *seedBytes, uint8_t *hashBytes,
+			const uint8_t *dataBytes, uint64_t size) const;
+
+};
+
+#endif // XXH3_128_CHECKSUM_CONFIG_HPP