wip

.github/workflows/test.yml

@@ -25,7 +25,7 @@ jobs:
           registry-url: 'https://registry.npmjs.org'
 
       - name: Build with Node.js ${{ matrix.node }} on ${{ matrix.os }}
-        run: npm install && npm run compile
+        run: npm install --ignore-scripts && bash etc/install-zstd.sh && npm run compile
         shell: bash
 
       - name: Test ${{ matrix.os }}

binding.gyp

@@ -4,28 +4,37 @@
     'type': 'loadable_module',
     'defines': ['ZSTD_STATIC_LINKING_ONLY'],
     'include_dirs': [
+        "<(module_root_dir)/deps/zstd/lib",
         "<!(node -p \"require('node-addon-api').include_dir\")"
     ],
     'variables': {
       'ARCH': '<(host_arch)',
-      'libmongocrypt_link_type%': 'static',
-      'mongocrypt_avoid_openssl_crypto%': 'false',
       'built_with_electron%': 0
     },
     'sources': [
-      'src/addon.cpp'
+      'src/addon.cpp',
+      'src/napi_utils.h',
+      'src/compressor.h',
+      'src/decompressor.h',
+      'src/compression_worker.h'
     ],
     'xcode_settings': {
       'GCC_ENABLE_CPP_EXCEPTIONS': 'YES',
       'CLANG_CXX_LIBRARY': 'libc++',
-      'MACOSX_DEPLOYMENT_TARGET': '10.12',
+      # mongodb-client-encryption targets 10.12, but our [email protected] targeted 10.13.
+      'MACOSX_DEPLOYMENT_TARGET': '10.13',
       'GCC_SYMBOLS_PRIVATE_EXTERN': 'YES', # -fvisibility=hidden
     },
     'cflags!': [ '-fno-exceptions' ],
     'cflags_cc!': [ '-fno-exceptions' ],
     'cflags_cc': [  '-std=c++17' ],
     'msvs_settings': {
       'VCCLCompilerTool': { 'ExceptionHandling': 1 },
-    }
+    },
+	  'link_settings': {
+      'libraries': [
+        '<(module_root_dir)/deps/zstd/lib/libzstd.a',
+      ]
+    },
   }]
 }

etc/install-zstd.sh

@@ -5,8 +5,11 @@ mkdir -p deps/zstd
 
 curl -L "https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz" | tar  -zxf - -C deps/zstd --strip-components 1
 
-cd deps/zstd/
+build_zstd() {
+	export MACOSX_DEPLOYMENT_TARGET=10.12
+	cd deps/zstd/
 
-echo "cwd"
-ls lib
-make --debug CXXFLAGS="-mmacosx-version-min=10.12"
+	make --debug CXXFLAGS="-mmacosx-version-min=10.12"
+}
+
+build_zstd

index.d.ts

@@ -1,2 +1,2 @@
-export declare function compress(data: Buffer, level?: number | undefined | null): Promise<Buffer>
-export declare function decompress(data: Buffer): Promise<Buffer>
+export declare function compress(data: Buffer, level?: number | undefined | null): Promise<Buffer>;
+export declare function decompress(data: Buffer): Promise<Buffer>;

src/addon.cpp

@@ -1,14 +1,70 @@
 #include <napi.h>
 
+#include <string>
+#include <vector>
+
+#include "compression_worker.h"
+#include "napi_utils.h"
+#include "compressor.h"
+#include "decompressor.h"
+
+#include "zstd.h"
+
 using namespace Napi;
 
-Napi::String Compress(const Napi::CallbackInfo& info) {
-    auto string = Napi::String::New(info.Env(), "compress()");
-    return string;
+Napi::Promise Compress(const Napi::CallbackInfo& info) {
+    auto number_of_args = info.Length();
+    size_t compression_level;
+    Napi::Uint8Array to_compress;
+
+    if (number_of_args == 0) {
+        std::string error_message =
+            "compress(uint8array) or compress(uint8array, compression level)";
+        throw TypeError::New(info.Env(), error_message);
+    } else if (number_of_args == 1) {
+        to_compress = Uint8ArrayFromValue(info[0], "buffer");
+        compression_level = 3;
+    } else if (number_of_args == 2) {
+        to_compress = Uint8ArrayFromValue(info[0], "buffer");
+        if (!info[1].IsNumber()) {
+            throw TypeError::New(info.Env(),
+                                 std::string("if provided, compression_level must be a number."));
+        }
+        compression_level = (size_t)info[1].ToNumber().Int32Value();
+    } else {
+        std::string error_message =
+            "compress(uint8array) or compress(uint8array, compression level)";
+        throw TypeError::New(info.Env(), error_message);
+    }
+
+    Compressor compressor = Compressor::fromUint8Array(to_compress, compression_level);
+    Worker<Compressor>* worker = new Worker<Compressor>(info.Env(), std::move(compressor));
+
+    worker->Queue();
+
+    return worker->GetPromise();
 }
-Napi::String Decompress(const Napi::CallbackInfo& info) {
-    auto string = Napi::String::New(info.Env(), "decompress()");
-    return string;
+
+Napi::Promise Decompress(const CallbackInfo& info) {
+    auto number_of_args = info.Length();
+    Napi::Uint8Array compressed_data;
+
+    if (number_of_args == 0) {
+        std::string error_message = "decompress(uint8array)";
+        throw TypeError::New(info.Env(), error_message);
+    } else if (number_of_args == 1) {
+        compressed_data = Uint8ArrayFromValue(info[0], "buffer");
+    } else {
+        std::string error_message = "decompress(uint8array)";
+        throw TypeError::New(info.Env(), error_message);
+    }
+
+    Decompressor decompressor = Decompressor::fromUint8Array(compressed_data);
+    Worker<Decompressor>* worker = new Worker<Decompressor>(info.Env(), decompressor);
+
+    worker->Queue();
+
+    return worker->GetPromise();
 }
 
 Napi::Object Init(Napi::Env env, Napi::Object exports) {

src/compression_worker.h

@@ -4,6 +4,10 @@
 
 using namespace Napi;
 
+/**
+ * @brief A class that represents the result of a compression operation.  Once the MACOS_DEPLOYMENT_TARGET can be raised to 10.13 and use 
+ * a c++17, we can remove this class and use a std::optional<std::variant<std::vector<uint8_t>, std::string>>> instead.
+ */
 struct CompressionResult {
     CompressionResult(std::string error,
                       std::vector<uint8_t> result,
@@ -37,6 +41,11 @@ struct CompressionResult {
     bool initialized;
 };
 
+/**
+ * @brief An asynchronous Napi::Worker that can be with any functor that produces CompressionResults.
+ * 
+ * @tparam TWorker - The functor to call asynchronously.
+ */
 template <typename TWorker>
 class Worker : public Napi::AsyncWorker {
    public:
@@ -59,15 +68,20 @@ class Worker : public Napi::AsyncWorker {
         if (!result.initialized) {
             m_deferred.Reject(
                 Napi::Error::New(
-                    Env(), "Decompression runtime error - did not receive a result or a value.")
+                    Env(), "zstd runtime error - async worker finished without a compression or decompression result.")
                     .Value());
         } else if (result.hasError) {
             m_deferred.Reject(Napi::Error::New(Env(), result.error).Value());
-        } else {
+        } else if (result.hasResult) {
             Uint8Array output = Uint8Array::New(m_deferred.Env(), result.result.size());
             std::copy(result.result.begin(), result.result.end(), output.Data());
 
             m_deferred.Resolve(output);
+        } else {
+            m_deferred.Reject(
+            Napi::Error::New(
+                Env(), "zstd runtime error - async worker finished without a compression or decompression result.")
+                .Value());
         }
     }
 

src/compressor.h

@@ -0,0 +1,43 @@
+
+#include <vector>
+#include <napi.h>
+
+#include "zstd.h"
+
+using namespace Napi;
+
+struct Compressor {
+    std::vector<uint8_t> data;
+    size_t compression_level;
+
+    Compressor(std::vector<uint8_t> data, size_t compression_level)
+        : data(data), compression_level(compression_level) {}
+
+    CompressionResult operator()() {
+        size_t output_buffer_size = ZSTD_compressBound(data.size());
+        std::vector<uint8_t> output(output_buffer_size);
+
+        size_t result_code = ZSTD_compress(
+            output.data(), output.size(), data.data(), data.size(), compression_level);
+
+        if (ZSTD_isError(result_code)) {
+            std::string error(ZSTD_getErrorName(result_code));
+            return CompressionResult::Error(error);
+        }
+
+        output.resize(result_code);
+
+        return CompressionResult::Ok(output);
+    }
+
+    static Compressor fromUint8Array(const Uint8Array& to_compress, size_t compression_level) {
+        const uint8_t* input_data = to_compress.Data();
+        size_t total = to_compress.ElementLength();
+
+        std::vector<uint8_t> data(to_compress.ElementLength());
+
+        std::copy(input_data, input_data + total, data.data());
+
+        return Compressor(std::move(data), compression_level);
+    }
+};

src/decompressor.h

@@ -0,0 +1,43 @@
+#include <napi.h>
+
+#include <vector>
+
+
+
+#include "zstd.h"
+
+using namespace Napi;
+
+struct Decompressor {
+    std::vector<uint8_t> data;
+    size_t buffer_size;
+
+    Decompressor(std::vector<uint8_t> data, size_t buffer_size)
+        : data(data), buffer_size(buffer_size) {}
+
+    CompressionResult operator()() {
+        std::vector<uint8_t> decompressed(buffer_size);
+
+        size_t _result =
+            ZSTD_decompress(decompressed.data(), decompressed.size(), data.data(), data.size());
+
+        if (ZSTD_isError(_result)) {
+            std::string error(ZSTD_getErrorName(_result));
+            return CompressionResult::Error(error);
+        }
+
+        decompressed.resize(_result);
+
+        return CompressionResult::Ok(decompressed);
+    }
+
+    static Decompressor fromUint8Array(const Uint8Array& compressed_data) {
+        const uint8_t* input_data = compressed_data.Data();
+        size_t total = compressed_data.ElementLength();
+
+        std::vector<uint8_t> data(total);
+        std::copy(input_data, input_data + total, data.data());
+
+        return Decompressor(data, total * 1000);
+    }
+};

src/napi_utils.h

@@ -2,13 +2,25 @@
 
 using namespace Napi;
 
+/**
+ * @brief Given a T* source and a T* destination, copies count 
+ * elements from source into destination.
+ */
 template <typename T>
 void copy_buffer_data(T* source, T* dest, size_t count) {
     for (size_t i = 0; i < count; ++i) {
         dest[i] = source[i];
     }
 }
 
+/**
+ * @brief Given an Napi;:Value, this function returns the value as a Uint8Array, if the 
+ * Value is a Uint8Array. Otherwise, this function throws.
+ * 
+ * @param v - An Napi::Value
+ * @param argument_name - the name of the value, to use when constructing an error message.
+ * @return Napi::Uint8Array 
+ */
 Uint8Array Uint8ArrayFromValue(Value v, std::string argument_name) {
     if (!v.IsTypedArray() || v.As<TypedArray>().TypedArrayType() != napi_uint8_array) {
         std::string error_message = "Parameter `" + argument_name + "` must be a Uint8Array.";