diff --git a/CMakeLists.txt b/CMakeLists.txt index 3cfa379f7..85f6bbef6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,42 @@ if(USE_FPGAOPENCL) add_definitions(-DUSE_FPGAOPENCL) endif() + +# HDFS library +# look through provided CMAKE_PREFIX_PATHs +option(USE_HDFS "Whether to activate compilation of HDSF support" OFF) +if(USE_HDFS) + foreach(path ${CMAKE_PREFIX_PATH}) + if(NOT DEFINED HDFS_LIB_FOUND) + set(HDFS_LIB_LOCATION ${path}/hdfs) + if(EXISTS ${HDFS_LIB_LOCATION}) + set(HDFS_LIB_FOUND TRUE) + else() + unset(HDFS_LIB_LOCATION) + unset(HDFS_LIB_FOUND) + endif() + endif() + endforeach(path) + + # fallback if not using CMAKE_PREFIX_PATH (e.g., system/container install) + if(NOT DEFINED CMAKE_PREFIX_PATH OR NOT DEFINED HDFS_LIB_FOUND) + set(HDFS_LIB_LOCATION /usr/local/include/hdfs) + if(EXISTS ${HDFS_LIB_LOCATION}) + set(HDFS_LIB_FOUND TRUE) + else() + unset(HDFS_LIB_LOCATION) + endif() + endif() + + include_directories(${PROJECT_SOURCE_DIR} ${HDFS_LIB_LOCATION}) + find_library(hdfs3 NAMES libhdfs3.so HINTS ${PROJECT_BINARY_DIR}/installed/lib REQUIRED) + message(STATUS "HDFS_LIB_FOUND: ${HDFS_LIB_FOUND}") + if(DEFINED HDFS_LIB_FOUND) + add_definitions(-DUSE_HDFS) + endif() +endif() + + set(CMAKE_VERBOSE_MAKEFILE ON) # ***************************************************************************** diff --git a/UserConfig.json b/UserConfig.json index 855c07ad3..316c22938 100644 --- a/UserConfig.json +++ b/UserConfig.json @@ -27,6 +27,9 @@ "taskPartitioningScheme": "STATIC", "numberOfThreads": -1, "minimumTaskSize": 1, + "useHdfs": false, + "hdfsAddress": "", + "hdfsUsername": "", "libdir": "{exedir}/../lib", "daphnedsl_import_paths": {}, "force_cuda": false, diff --git a/src/api/cli/DaphneUserConfig.h b/src/api/cli/DaphneUserConfig.h index 58f5aea4d..071e15812 100644 --- a/src/api/cli/DaphneUserConfig.h +++ b/src/api/cli/DaphneUserConfig.h @@ -86,7 +86,12 @@ struct DaphneUserConfig { size_t max_distributed_serialization_chunk_size = std::numeric_limits::max() - 1024; // 2GB (-1KB to make up for gRPC headers etc.) - which is the maximum size allowed by gRPC / MPI. TODO: Investigate what might be the optimal. int numberOfThreads = -1; int minimumTaskSize = 1; - + + // hdfs + bool use_hdfs = false; + std::string hdfs_Address = ""; + std::string hdfs_username = ""; + // minimum considered log level (e.g., no logging below ERROR (essentially suppressing WARN, INFO, DEBUG and TRACE) spdlog::level::level_enum log_level_limit = spdlog::level::err; std::vector loggers; diff --git a/src/api/internal/daphne_internal.cpp b/src/api/internal/daphne_internal.cpp index fc8daf3de..d406f954d 100644 --- a/src/api/internal/daphne_internal.cpp +++ b/src/api/internal/daphne_internal.cpp @@ -132,6 +132,7 @@ int startDAPHNE(int argc, const char** argv, DaphneLibResult* daphneLibRes, int static OptionCategory daphneOptions("DAPHNE Options"); static OptionCategory schedulingOptions("Advanced Scheduling Knobs"); static OptionCategory distributedBackEndSetupOptions("Distributed Backend Knobs"); + static OptionCategory HDFSOptions("HDFS Knobs"); // Options ---------------------------------------------------------------- @@ -154,6 +155,22 @@ int startDAPHNE(int argc, const char** argv, DaphneLibResult* daphneLibRes, int init(std::numeric_limits::max() - 1024) ); + // HDFS knobs + static opt use_hdfs( + "enable-hdfs", cat(HDFSOptions), + desc("Enable HDFS filesystem") + ); + static opt hdfs_Address( + "hdfs-ip", cat(HDFSOptions), + desc("IP of the HDFS filesystem (including port)."), + init("") + ); + static opt hdfs_username( + "hdfs-username", cat(HDFSOptions), + desc("Username of the HDFS filesystem."), + init("") + ); + // Scheduling options @@ -404,6 +421,7 @@ int startDAPHNE(int argc, const char** argv, DaphneLibResult* daphneLibRes, int visibleCategories.push_back(&daphneOptions); visibleCategories.push_back(&schedulingOptions); visibleCategories.push_back(&distributedBackEndSetupOptions); + visibleCategories.push_back(&HDFSOptions); HideUnrelatedOptions(visibleCategories); @@ -484,7 +502,27 @@ int startDAPHNE(int argc, const char** argv, DaphneLibResult* daphneLibRes, int if(user_config.distributedBackEndSetup!=ALLOCATION_TYPE::DIST_MPI && user_config.distributedBackEndSetup!=ALLOCATION_TYPE::DIST_GRPC_SYNC && user_config.distributedBackEndSetup!=ALLOCATION_TYPE::DIST_GRPC_ASYNC) spdlog::warn("No backend has been selected. Wiil use the default 'MPI'"); } - user_config.max_distributed_serialization_chunk_size = maxDistrChunkSize; + user_config.max_distributed_serialization_chunk_size = maxDistrChunkSize; + + // only overwrite with non-defaults + if (use_hdfs) { + user_config.use_hdfs = use_hdfs; + } + if (hdfs_Address != "") { + user_config.hdfs_Address = hdfs_Address; + } + if (hdfs_username != "") { + user_config.hdfs_username = hdfs_username; + } + if (user_config.use_hdfs && (user_config.hdfs_Address == "" || user_config.hdfs_username == "")){ + spdlog::warn("HDFS is enabled, but the HDFS IP address or username were not provided."); + } +#ifndef USE_HDFS + if (user_config.use_hdfs){ + throw std::runtime_error("you are trying to use HDFS, but Daphne was not build with --hdfs option\n"); + } +#endif + for (auto explain : explainArgList) { switch (explain) { case kernels: diff --git a/src/compiler/lowering/InsertDaphneContextPass.cpp b/src/compiler/lowering/InsertDaphneContextPass.cpp index ecb4469b2..c801cda8f 100644 --- a/src/compiler/lowering/InsertDaphneContextPass.cpp +++ b/src/compiler/lowering/InsertDaphneContextPass.cpp @@ -63,6 +63,11 @@ void InsertDaphneContextPass::runOnOperation() if (user_config.use_distributed){ builder.create(loc); } +#ifdef USE_HDFS + if(user_config.use_hdfs) { + builder.create(loc); + } +#endif #ifdef USE_FPGAOPENCL if(user_config.use_fpgaopencl) { builder.create(loc); diff --git a/src/ir/daphneir/DaphneOps.td b/src/ir/daphneir/DaphneOps.td index 00a907d24..d934f8463 100644 --- a/src/ir/daphneir/DaphneOps.td +++ b/src/ir/daphneir/DaphneOps.td @@ -1537,6 +1537,11 @@ def Daphne_CreateDistributedContextOp : Daphne_Op<"createDistributedContext", [] let results = (outs); } +def Daphne_CreateHDFSContextOp : Daphne_Op<"createHDFSContext", []> { + let arguments = (ins); + let results = (outs); +} + def Daphne_CreateFPGAContextOp : Daphne_Op<"createFPGAContext", [FPGAOPENCLSupport]> { let arguments = (ins); let results = (outs); diff --git a/src/parser/config/ConfigParser.cpp b/src/parser/config/ConfigParser.cpp index d21c3c1c1..2040286e6 100644 --- a/src/parser/config/ConfigParser.cpp +++ b/src/parser/config/ConfigParser.cpp @@ -111,6 +111,12 @@ void ConfigParser::readUserConfig(const std::string& filename, DaphneUserConfig& config.numberOfThreads = jf.at(DaphneConfigJsonParams::NUMBER_OF_THREADS).get(); if (keyExists(jf, DaphneConfigJsonParams::MINIMUM_TASK_SIZE)) config.minimumTaskSize = jf.at(DaphneConfigJsonParams::MINIMUM_TASK_SIZE).get(); + if (keyExists(jf, DaphneConfigJsonParams::USE_HDFS_)) + config.use_hdfs = jf.at(DaphneConfigJsonParams::USE_HDFS_).get(); + if (keyExists(jf, DaphneConfigJsonParams::HDFS_ADDRESS)) + config.hdfs_Address = jf.at(DaphneConfigJsonParams::HDFS_ADDRESS).get(); + if (keyExists(jf, DaphneConfigJsonParams::HDFS_USERNAME)) + config.hdfs_username = jf.at(DaphneConfigJsonParams::HDFS_USERNAME).get(); #ifdef USE_CUDA if (keyExists(jf, DaphneConfigJsonParams::CUDA_DEVICES)) config.cuda_devices = jf.at(DaphneConfigJsonParams::CUDA_DEVICES).get>(); diff --git a/src/parser/config/JsonParams.h b/src/parser/config/JsonParams.h index 9816ce768..3a8b9d8af 100644 --- a/src/parser/config/JsonParams.h +++ b/src/parser/config/JsonParams.h @@ -57,6 +57,9 @@ struct DaphneConfigJsonParams { inline static const std::string TASK_PARTITIONING_SCHEME = "taskPartitioningScheme"; inline static const std::string NUMBER_OF_THREADS = "numberOfThreads"; inline static const std::string MINIMUM_TASK_SIZE = "minimumTaskSize"; + inline static const std::string USE_HDFS_ = "useHdfs"; + inline static const std::string HDFS_ADDRESS = "hdfsAddress"; + inline static const std::string HDFS_USERNAME = "hdfsUsername"; inline static const std::string CUDA_DEVICES = "cuda_devices"; inline static const std::string LIB_DIR = "libdir"; inline static const std::string DAPHNEDSL_IMPORT_PATHS = "daphnedsl_import_paths"; @@ -97,6 +100,9 @@ struct DaphneConfigJsonParams { TASK_PARTITIONING_SCHEME, NUMBER_OF_THREADS, MINIMUM_TASK_SIZE, + USE_HDFS_, + HDFS_ADDRESS, + HDFS_USERNAME, CUDA_DEVICES, LIB_DIR, DAPHNEDSL_IMPORT_PATHS, diff --git a/src/parser/metadata/JsonKeys.h b/src/parser/metadata/JsonKeys.h index 43bbf48cb..d91c80f44 100644 --- a/src/parser/metadata/JsonKeys.h +++ b/src/parser/metadata/JsonKeys.h @@ -36,6 +36,11 @@ struct JsonKeys { // optional key inline static const std::string NUM_NON_ZEROS = "numNonZeros"; // int (default: -1) + inline static const std::string HDFS = "hdfs"; // json + struct HDFSKeys { + inline static const std::string isHDFS = "isHDFS"; + inline static const std::string HDFSFilename = "HDFSFilename"; + }; }; #endif \ No newline at end of file diff --git a/src/parser/metadata/MetaDataParser.cpp b/src/parser/metadata/MetaDataParser.cpp index 628c3e098..300a911a7 100644 --- a/src/parser/metadata/MetaDataParser.cpp +++ b/src/parser/metadata/MetaDataParser.cpp @@ -18,14 +18,20 @@ #include #include +#include +#include FileMetaData MetaDataParser::readMetaData(const std::string& filename_) { std::string metaFilename = filename_ + ".meta"; std::ifstream ifs(metaFilename, std::ios::in); if (!ifs.good()) throw std::runtime_error("Could not open file '" + metaFilename + "' for reading meta data."); - - nlohmann::json jf = nlohmann::json::parse(ifs); + std::stringstream buffer; + buffer << ifs.rdbuf(); + return MetaDataParser::readMetaDataFromString(buffer.str()); +} +FileMetaData MetaDataParser::readMetaDataFromString(const std::string& str) { + nlohmann::json jf = nlohmann::json::parse(str); if (!keyExists(jf, JsonKeys::NUM_ROWS) || !keyExists(jf, JsonKeys::NUM_COLS)) { throw std::invalid_argument("A meta data JSON file should always contain \"" + JsonKeys::NUM_ROWS + "\" and \"" @@ -34,14 +40,21 @@ FileMetaData MetaDataParser::readMetaData(const std::string& filename_) { const size_t numRows = jf.at(JsonKeys::NUM_ROWS).get(); const size_t numCols = jf.at(JsonKeys::NUM_COLS).get(); + const bool isHDFS = (keyExists(jf, JsonKeys::HDFS)); const bool isSingleValueType = !(keyExists(jf, JsonKeys::SCHEMA)); const ssize_t numNonZeros = (keyExists(jf, JsonKeys::NUM_NON_ZEROS)) ? jf.at(JsonKeys::NUM_NON_ZEROS).get() : -1; + HDFSMetaData hdfs; + if (isHDFS){ + // TODO check if key exist and throw errors if not + hdfs.isHDFS = jf.at(JsonKeys::HDFS)["isHDFS"];; + hdfs.HDFSFilename = jf.at(JsonKeys::HDFS)["HDFSFilename"]; + } if (isSingleValueType) { if (keyExists(jf, JsonKeys::VALUE_TYPE)) { ValueTypeCode vtc = jf.at(JsonKeys::VALUE_TYPE).get(); - return {numRows, numCols, isSingleValueType, vtc, numNonZeros}; + return {numRows, numCols, isSingleValueType, vtc, numNonZeros, hdfs}; } else { throw std::invalid_argument("A (matrix) meta data JSON file should contain the \"" + JsonKeys::VALUE_TYPE @@ -68,7 +81,7 @@ FileMetaData MetaDataParser::readMetaData(const std::string& filename_) { schema.emplace_back(vtc); labels.emplace_back(column.getLabel()); } - return {numRows, numCols, isSingleValueType, schema, labels, numNonZeros}; + return {numRows, numCols, isSingleValueType, schema, labels, numNonZeros, hdfs}; } else { throw std::invalid_argument("A (frame) meta data JSON file should contain the \"" + JsonKeys::SCHEMA @@ -77,39 +90,50 @@ FileMetaData MetaDataParser::readMetaData(const std::string& filename_) { } } +std::string MetaDataParser::writeMetaDataToString(const FileMetaData& metaData) { + nlohmann::json json; + + json[JsonKeys::NUM_ROWS] = metaData.numRows; + json[JsonKeys::NUM_COLS] = metaData.numCols; + + if (metaData.isSingleValueType) { + if (metaData.schema.size() != 1) + throw std::runtime_error("inappropriate meta data tried to be written to file"); + json[JsonKeys::VALUE_TYPE] = metaData.schema[0]; + } + else { + std::vector schemaColumns; + // assume that the schema and labels are the same lengths + for (unsigned int i = 0; i < metaData.schema.size(); i++) { + SchemaColumn schemaColumn; + schemaColumn.setLabel(metaData.labels[i]); + schemaColumn.setValueType(metaData.schema[i]); + schemaColumns.emplace_back(schemaColumn); + } + json[JsonKeys::SCHEMA] = schemaColumns; + } + + if (metaData.numNonZeros != -1) + json[JsonKeys::NUM_NON_ZEROS] = metaData.numNonZeros; + + // HDFS + if (metaData.hdfs.isHDFS){ + json[JsonKeys::HDFS][JsonKeys::HDFSKeys::isHDFS] = metaData.hdfs.isHDFS; + std::filesystem::path filePath(metaData.hdfs.HDFSFilename); + auto baseFileName = filePath.filename().string(); + + json[JsonKeys::HDFS][JsonKeys::HDFSKeys::HDFSFilename] = "/" + baseFileName; + } + return json.dump(); +} void MetaDataParser::writeMetaData(const std::string& filename_, const FileMetaData& metaData) { - std::string metaFilename = filename_ + ".meta"; + std::string metaFilename = filename_ + ".meta"; std::ofstream ofs(metaFilename, std::ios::out); if (!ofs.good()) throw std::runtime_error("could not open file '" + metaFilename + "' for writing meta data"); if(ofs.is_open()) { - nlohmann::json json; - - json[JsonKeys::NUM_ROWS] = metaData.numRows; - json[JsonKeys::NUM_COLS] = metaData.numCols; - - if (metaData.isSingleValueType) { - if (metaData.schema.size() != 1) - throw std::runtime_error("inappropriate meta data tried to be written to file"); - json[JsonKeys::VALUE_TYPE] = metaData.schema[0]; - } - else { - std::vector schemaColumns; - // assume that the schema and labels are the same lengths - for (unsigned int i = 0; i < metaData.schema.size(); i++) { - SchemaColumn schemaColumn; - schemaColumn.setLabel(metaData.labels[i]); - schemaColumn.setValueType(metaData.schema[i]); - schemaColumns.emplace_back(schemaColumn); - } - json[JsonKeys::SCHEMA] = schemaColumns; - } - - if (metaData.numNonZeros != -1) - json[JsonKeys::NUM_NON_ZEROS] = metaData.numNonZeros; - - ofs << json.dump(); + ofs << MetaDataParser::writeMetaDataToString(metaData); } else throw std::runtime_error("could not open file '" + metaFilename + "' for writing meta data"); diff --git a/src/parser/metadata/MetaDataParser.h b/src/parser/metadata/MetaDataParser.h index 47cccfcc3..b998037b7 100644 --- a/src/parser/metadata/MetaDataParser.h +++ b/src/parser/metadata/MetaDataParser.h @@ -67,7 +67,7 @@ class MetaDataParser { * keys or if the file doesn't contain all the metadata. */ static FileMetaData readMetaData(const std::string& filename); - + static FileMetaData readMetaDataFromString(const std::string& str); /** * @brief Saves the file meta data to the specified file. * @@ -76,6 +76,7 @@ class MetaDataParser { * @throws std::runtime_error Thrown if the specified file could not be openn. */ static void writeMetaData(const std::string& filename, const FileMetaData& metaData); + static std::string writeMetaDataToString(const FileMetaData& metaData); private: /** diff --git a/src/runtime/distributed/coordinator/kernels/DistributedRead.h b/src/runtime/distributed/coordinator/kernels/DistributedRead.h index 1f21b2027..981dd8f8b 100644 --- a/src/runtime/distributed/coordinator/kernels/DistributedRead.h +++ b/src/runtime/distributed/coordinator/kernels/DistributedRead.h @@ -14,29 +14,140 @@ * limitations under the License. */ -#ifndef SRC_RUNTIME_DISTRIBUTED_COORDINATOR_KERNELS_DISTRIBUTEDREAD_H -#define SRC_RUNTIME_DISTRIBUTED_COORDINATOR_KERNELS_DISTRIBUTEDREAD_H +#pragma once -#include +#include #include #include #include #include +#include + +#include +#include +#include +#include + +#ifdef USE_MPI + #include +#endif + #include +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** +template +struct DistributedRead { + static void apply(DTRes *&res, const char *filename, DCTX(dctx)) = delete; +}; // **************************************************************************** // Convenience function // **************************************************************************** -template -void distributedRead(Handle
*&res, const char * filename, DCTX(ctx)) +template +void distributedRead(DTRes *&res, const char *filename, DCTX(dctx)) { - FileMetaData fmd = MetaDataParser::readMetaData(filename); - - readCsv(res, filename, fmd.numRows, fmd.numCols, ','); + const auto allocation_type = dctx->getUserConfig().distributedBackEndSetup; + if (allocation_type == ALLOCATION_TYPE::DIST_MPI) + { +#ifdef USE_MPI + DistributedRead::apply(res, filename, dctx); +#endif + } + else if (allocation_type == ALLOCATION_TYPE::DIST_GRPC_ASYNC) + { + DistributedRead::apply(res, filename, dctx); + } + else if (allocation_type == ALLOCATION_TYPE::DIST_GRPC_SYNC) + { + DistributedRead::apply(res, filename, dctx); + } } -#endif //SRC_RUNTIME_DISTRIBUTED_COORDINATOR_KERNELS_DISTRIBUTEDREAD_H +// **************************************************************************** +// (Partial) template specializations for different distributed backends +// **************************************************************************** + +#ifdef USE_MPI +// ---------------------------------------------------------------------------- +// MPI +// ---------------------------------------------------------------------------- +template +struct DistributedRead +{ + static void apply(DTRes *&res, const char *filename, DCTX(dctx)) { + throw std::runtime_error("not implemented"); + } +}; +#endif + +// ---------------------------------------------------------------------------- +// Asynchronous GRPC +// ---------------------------------------------------------------------------- + +template +struct DistributedRead +{ + static void apply(DTRes *&res, const char *filename, DCTX(dctx)) { + throw std::runtime_error("not implemented"); + } +}; + +// ---------------------------------------------------------------------------- +// Synchronous GRPC +// ---------------------------------------------------------------------------- + +template +struct DistributedRead +{ + static void apply(DTRes *&res, const char *filename, DCTX(dctx)) { +#if USE_HDFS + auto ctx = DistributedContext::get(dctx); + auto workers = ctx->getWorkers(); + + // Generate metadata for the object based on MetaDataFile and + // when the worker needs the data it will read it automatically + + std::vector threads_vector; + LoadPartitioningDistributed partioner(DistributionSchema::DISTRIBUTE, res, dctx); + while (partioner.HasNextChunk()){ + auto hdfsFn = std::string(filename); + auto dp = partioner.GetNextChunk(); + + auto workerAddr = dynamic_cast(dp->allocation.get())->getLocation(); + std::thread t([=, &res]() + { + auto stub = ctx->stubs[workerAddr].get(); + + distributed::HDFSFile fileData; + fileData.set_filename(hdfsFn); + fileData.set_start_row(dp->range->r_start); + fileData.set_num_rows(dp->range->r_len); + fileData.set_num_cols(dp->range->c_len); + + grpc::ClientContext grpc_ctx; + distributed::StoredData response; + + auto status = stub->ReadHDFS(&grpc_ctx, fileData, &response); + if (!status.ok()) + throw std::runtime_error(status.error_message()); + + DistributedData newData; + newData.identifier = response.identifier(); + newData.numRows = response.num_rows(); + newData.numCols = response.num_cols(); + newData.isPlacedAtWorker = true; + dynamic_cast(*(dp->allocation)).updateDistributedData(newData); + }); + threads_vector.push_back(move(t)); + } + + for (auto &thread : threads_vector) + thread.join(); +#endif + } +}; diff --git a/src/runtime/distributed/coordinator/kernels/DistributedWrite.h b/src/runtime/distributed/coordinator/kernels/DistributedWrite.h new file mode 100644 index 000000000..910ea5b2e --- /dev/null +++ b/src/runtime/distributed/coordinator/kernels/DistributedWrite.h @@ -0,0 +1,176 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#ifdef USE_MPI + #include +#endif + +#include +#include + + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template +struct DistributedWrite { + static void apply(const DTArg *mat, const char *filename, DCTX(dctx)) = delete; +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void distributedWrite(const DTArg *mat, const char *filename, DCTX(dctx)) +{ + const auto allocation_type = dctx->getUserConfig().distributedBackEndSetup; + if (allocation_type == ALLOCATION_TYPE::DIST_MPI) + { +#ifdef USE_MPI + DistributedWrite::apply(mat, filename, dctx); +#endif + } + else if (allocation_type == ALLOCATION_TYPE::DIST_GRPC_ASYNC) + { + DistributedWrite::apply(mat, filename, dctx); + } + else if (allocation_type == ALLOCATION_TYPE::DIST_GRPC_SYNC) + { + DistributedWrite::apply(mat, filename, dctx); + } +} + + +// **************************************************************************** +// (Partial) template specializations for different distributed backends +// **************************************************************************** + +#ifdef USE_MPI +// ---------------------------------------------------------------------------- +// MPI +// ---------------------------------------------------------------------------- +template +struct DistributedWrite +{ + static void apply(const DTArg *mat, const char *filename, DCTX(dctx)) { + throw std::runtime_error("not implemented"); + } +}; +#endif + +// ---------------------------------------------------------------------------- +// Asynchronous GRPC +// ---------------------------------------------------------------------------- + +template +struct DistributedWrite +{ + static void apply(const DTArg *mat, const char *filename, DCTX(dctx)) { + throw std::runtime_error("not implemented"); + } +}; + +// ---------------------------------------------------------------------------- +// Synchronous GRPC +// ---------------------------------------------------------------------------- +#ifdef USE_HDFS + +template +struct DistributedWrite +{ + static void apply(const DTArg *mat, const char *filename, DCTX(dctx)) { + auto ctx = DistributedContext::get(dctx); + auto workers = ctx->getWorkers(); + + if(mat == nullptr) { + throw std::runtime_error("matrix argument is null"); + } + + std::filesystem::path filePath(filename); + auto directoryName = "/" + filePath.filename().string(); + auto baseFileName = directoryName + "/" + filePath.filename().string(); + + // Get nested file extension + auto extension = filePath.stem().extension().string(); + + auto hdfsCtx = HDFSContext::get(dctx); + auto fs = hdfsCtx->getConnection(); + if (fs == NULL) { + std::cout << "Error connecting to HDFS" << std::endl; + } + // Initialize chunk directory + if (hdfsCreateDirectory(*fs, directoryName.c_str())) { + throw std::runtime_error("Directory failed"); + } + size_t chunkId = 1; + std::vector threads_vector; + for (auto workerAddr: workers){ + auto hdfsfilename = baseFileName + "_segment_" + std::to_string(chunkId++); + DataPlacement *dp; + if ((dp = mat->getMetaDataObject()->getDataPlacementByLocation(workerAddr))) { + auto data = dynamic_cast(*(dp->allocation)).getDistributedData(); + if (data.isPlacedAtWorker) { + std::thread t([=, &mat]() + { + auto stub = ctx->stubs[workerAddr].get(); + + distributed::HDFSWriteInfo fileData; + fileData.mutable_matrix()->set_identifier(data.identifier); + fileData.mutable_matrix()->set_num_rows(data.numRows); + fileData.mutable_matrix()->set_num_cols(data.numCols); + + fileData.set_dirname(hdfsfilename.c_str()); + fileData.set_segment(std::to_string(chunkId).c_str()); + + grpc::ClientContext grpc_ctx; + distributed::Empty empty; + + auto status = stub->WriteHDFS(&grpc_ctx, fileData, &empty); + if (!status.ok()) + throw std::runtime_error(status.error_message()); + }); + threads_vector.push_back(move(t)); + } else { + auto slicedMat = mat->sliceRow(dp->range.get()->r_start, dp->range.get()->r_start + dp->range.get()->r_len); + if (extension == ".csv") { + writeHDFSCsv(slicedMat, hdfsfilename.c_str(), dctx); + } else if (extension == ".dbdf") { + writeDaphneHDFS(slicedMat, hdfsfilename.c_str(), dctx); + } + } + } else { + continue; + } + // TODO we should also store ranges that did not have a dataplacement associated with them + } + for (auto &thread : threads_vector) + thread.join(); + } +}; +#endif \ No newline at end of file diff --git a/src/runtime/distributed/proto/data.proto b/src/runtime/distributed/proto/data.proto index ffcc94a3f..dfed76bc1 100644 --- a/src/runtime/distributed/proto/data.proto +++ b/src/runtime/distributed/proto/data.proto @@ -25,3 +25,15 @@ message StoredData { uint64 num_rows = 2; uint64 num_cols = 3; } + +message HDFSFile { + string filename = 1; + uint64 start_row = 2; + uint64 num_rows = 3; + uint64 num_cols = 4; +} +message HDFSWriteInfo { + string dirName = 1; + string segment = 2; + StoredData matrix = 3; +} diff --git a/src/runtime/distributed/proto/worker.proto b/src/runtime/distributed/proto/worker.proto index 2237a0178..d2d41cc1a 100644 --- a/src/runtime/distributed/proto/worker.proto +++ b/src/runtime/distributed/proto/worker.proto @@ -23,6 +23,8 @@ option optimize_for = SPEED; import "data.proto"; service Worker { + rpc WriteHDFS (HDFSWriteInfo) returns (Empty) {} + rpc ReadHDFS (HDFSFile) returns (StoredData) {} rpc Store (stream Data) returns (StoredData) {} rpc Compute (Task) returns (ComputeResult) {} rpc Transfer (StoredData) returns (Data) {} diff --git a/src/runtime/distributed/worker/CMakeLists.txt b/src/runtime/distributed/worker/CMakeLists.txt index 319cd6685..3796f3068 100644 --- a/src/runtime/distributed/worker/CMakeLists.txt +++ b/src/runtime/distributed/worker/CMakeLists.txt @@ -28,6 +28,13 @@ set(SOURCES get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS) get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS) + +if(USE_HDFS) + find_library(LIBHDFS3 NAMES libhdfs3.so HINTS ${PROJECT_BINARY_DIR}/installed/lib REQUIRED) +else() + set(LIBHDFS3 "") +endif() + set(LIBS ${dialect_libs} ${conversion_libs} @@ -40,9 +47,11 @@ set(LIBS CallData Proto DaphneCatalogParser + ${LIBHDFS3} DaphneMetaDataParser Arrow::arrow_shared Parquet::parquet_shared + DaphneConfigParser Util) add_library(WorkerImpl ${SOURCES}) diff --git a/src/runtime/distributed/worker/WorkerImplGRPCSync.cpp b/src/runtime/distributed/worker/WorkerImplGRPCSync.cpp index b78ecb031..d79f8b001 100644 --- a/src/runtime/distributed/worker/WorkerImplGRPCSync.cpp +++ b/src/runtime/distributed/worker/WorkerImplGRPCSync.cpp @@ -23,6 +23,18 @@ #include #include +#if USE_HDFS + #include + #include + #include + #include + #include +#endif +#include +#include +#include + + WorkerImplGRPCSync::WorkerImplGRPCSync(const std::string& addr, DaphneUserConfig& _cfg) : WorkerImpl(_cfg) { @@ -39,7 +51,7 @@ void WorkerImplGRPCSync::Wait() { grpc::Status WorkerImplGRPCSync::Store(::grpc::ServerContext *context, ::grpc::ServerReader< ::distributed::Data>* reader, - ::distributed::StoredData *response) + ::distributed::StoredData *response) { StoredInfo storedInfo; distributed::Data data; @@ -50,21 +62,21 @@ grpc::Status WorkerImplGRPCSync::Store(::grpc::ServerContext *context, if (DF_Dtype(buffer) == DF_data_t::Value_t) { double val = DaphneSerializer::deserialize(buffer); storedInfo = WorkerImpl::Store(&val); - + response->set_identifier(storedInfo.identifier); response->set_num_rows(storedInfo.numRows); response->set_num_cols(storedInfo.numCols); } else { deserializer.reset(new DaphneDeserializerChunks(&mat, len)); - deserializerIter.reset(new DaphneDeserializerChunks::Iterator(deserializer->begin())); + deserializerIter.reset(new DaphneDeserializerChunks::Iterator(deserializer->begin())); (*deserializerIter)->second->resize(len); (*deserializerIter)->first = len; - + if ((*deserializerIter)->second->size() < len) (*deserializerIter)->second->resize(len); (*deserializerIter)->second->assign(static_cast(buffer), static_cast(buffer) + len); - + // advance iterator, this also partially deserializes ++(*deserializerIter); while (reader->Read(&data)){ @@ -74,7 +86,7 @@ grpc::Status WorkerImplGRPCSync::Store(::grpc::ServerContext *context, if ((*deserializerIter)->second->size() < len) (*deserializerIter)->second->resize(len); (*deserializerIter)->second->assign(static_cast(buffer), static_cast(buffer) + len); - + // advance iterator, this also partially deserializes ++(*deserializerIter); } @@ -99,8 +111,8 @@ grpc::Status WorkerImplGRPCSync::Compute(::grpc::ServerContext *context, inputs.push_back(StoredInfo({stored.identifier(), stored.num_rows(), stored.num_cols()})); } auto respMsg = WorkerImpl::Compute(&outputs, inputs, request->mlir_code()); - for (auto output : outputs){ - distributed::WorkData workData; + for (auto output : outputs){ + distributed::WorkData workData; workData.mutable_stored()->set_identifier(output.identifier); workData.mutable_stored()->set_num_rows(output.numRows); workData.mutable_stored()->set_num_cols(output.numCols); @@ -109,7 +121,7 @@ grpc::Status WorkerImplGRPCSync::Compute(::grpc::ServerContext *context, if (respMsg.ok()) return ::grpc::Status::OK; else - return ::grpc::Status(grpc::StatusCode::ABORTED, respMsg.error_message()); + return ::grpc::Status(grpc::StatusCode::ABORTED, respMsg.error_message()); } grpc::Status WorkerImplGRPCSync::Transfer(::grpc::ServerContext *context, @@ -123,4 +135,42 @@ grpc::Status WorkerImplGRPCSync::Transfer(::grpc::ServerContext *context, bufferLength = DaphneSerializer::serialize(mat, buffer); response->set_bytes(buffer.data(), bufferLength); return ::grpc::Status::OK; -} \ No newline at end of file +} + + +#if USE_HDFS +grpc::Status WorkerImplGRPCSync::ReadHDFS(::grpc::ServerContext *context, + const ::distributed::HDFSFile *request, + ::distributed::StoredData *response) +{ + DaphneContext ctx(cfg,KernelDispatchMapping::instance(), Statistics::instance(), StringRefCounter::instance()); + createHDFSContext(&ctx); + DenseMatrix *res = DataObjectFactory::create>(request->num_rows(), request->num_cols(), false); + if (request->filename().find("csv") != std::string::npos) + readHDFSCsv(res, request->filename().c_str(), request->num_rows(), request->num_cols(), ',', &ctx, request->start_row()); + else if (request->filename().find("dbdf") != std::string::npos) + readDaphneHDFS(res, request->filename().c_str(), &ctx, request->start_row()); + auto storedInfo = WorkerImpl::Store(dynamic_cast(res)); + + response->set_identifier(storedInfo.identifier); + response->set_num_rows(storedInfo.numRows); + response->set_num_cols(storedInfo.numCols); + return ::grpc::Status::OK; + +} + +grpc::Status WorkerImplGRPCSync::WriteHDFS(::grpc::ServerContext *context, + const ::distributed::HDFSWriteInfo *request, + ::distributed::Empty *response) +{ + DaphneContext ctx(cfg,KernelDispatchMapping::instance(), Statistics::instance(), StringRefCounter::instance()); + createHDFSContext(&ctx); + StoredInfo si({request->matrix().identifier(), request->matrix().num_rows(), request->matrix().num_cols()}); + auto mat = dynamic_cast*>(WorkerImpl::Transfer(si)); + if (request->dirname().find("csv") != std::string::npos) + writeHDFSCsv(mat, request->dirname().c_str(), &ctx); + else if (request->dirname().find("dbdf") != std::string::npos) + writeDaphneHDFS(mat, request->dirname().c_str(), &ctx); + return ::grpc::Status::OK; +} +#endif \ No newline at end of file diff --git a/src/runtime/distributed/worker/WorkerImplGRPCSync.h b/src/runtime/distributed/worker/WorkerImplGRPCSync.h index 75b591d2f..8f9dd1607 100644 --- a/src/runtime/distributed/worker/WorkerImplGRPCSync.h +++ b/src/runtime/distributed/worker/WorkerImplGRPCSync.h @@ -39,6 +39,14 @@ class WorkerImplGRPCSync : public WorkerImpl, public distributed::Worker::Servic public: explicit WorkerImplGRPCSync(const std::string& addr, DaphneUserConfig& _cfg); void Wait() override; +#if USE_HDFS + grpc::Status WriteHDFS(::grpc::ServerContext *context, + const ::distributed::HDFSWriteInfo *request, + ::distributed::Empty *response) override; + grpc::Status ReadHDFS(::grpc::ServerContext *context, + const ::distributed::HDFSFile *request, + ::distributed::StoredData *response) override; +#endif grpc::Status Store(::grpc::ServerContext *context, ::grpc::ServerReader<::distributed::Data>* reader, ::distributed::StoredData *response) override; diff --git a/src/runtime/distributed/worker/main.cpp b/src/runtime/distributed/worker/main.cpp index cf79052ee..d8a87cdb0 100644 --- a/src/runtime/distributed/worker/main.cpp +++ b/src/runtime/distributed/worker/main.cpp @@ -20,16 +20,33 @@ #include "WorkerImpl.h" #include "WorkerImplGRPCAsync.h" #include "WorkerImplGRPCSync.h" +#include int main(int argc, char *argv[]) { DaphneUserConfig user_config{}; + std::string configFile = "WorkerConfig.json"; + + if (argc == 3) { + configFile = argv[2]; + } + + try { + if (ConfigParser::fileExists(configFile)) { + ConfigParser::readUserConfig(configFile, user_config); + } + } + catch(std::exception & e) { + spdlog::error("Parser error while reading worker config:\n{}", e.what()); + spdlog::error("You can create a WorkerConfig.json to configure the worker.\n"); + } + user_config.resolveLibDir(); auto logger = std::make_unique(user_config); - if (argc != 2) { - std::cout << "Usage: " << argv[0] << " " << std::endl; + if (argc < 2 || argc > 3) { + std::cout << "Usage: " << argv[0] << " [ConfigFile]" << std::endl; exit(1); } auto addr = argv[1]; diff --git a/src/runtime/local/context/DaphneContext.h b/src/runtime/local/context/DaphneContext.h index f7469c7e7..a30e9ba4b 100644 --- a/src/runtime/local/context/DaphneContext.h +++ b/src/runtime/local/context/DaphneContext.h @@ -60,6 +60,7 @@ struct DaphneContext { std::vector> fpga_contexts; std::unique_ptr distributed_context; + std::unique_ptr hdfs_context; /** * @brief The user configuration (including information passed via CLI @@ -122,6 +123,11 @@ struct DaphneContext { [[nodiscard]] IContext *getDistributedContext() const { return distributed_context.get(); } +#ifdef USE_HDFS + [[nodiscard]] IContext* getHDFSContext() const { + return hdfs_context.get(); + } +#endif [[nodiscard]] DaphneUserConfig &getUserConfig() const { return config; } diff --git a/src/runtime/local/context/HDFSContext.h b/src/runtime/local/context/HDFSContext.h new file mode 100644 index 000000000..9745442d3 --- /dev/null +++ b/src/runtime/local/context/HDFSContext.h @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#if USE_HDFS + #include +#endif + +#include +#include + + +// TODO: Separate implementation in a .cpp file? +class HDFSContext final : public IContext { +private: + std::vector workers{}; +public: +#if USE_HDFS + std::unique_ptr fs; +#endif + HDFSContext(const DaphneUserConfig &cfg) { +#if USE_HDFS + auto IpPort = HDFSUtils::parseIPAddress(cfg.hdfs_Address); + fs = std::make_unique(hdfsConnectAsUser(std::get<0>(IpPort).c_str(), std::get<1>(IpPort), cfg.hdfs_username.c_str())); +#endif + } + ~HDFSContext() { +#if USE_HDFS + hdfsDisconnect(*fs); +#endif + }; + + static std::unique_ptr createHDFSContext(const DaphneUserConfig &cfg) { + auto ctx = std::unique_ptr(new HDFSContext(cfg)); + return ctx; + }; + + void destroy() override { + // Clean up + }; +#if USE_HDFS + static HDFSContext* get(DaphneContext *ctx) { return dynamic_cast(ctx->getHDFSContext()); }; + hdfsFS* getConnection() { return fs.get(); }; +#endif +}; \ No newline at end of file diff --git a/src/runtime/local/datastructures/AllocationDescriptorGRPC.h b/src/runtime/local/datastructures/AllocationDescriptorGRPC.h index bb03310bf..dbcaa8a32 100644 --- a/src/runtime/local/datastructures/AllocationDescriptorGRPC.h +++ b/src/runtime/local/datastructures/AllocationDescriptorGRPC.h @@ -16,11 +16,13 @@ #pragma once -#include -#include +#include "DataPlacement.h" + #include +#include #include +#include class AllocationDescriptorGRPC : public IAllocationDescriptor { private: diff --git a/src/runtime/local/datastructures/DenseMatrix.h b/src/runtime/local/datastructures/DenseMatrix.h index 2edd05ba8..8d082f974 100644 --- a/src/runtime/local/datastructures/DenseMatrix.h +++ b/src/runtime/local/datastructures/DenseMatrix.h @@ -94,6 +94,8 @@ class DenseMatrix : public Matrix */ DenseMatrix(const DenseMatrix * src, int64_t rowLowerIncl, int64_t rowUpperExcl, int64_t colLowerIncl, int64_t colUpperExcl); + DenseMatrix(const DenseMatrix * src, int64_t rowLowerIncl, int64_t rowUpperExcl) : DenseMatrix(src, rowLowerIncl, rowUpperExcl, 0, src->numCols) {}; + /** * @brief Creates a `DenseMatrix` around an existing array of values without copying the data. diff --git a/src/runtime/local/datastructures/IAllocationDescriptor.h b/src/runtime/local/datastructures/IAllocationDescriptor.h index 75eb9fa88..732587484 100644 --- a/src/runtime/local/datastructures/IAllocationDescriptor.h +++ b/src/runtime/local/datastructures/IAllocationDescriptor.h @@ -51,7 +51,7 @@ class IAllocationDescriptor { virtual ~IAllocationDescriptor() = default; [[nodiscard]] virtual ALLOCATION_TYPE getType() const = 0; virtual void createAllocation(size_t size, bool zero) = 0; - virtual std::string getLocation() const = 0; + [[nodiscard]] virtual std::string getLocation() const = 0; virtual std::shared_ptr getData() = 0; virtual void transferTo(std::byte* src, size_t size) = 0; virtual void transferFrom(std::byte* dst, size_t size) = 0; diff --git a/src/runtime/local/datastructures/MetaDataObject.h b/src/runtime/local/datastructures/MetaDataObject.h index 75d1ea3b4..67e022378 100644 --- a/src/runtime/local/datastructures/MetaDataObject.h +++ b/src/runtime/local/datastructures/MetaDataObject.h @@ -16,7 +16,8 @@ #pragma once -class DataPlacement; +struct DataPlacement; + #include "IAllocationDescriptor.h" #include "Range.h" diff --git a/src/runtime/local/io/FileMetaData.h b/src/runtime/local/io/FileMetaData.h index e8e3e6044..603e80364 100644 --- a/src/runtime/local/io/FileMetaData.h +++ b/src/runtime/local/io/FileMetaData.h @@ -22,6 +22,11 @@ #include #include +struct HDFSMetaData{ + bool isHDFS = false; + std::string HDFSFilename; +}; + /** * @brief Very simple representation of basic file meta data. * @@ -34,6 +39,7 @@ struct FileMetaData { std::vector schema; std::vector labels; const ssize_t numNonZeros; + HDFSMetaData hdfs; /** * @brief Construct a new File Meta Data object for Frames @@ -44,13 +50,15 @@ struct FileMetaData { bool isSingleValueType, std::vector schema, std::vector labels, - ssize_t numNonZeros = -1 + ssize_t numNonZeros = -1, + HDFSMetaData hdfs = {} ) : numRows(numRows), numCols(numCols), isSingleValueType(isSingleValueType), schema(std::move(schema)), labels(std::move(labels)), - numNonZeros(numNonZeros) {} + numNonZeros(numNonZeros), + hdfs(hdfs) {} /** * @brief Construct a new File Meta Data object for Matrix @@ -60,11 +68,13 @@ struct FileMetaData { size_t numCols, bool isSingleValueType, ValueTypeCode valueType, - ssize_t numNonZeros = -1 + ssize_t numNonZeros = -1, + HDFSMetaData hdfs = {} ) : numRows(numRows), numCols(numCols), isSingleValueType(isSingleValueType), - numNonZeros(numNonZeros) + numNonZeros(numNonZeros), + hdfs(hdfs) { schema.emplace_back(valueType); } diff --git a/src/runtime/local/io/HDFS/HDFSUtils.h b/src/runtime/local/io/HDFS/HDFSUtils.h new file mode 100644 index 000000000..9a38fc613 --- /dev/null +++ b/src/runtime/local/io/HDFS/HDFSUtils.h @@ -0,0 +1,82 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#if USE_HDFS + +#include + +#include + +struct HDFSUtils { + static FileMetaData parseHDFSMetaData(std::string filename, const hdfsFS &fs) { + auto metaFn = (filename + ".meta"); + // Read related metadata in hdfs + hdfsFile hFile = hdfsOpenFile(fs, metaFn.c_str(), O_RDONLY, 0, 0, 0); + hdfsFileInfo *fileInfo = hdfsGetPathInfo(fs, metaFn.c_str()); + size_t fileSize = fileInfo->mSize; + hdfsFreeFileInfo(fileInfo, 1); + std::vector buffer(fileSize); + hdfsRead(fs, hFile, buffer.data(), fileSize); + std::string fmdStr(buffer.data(), buffer.size()); + return MetaDataParser::readMetaDataFromString(fmdStr); + } + + static inline std::string getBaseFile(const char * fn){ + std::filesystem::path filePath(fn); + return filePath.filename().string(); + } + + static std::tuple findSegmendAndOffset(const hdfsFS &fs, size_t startOffset, size_t startRow, const char * filename, size_t offsetMuliplier) { + size_t skippedRows = 0; + int seg = 1; + + while (skippedRows != startRow) { + std::string segf = std::string(filename) + "/" + HDFSUtils::getBaseFile(filename) + "_segment_" + std::to_string(seg++); + auto fmd = HDFSUtils::parseHDFSMetaData(segf, fs); + skippedRows += fmd.numRows; + // We need offset within segment + if (skippedRows > startRow) { + seg--; // adjust segment and skipped rows + skippedRows -= fmd.numRows; + startOffset += (startRow - skippedRows) * offsetMuliplier; + skippedRows = startRow; + } + } + return std::make_tuple(seg, startOffset); + } + + static std::tuple parseIPAddress(const std::string& input) { + std::string ip; + uint16_t port = 9000; // Default port + size_t colonPos = input.find(':'); + + if (colonPos != std::string::npos) { + // If there's a colon, split IP and port + ip = input.substr(0, colonPos); + std::string portStr = input.substr(colonPos + 1); + port = static_cast(std::stoi(portStr)); + } else { + // If no colon, the entire input is the IP + ip = input; + } + + return std::make_tuple(ip, port); + } +}; +#endif \ No newline at end of file diff --git a/src/runtime/local/io/HDFS/ReadDaphneHDFS.h b/src/runtime/local/io/HDFS/ReadDaphneHDFS.h new file mode 100644 index 000000000..1c91f52da --- /dev/null +++ b/src/runtime/local/io/HDFS/ReadDaphneHDFS.h @@ -0,0 +1,148 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include // For setfill and setw +#include + +#include + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template +struct ReadDaphneHDFS +{ + static void apply(DTRes *&res, const char *hdfsDir, DCTX(dctx), size_t startRow = 0) = delete; +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void readDaphneHDFS(DTRes *&res, const char *hdfsDir, DCTX(dctx), size_t startRow = 0) +{ + ReadDaphneHDFS::apply(res, hdfsDir, dctx, startRow); +} + +// **************************************************************************** +// (Partial) template specializations for different data/value types +// **************************************************************************** + +// ---------------------------------------------------------------------------- +// DenseMatrix +// ---------------------------------------------------------------------------- + +template +struct ReadDaphneHDFS> +{ + static void apply(DenseMatrix *&res, const char *hdfsDir, DCTX(dctx), size_t startRow = 0) + { + if (res == NULL) { + throw std::runtime_error("Could not initialize result matrix"); + } + + size_t numRows = res->getNumRows(); + size_t numCols = res->getNumCols(); + + auto hdfsCtx = HDFSContext::get(dctx); + auto fs = hdfsCtx->getConnection(); + if (fs == NULL) + { + std::cout << "Error connecting to HDFS" << std::endl; + } + + auto headerSize = DaphneSerializer>::headerSize(res); + + auto [startSegment, offset] = HDFSUtils::findSegmendAndOffset(*fs, headerSize, startRow, hdfsDir, numCols * sizeof(VT)); + + size_t parsedRows = 0; + auto segment = startSegment; + size_t startSerByte = headerSize; + while (parsedRows < numRows){ + auto hdfsFn = std::string(hdfsDir) + "/" + HDFSUtils::getBaseFile(hdfsDir) + "_segment_" + std::to_string(segment++); + auto segFmd = HDFSUtils::parseHDFSMetaData(hdfsFn, *fs); + + hdfsFile hFile = hdfsOpenFile(*fs, hdfsFn.c_str(), O_RDONLY, 0, 0, 0); + if (hFile == NULL) + { + throw std::runtime_error("Error opening HDFS file"); + } + + // Find out the size of the file to allocate a buffer + hdfsFileInfo *fileInfo = hdfsGetPathInfo(*fs, hdfsFn.c_str()); + if (fileInfo == NULL) + { + hdfsCloseFile(*fs, hFile); + throw std::runtime_error("Error getting file info"); + return; + } + tSize fileSize = fileInfo->mSize; + hdfsFreeFileInfo(fileInfo, 1); + + // Allocate buffer + std::vector buffer(fileSize); + + // If started parsing rows, set offset to headerSize + // Simply skip buffer header, except first time. + offset = parsedRows == 0 ? offset : headerSize; + hdfsSeek(*fs, hFile, offset); + // Read the file into the buffer + tSize bytesRead = hdfsRead(*fs, hFile, buffer.data(), fileSize); + if (bytesRead == -1) + { + hdfsCloseFile(*fs, hFile); + throw std::runtime_error("Error reading file"); + return; + } + size_t bufferEnd = fileSize; + // If segment is bigger than end row, bufferend should not be equal to whole segment + if (numRows - parsedRows < segFmd.numRows){ + bufferEnd = (numRows - parsedRows) * numCols * sizeof(VT); + } + res = DaphneSerializer>::deserialize(buffer.data(), bufferEnd, res, startSerByte); + startSerByte += bufferEnd - offset; + + hdfsCloseFile(*fs, hFile); + parsedRows += segFmd.numRows; + } + } +}; diff --git a/src/runtime/local/io/HDFS/ReadHDFS.h b/src/runtime/local/io/HDFS/ReadHDFS.h new file mode 100644 index 000000000..b53903989 --- /dev/null +++ b/src/runtime/local/io/HDFS/ReadHDFS.h @@ -0,0 +1,87 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include // For setfill and setw +#include + +#include + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template +struct ReadHDFS +{ + static void apply(DTRes *&res, const char *filename, DCTX(dctx)) + { + FileMetaData fmd = MetaDataParser::readMetaData(filename); + res = DataObjectFactory::create(fmd.numRows, fmd.numCols, false); + + auto hdfsFilename = fmd.hdfs.HDFSFilename.c_str(); + std::filesystem::path filePath(hdfsFilename); + + // Get nested file extension + auto extension = filePath.stem().extension().string(); + + if (dctx->config.use_distributed) { + distributedRead(res, hdfsFilename, dctx); + } else { + if (extension == ".csv") { + readHDFSCsv(res, hdfsFilename, fmd.numRows, fmd.numCols, ',', dctx); + } else if (extension == ".dbdf") { + readDaphneHDFS(res, hdfsFilename, dctx); + } + } + } +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void readHDFS(DTRes *&res, const char *hdfsFilename, DCTX(dctx)) +{ + ReadHDFS::apply(res, hdfsFilename, dctx); +} diff --git a/src/runtime/local/io/HDFS/ReadHDFSCsv.h b/src/runtime/local/io/HDFS/ReadHDFSCsv.h new file mode 100644 index 000000000..c941246cd --- /dev/null +++ b/src/runtime/local/io/HDFS/ReadHDFSCsv.h @@ -0,0 +1,154 @@ +/* + * Copyright 2023 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template struct ReadHDFSCsv { + static void apply(DTRes *&res, const char *hdfsDir, size_t numRows, size_t numCols, + char delim, DCTX(dctx), size_t startRow = 0) = delete; +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void readHDFSCsv(DTRes *&res, const char *hdfsDir, size_t numRows, size_t numCols, + char delim, DCTX(dctx), size_t startRow = 0) { + ReadHDFSCsv::apply(res, hdfsDir, numRows, numCols, delim, dctx, startRow); +} + + +// **************************************************************************** +// (Partial) template specializations for different data/value types +// **************************************************************************** + +// ---------------------------------------------------------------------------- +// DenseMatrix +// ---------------------------------------------------------------------------- + +template struct ReadHDFSCsv> { + static void apply(DenseMatrix *&res, const char *hdfsDir, size_t numRows, + size_t numCols, char delim, DCTX(dctx), size_t startRow = 0) { + if(hdfsDir == nullptr) { + throw std::runtime_error("File required"); + } + if (numRows <= 0) { + throw std::runtime_error("numRows must be > 0"); + } + if (numRows <= 0) { + throw std::runtime_error("numCols must be > 0"); + } + + if (res == nullptr) { + res = DataObjectFactory::create>(numRows, numCols, false); + } + + auto hdfsCtx = HDFSContext::get(dctx); + auto fs = hdfsCtx->getConnection(); + if (fs == NULL) { + std::cerr << "Error connecting to HDFS" << std::endl; + } + + [[maybe_unused]] auto [startSegment, dummy] = HDFSUtils::findSegmendAndOffset(*fs, 0, startRow, hdfsDir, numCols * sizeof(VT)); + // TODO verify file exists + + size_t parsedRows = 0; + auto segment = startSegment; + if (res == NULL){ + throw std::runtime_error("Could not initialize result matrix"); + } + + VT *valuesRes = res->getValues(); + + while (parsedRows < numRows) { + auto hdfsFn = std::string(hdfsDir) + "/" + HDFSUtils::getBaseFile(hdfsDir) + "_segment_" + std::to_string(segment++); + auto segFmd = HDFSUtils::parseHDFSMetaData(hdfsFn, *fs); + + hdfsFile hFile = hdfsOpenFile(*fs, hdfsFn.c_str(), O_RDONLY, 0, 0, 0); + if (hFile == NULL) { + throw std::runtime_error("Error opening HDFS file"); + } + + char buffer[1UL << 20]; + char *cur = nullptr; + size_t n = 0; + + for (size_t r = 0; r < segFmd.numRows; r++) { + std::string line; + + do { + if (cur == nullptr) { + n = hdfsRead(*fs, hFile, buffer, 1UL << 20); + if (n <= 0) { + throw std::runtime_error("Could not read hdfs file"); + } + cur = buffer; + } + + char *eol = (char *)std::memchr(cur, '\n', n); + if (eol == nullptr || static_cast(eol - cur) >= n) { + line.append(cur, n); + cur = nullptr; + } else { + line.append(cur, eol - cur); + cur = eol + 1; + } + } while (cur == nullptr); + // If first segment, skip rows + if (parsedRows == 0 && startRow > (segment - 2) * segFmd.numRows + r) + continue; + + size_t pos = 0; + for(size_t c = 0; c < numCols; c++) { + VT val; + convertCstr(line.c_str() + pos, &val); + + // TODO This assumes that rowSkip == numCols. + *valuesRes = val; + valuesRes++; + // TODO We could even exploit the fact that the strtoX functions can + // return a pointer to the first character after the parsed input, then + // we wouldn't have to search for that ourselves, just would need to + // check if it is really the delimiter. + if(c < numCols - 1) { + while(line[pos] != delim) pos++; + pos++; // skip delimiter + } + } + parsedRows++; + if (parsedRows == numRows) + break; + } + + hdfsCloseFile(*fs, hFile); + } + } +}; diff --git a/src/runtime/local/io/HDFS/WriteDaphneHDFS.h b/src/runtime/local/io/HDFS/WriteDaphneHDFS.h new file mode 100644 index 000000000..62ab17f94 --- /dev/null +++ b/src/runtime/local/io/HDFS/WriteDaphneHDFS.h @@ -0,0 +1,115 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template +struct WriteDaphneHDFS +{ + static void apply(const DTArg *arg, const char *hdfsFilename, DCTX(dctx)) = delete; +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void writeDaphneHDFS(const DTArg *arg, const char *hdfsFilename, DCTX(dctx)) +{ + WriteDaphneHDFS::apply(arg, hdfsFilename, dctx); +} + +// **************************************************************************** +// (Partial) template specializations for different data/value types +// **************************************************************************** + +// ---------------------------------------------------------------------------- +// DenseMatrix +// ---------------------------------------------------------------------------- + +template +struct WriteDaphneHDFS>{ + static void apply(const DenseMatrix *arg, const char *hdfsFilename, DCTX(dctx)) + { + size_t length; + length = DaphneSerializer>::length(arg); + + std::vector buffer(length); + DaphneSerializer>::serialize(arg, buffer); + + auto hdfsCtx = HDFSContext::get(dctx); + auto fs = hdfsCtx->getConnection(); + if (fs == NULL) + { + std::cout << "Error connecting to HDFS" << std::endl; + } + + // Write related fmd + FileMetaData fmd(arg->getNumRows(), arg->getNumCols(), true, ValueTypeUtils::codeFor); + auto fmdStr = MetaDataParser::writeMetaDataToString(fmd); + auto fn = std::string(hdfsFilename); + auto mdtFn = fn + ".meta"; + hdfsFile hdfsFile = hdfsOpenFile(*fs, mdtFn.c_str(), O_WRONLY, 0, 0, 0); + if (hdfsFile == NULL) { + throw std::runtime_error("Error opening HDFS file"); + } + hdfsWrite(*fs, hdfsFile, static_cast(fmdStr.c_str()), fmdStr.size()); + hdfsCloseFile(*fs, hdfsFile); + + // Write binary + hdfsFile = hdfsOpenFile(*fs, hdfsFilename, O_WRONLY, 0, 0, 0); + if (hdfsFile == NULL) + { + throw std::runtime_error("Error opening HDFS file"); + } + + hdfsWrite(*fs, hdfsFile, buffer.data(), length); + if (hdfsCloseFile(*fs, hdfsFile) == -1) + { + throw std::runtime_error("Failed to close HDFS file"); + } + } +}; diff --git a/src/runtime/local/io/HDFS/WriteHDFS.h b/src/runtime/local/io/HDFS/WriteHDFS.h new file mode 100644 index 000000000..8a8315bbc --- /dev/null +++ b/src/runtime/local/io/HDFS/WriteHDFS.h @@ -0,0 +1,109 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template +struct WriteHDFS +{ + static void apply(const DTArg *arg, const char *filename, DCTX(dctx)) + { + auto hdfsCtx = HDFSContext::get(dctx); + auto fs = hdfsCtx->getConnection(); + if (fs == NULL) + { + std::cout << "Error connecting to HDFS" << std::endl; + } + + // TODO for now store to /filename/ directory in hdfs. + std::filesystem::path filePath(filename); + auto dirFileName = "/" + filePath.filename().string(); + auto baseFileName = dirFileName + "/" + filePath.filename().string(); + + // Get nested file extension + auto extension = filePath.stem().extension().string(); + + // Check if the directory already exists + if (hdfsExists(*fs, dirFileName.c_str()) == -1) { + // The file does not exist, so create the directory structure + // and the file + if(hdfsCreateDirectory(*fs, dirFileName.c_str()) == -1) + throw std::runtime_error("Failed to create file"); + } else { + // clear directory + int numEntries; + hdfsFileInfo * files = hdfsListDirectory(*fs, dirFileName.c_str(), &numEntries); + for (int i = 0; i < numEntries; i++) + hdfsDelete(*fs, files[i].mName, 1); + hdfsFreeFileInfo(files, numEntries); + } + + if (dctx->config.use_distributed) { + distributedWrite(arg, filename, dctx); + + } else { + // Write one segment + auto hdfsfilename = baseFileName + "_segment_1"; + if (extension == ".csv") { + writeHDFSCsv(arg, hdfsfilename.c_str(), dctx); + } else if (extension == ".dbdf") { + writeDaphneHDFS(arg, hdfsfilename.c_str(), dctx); + } + } + } +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void writeHDFS(const DTArg *arg, const char *filename, DCTX(dctx)) +{ + WriteHDFS::apply(arg, filename, dctx); +} diff --git a/src/runtime/local/io/HDFS/WriteHDFSCsv.h b/src/runtime/local/io/HDFS/WriteHDFSCsv.h new file mode 100644 index 000000000..d0587d0d4 --- /dev/null +++ b/src/runtime/local/io/HDFS/WriteHDFSCsv.h @@ -0,0 +1,140 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// **************************************************************************** +// Struct for partial template specialization +// **************************************************************************** + +template +struct WriteHDFSCsv { + static void apply(const DTArg *arg, const char *hdfsFilename, DCTX(dctx)) = delete; +}; + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +template +void writeHDFSCsv(const DTArg *arg, const char *hdfsFilename, DCTX(dctx)) { + WriteHDFSCsv::apply(arg, hdfsFilename, dctx); +} + +// **************************************************************************** +// (Partial) template specializations for different data/value types +// **************************************************************************** + +// ---------------------------------------------------------------------------- +// DenseMatrix +// ---------------------------------------------------------------------------- + +template +struct WriteHDFSCsv> { + static void apply(const DenseMatrix *arg, const char *hdfsFilename, DCTX(dctx)) { + if (hdfsFilename == nullptr) { + throw std::runtime_error("Could not read hdfs file"); + } + + std::string fn(hdfsFilename); + + auto hdfsCtx = HDFSContext::get(dctx); + auto fs = hdfsCtx->getConnection(); + if (fs == NULL) { + std::cout << "Error connecting to HDFS" << std::endl; + } + + // Check if path exists + std::filesystem::path filePath(hdfsFilename); + auto dirFileName = filePath.parent_path(); + + if (hdfsExists(*fs, dirFileName.c_str()) == -1) { + // The file does not exist, so create the directory structure + // and the file + // TODO extract directory path from filename + if (fn.find("/") == std::string::npos) + throw std::runtime_error("HDFS subdirectories not supported atm"); + // if(hdfsCreateDirectory(fs, fn.c_str()) == -1) + // throw std::runtime_error("Failed to create file"); + } + // If not, add "/" at the beginning + if (fn.find("/") == std::string::npos) + fn = "/" + fn; + + // Write related fmd + FileMetaData fmd(arg->getNumRows(), arg->getNumCols(), true, ValueTypeUtils::codeFor); + auto fmdStr = MetaDataParser::writeMetaDataToString(fmd); + auto mdtFn = fn + ".meta"; + hdfsFile hdfsFile = hdfsOpenFile(*fs, mdtFn.c_str(), O_WRONLY, 0, 0, 0); + if (hdfsFile == NULL) { + throw std::runtime_error("Error opening HDFS file"); + } + hdfsWrite(*fs, hdfsFile, static_cast(fmdStr.c_str()), fmdStr.size()); + hdfsCloseFile(*fs, hdfsFile); + // Open the HDFS file for writing + hdfsFile = hdfsOpenFile(*fs, fn.c_str(), O_WRONLY, 0, 0, 0); + if (hdfsFile == NULL) { + throw std::runtime_error("Error opening HDFS file"); + } + + const VT *valuesArg = arg->getValues(); + size_t cell = 0; + for (size_t i = 0; i < arg->getNumRows(); ++i) { + for (size_t j = 0; j < arg->getNumCols(); ++j) { + // Convert the numeric value to a string and add a comma + std::ostringstream oss; + oss << valuesArg[cell]; + oss << (j < (arg->getNumCols() - 1) + ? "," + : ""); // Add a comma unless it's the last column + const std::string &valueStr = oss.str(); + + // Write the value string to the HDFS file + if (hdfsWrite(*fs, hdfsFile, static_cast(valueStr.c_str()), valueStr.size()) == -1) { + hdfsCloseFile(*fs, hdfsFile); + throw std::runtime_error("Failed to write to HDFS file"); + } + cell++; + } + // Add a newline character at the end of each row + const char newline = '\n'; + if (hdfsWrite(*fs, hdfsFile, static_cast(&newline), + 1) == -1) { + hdfsCloseFile(*fs, hdfsFile); + throw std::runtime_error("Failed to write to HDFS file"); + } + } + + // Close the HDFS file and disconnect + if (hdfsCloseFile(*fs, hdfsFile) == -1) { + throw std::runtime_error("Failed to close HDFS file"); + } + } +}; diff --git a/src/runtime/local/kernels/CMakeLists.txt b/src/runtime/local/kernels/CMakeLists.txt index 0996ca58c..1ddf4e18d 100644 --- a/src/runtime/local/kernels/CMakeLists.txt +++ b/src/runtime/local/kernels/CMakeLists.txt @@ -122,4 +122,9 @@ endif() find_library(HWLOC_LIB NAMES libhwloc.so HINTS ${PROJECT_BINARY_DIR}/installed/lib REQUIRED) -target_link_libraries(AllKernels PUBLIC ${LIBS} ${MPI_LIBRARIES} ${PAPI_LIB} ${HWLOC_LIB}) +if(USE_HDFS) + target_include_directories(AllKernels PUBLIC ${PROJECT_SOURCE_DIR}/thirdparty/installed/include/hdfs) + find_library(LIBHDFS3 NAMES libhdfs3.so HINTS ${PROJECT_BINARY_DIR}/installed/lib REQUIRED) +endif() + +target_link_libraries(AllKernels PUBLIC ${LIBS} ${MPI_LIBRARIES} ${PAPI_LIB} ${HWLOC_LIB} ${LIBHDFS3}) diff --git a/src/runtime/local/kernels/CreateHDFSContext.h b/src/runtime/local/kernels/CreateHDFSContext.h new file mode 100644 index 000000000..0aa68df39 --- /dev/null +++ b/src/runtime/local/kernels/CreateHDFSContext.h @@ -0,0 +1,28 @@ +/* + * Copyright 2021 The DAPHNE Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "runtime/local/context/DaphneContext.h" +#include "runtime/local/context/HDFSContext.h" + +// **************************************************************************** +// Convenience function +// **************************************************************************** + +static void createHDFSContext(DCTX(ctx)) { + ctx->hdfs_context = HDFSContext::createHDFSContext(ctx->config); +} diff --git a/src/runtime/local/kernels/Read.h b/src/runtime/local/kernels/Read.h index f092d3377..6436cb8f5 100644 --- a/src/runtime/local/kernels/Read.h +++ b/src/runtime/local/kernels/Read.h @@ -27,7 +27,9 @@ #include #include #include - +#if USE_HDFS + #include +#endif #include #include #include @@ -39,6 +41,9 @@ struct FileExt { m["mtx"] = 1; m["parquet"] = 2; m["dbdf"] = 3; +#if USE_HDFS + m["hdfs"] = 4; +#endif return m; } static const std::map map; @@ -98,9 +103,18 @@ struct Read> { break; case 3: readDaphne(res, filename); - break; - default: - throw std::runtime_error("File extension not supported"); + break; +#if USE_HDFS + case 4: + if(res == nullptr) + res = DataObjectFactory::create>( + fmd.numRows, fmd.numCols, false + ); + readHDFS(res, filename, ctx); + break; +#endif + default: + throw std::runtime_error("File extension not supported"); } } }; diff --git a/src/runtime/local/kernels/Write.h b/src/runtime/local/kernels/Write.h index 303f4c5d8..44f6f03cd 100644 --- a/src/runtime/local/kernels/Write.h +++ b/src/runtime/local/kernels/Write.h @@ -26,7 +26,9 @@ #include #include #include - +#if USE_HDFS + #include +#endif // **************************************************************************** // Struct for partial template specialization @@ -70,8 +72,19 @@ struct Write> { FileMetaData metaData(arg->getNumRows(), arg->getNumCols(), true, ValueTypeUtils::codeFor); MetaDataParser::writeMetaData(filename, metaData); writeDaphne(arg, filename); - } else { - throw std::runtime_error( "[Write.h] - unsupported file extension in write kernel."); +#if USE_HDFS + } else if (ext == "hdfs"){ + HDFSMetaData hdfs = {true, filename}; + FileMetaData metaData(arg->getNumRows(), arg->getNumCols(), true, ValueTypeUtils::codeFor, -1, hdfs); + // Get file extension before .hdfs (e.g. file.csv.hdfs) + auto posHdfs = pos; + auto posExt = fn.find_last_of('.', pos-1); + std::string nestedExt(fn.substr(posExt + 1, posHdfs - posExt - 1)); + MetaDataParser::writeMetaData(filename, metaData); + + // call WriteHDFS + writeHDFS(arg, filename, ctx); +#endif } } }; @@ -104,18 +117,18 @@ struct Write { template struct Write> { static void apply(const Matrix * arg, const char * filename, DCTX(ctx)) { - std::string fn(filename); - auto pos = fn.find_last_of('.'); - std::string ext(fn.substr(pos+1)); - if (ext == "csv") { - File * file = openFileForWrite(filename); - FileMetaData metaData(arg->getNumRows(), arg->getNumCols(), true, ValueTypeUtils::codeFor); - MetaDataParser::writeMetaData(filename, metaData); - writeCsv(arg, file); - closeFile(file); - } else { - throw std::runtime_error( "[Write.h] - generic Matrix type currently only supports csv file extension."); - } + std::string fn(filename); + auto pos = fn.find_last_of('.'); + std::string ext(fn.substr(pos+1)); + if (ext == "csv") { + File * file = openFileForWrite(filename); + FileMetaData metaData(arg->getNumRows(), arg->getNumCols(), true, ValueTypeUtils::codeFor); + MetaDataParser::writeMetaData(filename, metaData); + writeCsv(arg, file); + closeFile(file); + } else { + throw std::runtime_error( "[Write.h] - generic Matrix type currently only supports csv file extension."); + } } }; diff --git a/src/runtime/local/kernels/kernels.json b/src/runtime/local/kernels/kernels.json index 5fa5d7ca7..a74a59dc6 100644 --- a/src/runtime/local/kernels/kernels.json +++ b/src/runtime/local/kernels/kernels.json @@ -27,16 +27,16 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - ["double", ["DenseMatrix", "double"]], - ["float", ["DenseMatrix", "float"]], + ["double", ["DenseMatrix", "double"]], + ["float", ["DenseMatrix", "float"]], ["int64_t", ["DenseMatrix", "int64_t"]] ], "opCodes": ["SUM", "MIN", "MAX"] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ ["float", ["DenseMatrix", "float"]], ["double", ["DenseMatrix", "double"]], @@ -95,31 +95,85 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ], "opCodes": ["SUM", "MIN", "MAX"] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["CSRMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["CSRMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["CSRMatrix", "int64_t"]], - [["DenseMatrix", "float"], ["CSRMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "float"], + ["CSRMatrix", "int64_t"] + ] ], - "opCodes": ["SUM", "MIN", "MAX", "MEAN", "STDDEV", "VAR", "IDXMIN", "IDXMAX"] + "opCodes": [ + "SUM", + "MIN", + "MAX", + "MEAN", + "STDDEV", + "VAR", + "IDXMIN", + "IDXMAX" + ] } ] }, @@ -154,10 +208,22 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"] + ] ], "opCodes": ["SUM", "PROD", "MIN", "MAX"] }, @@ -193,34 +259,94 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ], "opCodes": ["SUM", "MIN", "MAX"] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "int64_t"]], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "int64_t"] + ], - [["DenseMatrix", "double"], ["CSRMatrix", "double"]], - [["DenseMatrix", "float"], ["CSRMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["CSRMatrix", "int64_t"]], - [["DenseMatrix", "float"], ["CSRMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["CSRMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["CSRMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "float"], + ["CSRMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "int64_t"] + ] ], - "opCodes": ["SUM", "MIN", "MAX", "MEAN", "STDDEV", "VAR", "IDXMIN", "IDXMAX"] + "opCodes": [ + "SUM", + "MIN", + "MAX", + "MEAN", + "STDDEV", + "VAR", + "IDXMIN", + "IDXMAX" + ] } ] }, @@ -245,9 +371,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -282,105 +406,372 @@ ["Frame", ["DenseMatrix", "double"]], ["Frame", ["DenseMatrix", "int64_t"]], ["Frame", ["DenseMatrix", "uint64_t"]], - - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "double"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "size_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "size_t"] + ], + + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "size_t"] + ], + + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "size_t"]], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "size_t"] + ], - [["DenseMatrix", "size_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "size_t"]], - - [["DenseMatrix","double"],["CSRMatrix","double"]], - [["DenseMatrix","float"],["CSRMatrix","float"]], - [["DenseMatrix","int64_t"],["CSRMatrix","int64_t"]], - [["CSRMatrix","double"], ["DenseMatrix","double"]], - [["CSRMatrix","float"], ["DenseMatrix","float"]], - [["CSRMatrix","int64_t"], ["DenseMatrix","int64_t"]], - [["CSRMatrix","double"], ["CSRMatrix","float"]], - [["CSRMatrix","float"], ["CSRMatrix","double"]] + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["CSRMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], + [ + ["CSRMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["CSRMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["CSRMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "float"] + ], + [ + ["CSRMatrix", "float"], + ["CSRMatrix", "double"] + ] ] }, { @@ -536,10 +927,18 @@ }, "api": [ { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"],["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"],["DenseMatrix", "float"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ] ] } ] @@ -848,19 +1247,39 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { - "name" : ["CPP"], + "name": ["CPP"], "instantiations": [ ["Frame", "Frame", "Frame"], - [["CSRMatrix", "double"], ["CSRMatrix", "double"], ["CSRMatrix", "double"]], - [["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"]] + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ] ] } ] @@ -894,9 +1313,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -908,10 +1325,10 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [[]] - - }] + } + ] }, { "kernelTemplate": { @@ -923,10 +1340,10 @@ }, "api": [ { - "name": ["FPGAOPENCL"], + "name": ["FPGAOPENCL"], "instantiations": [[]] - - }] + } + ] }, { "kernelTemplate": { @@ -936,9 +1353,17 @@ "templateParams": [], "runtimeParams": [] }, - "instantiations": [ - [] - ] + "instantiations": [[]] + }, + { + "kernelTemplate": { + "header": "CreateHDFSContext.h", + "opName": "createHDFSContext", + "returnType": "void", + "templateParams": [], + "runtimeParams": [] + }, + "instantiations": [[]] }, { "kernelTemplate": { @@ -1051,9 +1476,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -1106,19 +1529,79 @@ ] }, "instantiations": [ - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t"], - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "double"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], "int64_t"], - [["DenseMatrix", "double"], ["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], "double"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], ["DenseMatrix", "double"], "int64_t"], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"], "double"], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "int64_t" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "double" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + "int64_t" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + "double" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "int64_t" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "double" + ], - [["CSRMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t"], - [["CSRMatrix", "double"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "double"], - [["CSRMatrix", "int64_t"], ["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], "int64_t"], - [["CSRMatrix", "double"], ["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], "double"], - [["CSRMatrix", "int64_t"], ["DenseMatrix", "double"], ["DenseMatrix", "double"], "int64_t"], - [["CSRMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"], "double"] + [ + ["CSRMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "int64_t" + ], + [ + ["CSRMatrix", "double"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "double" + ], + [ + ["CSRMatrix", "int64_t"], + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + "int64_t" + ], + [ + ["CSRMatrix", "double"], + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + "double" + ], + [ + ["CSRMatrix", "int64_t"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "int64_t" + ], + [ + ["CSRMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "double" + ] ] }, { @@ -1129,9 +1612,7 @@ "templateParams": [], "runtimeParams": [] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -1160,18 +1641,54 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["CSRMatrix", "double"], ["DenseMatrix", "double"]], - [["CSRMatrix", "float"], ["DenseMatrix", "float"]], - [["CSRMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["CSRMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["CSRMatrix", "double"], ["CSRMatrix", "double"]], - [["CSRMatrix", "float"], ["CSRMatrix", "float"]], - [["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"]], - [["CSRMatrix", "int32_t"], ["CSRMatrix", "int32_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["CSRMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["CSRMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["CSRMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["CSRMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["CSRMatrix", "float"], + ["CSRMatrix", "float"] + ], + [ + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], + [ + ["CSRMatrix", "int32_t"], + ["CSRMatrix", "int32_t"] + ] ] }, { @@ -1201,14 +1718,38 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "double"], ["CSRMatrix", "double"]], - [["DenseMatrix", "float"], ["CSRMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["CSRMatrix", "int64_t"]], - [["DenseMatrix", "int32_t"], ["CSRMatrix", "int32_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["CSRMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["CSRMatrix", "int32_t"] + ] ] }, { @@ -1251,34 +1792,113 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"] + ] ], - "opCodes": ["ADD", "SUB", "MUL", "DIV", "POW", "LOG", "MOD", "EQ", "NEQ", "LT", "LE", "GT", "GE", "MIN", "MAX", "AND", "OR"] + "opCodes": [ + "ADD", + "SUB", + "MUL", + "DIV", + "POW", + "LOG", + "MOD", + "EQ", + "NEQ", + "LT", + "LE", + "GT", + "GE", + "MIN", + "MAX", + "AND", + "OR" + ] }, { - "name": ["example_for_commented_or_unimplemented"], + "name": ["example_for_commented_or_unimplemented"], "instantiations": [ - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"]] + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"] + ] ], "opCodes": ["BIT_AND", "BIT_OR"] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["CSRMatrix", "double"], ["CSRMatrix", "double"], ["DenseMatrix", "double"]], - [["CSRMatrix", "double"], ["CSRMatrix", "double"], ["CSRMatrix", "double"]], - [["CSRMatrix", "float"], ["CSRMatrix", "float"], ["DenseMatrix", "float"]], - [["CSRMatrix", "float"], ["CSRMatrix", "float"], ["CSRMatrix", "float"]] - + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["CSRMatrix", "float"], + ["CSRMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["CSRMatrix", "float"], + ["CSRMatrix", "float"], + ["CSRMatrix", "float"] + ] ], - "opCodes": ["ADD", "SUB", "MUL", "DIV", "POW", "LOG", "MOD", "EQ", "NEQ", "LT", "LE", "GT", "GE", "MIN", "MAX", "AND", "OR"] + "opCodes": [ + "ADD", + "SUB", + "MUL", + "DIV", + "POW", + "LOG", + "MOD", + "EQ", + "NEQ", + "LT", + "LE", + "GT", + "GE", + "MIN", + "MAX", + "AND", + "OR" + ] } ] }, @@ -1322,25 +1942,87 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"], "float"], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], "double"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t"], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], "int32_t"], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"], "uint32_t"], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"], "uint64_t"] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + "float" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "double" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "int64_t" + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + "int32_t" + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"], + "uint32_t" + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"], + "uint64_t" + ] ], - "opCodes": ["ADD", "SUB", "MUL", "DIV", "POW", "LOG", "MOD", "EQ", "NEQ", "LT", "LE", "GT", "GE", "MIN", "MAX", "AND", "OR", "BITWISE_AND"] + "opCodes": [ + "ADD", + "SUB", + "MUL", + "DIV", + "POW", + "LOG", + "MOD", + "EQ", + "NEQ", + "LT", + "LE", + "GT", + "GE", + "MIN", + "MAX", + "AND", + "OR", + "BITWISE_AND" + ] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ ["Frame", "Frame", "float"], ["Frame", "Frame", "double"], ["Frame", "Frame", "int64_t"] ], - "opCodes": ["ADD", "SUB", "MUL", "DIV", "POW", "LOG", "MOD", "EQ", "NEQ", "LT", "LE", "GT", "GE", "MIN", "MAX", "AND", "OR", "BITWISE_AND"] + "opCodes": [ + "ADD", + "SUB", + "MUL", + "DIV", + "POW", + "LOG", + "MOD", + "EQ", + "NEQ", + "LT", + "LE", + "GT", + "GE", + "MIN", + "MAX", + "AND", + "OR", + "BITWISE_AND" + ] } ] }, @@ -1388,7 +2070,27 @@ ["const char *", "const char *", "const char *"], ["int64_t", "const char *", "const char *"] ], - "opCodes": ["ADD", "SUB", "MUL", "DIV", "POW", "LOG", "MOD", "EQ", "NEQ", "LT", "LE", "GT", "GE", "MIN", "MAX", "AND", "OR", "BITWISE_AND", "CONCAT"] + "opCodes": [ + "ADD", + "SUB", + "MUL", + "DIV", + "POW", + "LOG", + "MOD", + "EQ", + "NEQ", + "LT", + "LE", + "GT", + "GE", + "MIN", + "MAX", + "AND", + "OR", + "BITWISE_AND", + "CONCAT" + ] }, { "kernelTemplate": { @@ -1429,16 +2131,45 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"] + ] ], "opCodes": [ - "ADD", "SUB", "MUL", "DIV", "POW", "MOD", "LOG", - "MIN", "MAX", - "AND", "OR", - "EQ", "NEQ", "LT", "LE", "GT", "GE" + "ADD", + "SUB", + "MUL", + "DIV", + "POW", + "MOD", + "LOG", + "MIN", + "MAX", + "AND", + "OR", + "EQ", + "NEQ", + "LT", + "LE", + "GT", + "GE" ] }, { @@ -1484,12 +2215,42 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "double"], ["DenseMatrix", "bool"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "bool"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "bool"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "bool"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -1535,12 +2296,42 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], "double"], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"], "double"], - [["DenseMatrix", "double"], ["DenseMatrix", "bool"], ["DenseMatrix", "double"], "double"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], "int64_t"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "bool"], ["DenseMatrix", "int64_t"], "int64_t"] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + "double" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "double" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "bool"], + ["DenseMatrix", "double"], + "double" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "int64_t" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + "int64_t" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "bool"], + ["DenseMatrix", "int64_t"], + "int64_t" + ] ] }, { @@ -1586,12 +2377,42 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], "double", ["DenseMatrix", "double"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], "double", ["DenseMatrix", "double"]], - [["DenseMatrix", "double"], ["DenseMatrix", "bool"], "double", ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t", ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], "int64_t", ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "bool"], "int64_t", ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + "double", + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "double", + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "bool"], + "double", + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "int64_t", + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + "int64_t", + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "bool"], + "int64_t", + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -1637,12 +2458,42 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"], "double", "double"], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], "double", "double"], - [["DenseMatrix", "double"], ["DenseMatrix", "bool"], "double", "double"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t", "int64_t"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], "int64_t", "int64_t"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "bool"], "int64_t", "int64_t"] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"], + "double", + "double" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + "double", + "double" + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "bool"], + "double", + "double" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "int64_t", + "int64_t" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + "int64_t", + "int64_t" + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "bool"], + "int64_t", + "int64_t" + ] ] }, { @@ -1681,18 +2532,32 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] + ] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ ["Frame", "Frame", "char"], ["Frame", "Frame", ["DenseMatrix", "int64_t"]], - ["Frame", "Frame", ["DenseMatrix", "size_t"]]] + ["Frame", "Frame", ["DenseMatrix", "size_t"]] + ] } ] }, @@ -1732,14 +2597,15 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ [["DenseMatrix", "float"], "float"], [["DenseMatrix", "double"], "double"], [["DenseMatrix", "int64_t"], "int64_t"], [["DenseMatrix", "uint64_t"], "uint64_t"], [["DenseMatrix", "uint8_t"], "uint8_t"], - [["DenseMatrix", "bool"], "bool"]] + [["DenseMatrix", "bool"], "bool"] + ] } ] }, @@ -1906,7 +2772,11 @@ [["DenseMatrix", "double"], ["DenseMatrix", "double"], "int64_t"], [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t"], [["DenseMatrix", "double"], ["DenseMatrix", "double"], "uint64_t"], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "uint64_t"], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + "uint64_t" + ], ["Frame", "Frame", "double"], ["Frame", "Frame", "int64_t"] ] @@ -1953,50 +2823,45 @@ } ] }, - "instantiations": [ - ["int64_t"], - ["size_t"] - ] + "instantiations": [["int64_t"], ["size_t"]] }, { - "kernelTemplate": { - "header": "InnerJoin.h", - "opName": "innerJoin", - "returnType": "void", - "templateParams": [], - "runtimeParams": [ - { - "type": "Frame *&", - "name": "res" - }, - { - "type": "const Frame *", - "name": "lhs" - }, - { - "type": "const Frame *", - "name": "rhs" - }, - { - "type": "const char *", - "name": "lhsOn" - }, - { - "type": "const char *", - "name": "rhsOn" - } - ] - }, - "instantiations": [ - [] - ] + "kernelTemplate": { + "header": "InnerJoin.h", + "opName": "innerJoin", + "returnType": "void", + "templateParams": [], + "runtimeParams": [ + { + "type": "Frame *&", + "name": "res" + }, + { + "type": "const Frame *", + "name": "lhs" + }, + { + "type": "const Frame *", + "name": "rhs" + }, + { + "type": "const char *", + "name": "lhsOn" + }, + { + "type": "const char *", + "name": "rhsOn" + } + ] + }, + "instantiations": [[]] }, { - "kernelTemplate": { - "header": "ThetaJoin.h", - "opName": "thetaJoin", - "returnType": "void", - "templateParams": [ + "kernelTemplate": { + "header": "ThetaJoin.h", + "opName": "thetaJoin", + "returnType": "void", + "templateParams": [ { "name": "DTRes", "isDataType": true @@ -2010,48 +2875,46 @@ "isDataType": true } ], - "runtimeParams": [ - { - "type": "DTRes *&", - "name": "res" - }, - { - "type": "const DTLhs *", - "name": "lhs" - }, - { - "type": "const DTRhs *", - "name": "rhs" - }, - { - "type": "const char **", - "name": "lhsOn" - }, - { - "type": "size_t", - "name": "numLhsOn" - }, - { - "type": "const char **", - "name": "rhsOn" - }, - { - "type": "size_t", - "name": "numRhsOn" - }, - { - "type": "CompareOperation *", - "name": "cmp" - }, - { - "type": "size_t", - "name": "numCmp" - } - ] - }, - "instantiations": [ - ["Frame", "Frame", "Frame"] - ] + "runtimeParams": [ + { + "type": "DTRes *&", + "name": "res" + }, + { + "type": "const DTLhs *", + "name": "lhs" + }, + { + "type": "const DTRhs *", + "name": "rhs" + }, + { + "type": "const char **", + "name": "lhsOn" + }, + { + "type": "size_t", + "name": "numLhsOn" + }, + { + "type": "const char **", + "name": "rhsOn" + }, + { + "type": "size_t", + "name": "numRhsOn" + }, + { + "type": "CompareOperation *", + "name": "cmp" + }, + { + "type": "size_t", + "name": "numCmp" + } + ] + }, + "instantiations": [["Frame", "Frame", "Frame"]] }, { "kernelTemplate": { @@ -2084,77 +2947,269 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "double"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "double"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "float"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "float"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint8_t"]], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint8_t"] + ], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint8_t"]] + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint8_t"] + ] ] }, { @@ -2201,28 +3256,68 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "double"], ["CSRMatrix", "double"], ["DenseMatrix", "double"]], - [["CSRMatrix", "double"], ["CSRMatrix", "double"], ["CSRMatrix", "double"]], - [["CSRMatrix", "float"], ["CSRMatrix", "float"], ["CSRMatrix", "float"]], - [["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"]], - [["CSRMatrix", "int32_t"], ["CSRMatrix", "int32_t"], ["CSRMatrix", "int32_t"]] + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["CSRMatrix", "float"], + ["CSRMatrix", "float"], + ["CSRMatrix", "float"] + ], + [ + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], + [ + ["CSRMatrix", "int32_t"], + ["CSRMatrix", "int32_t"], + ["CSRMatrix", "int32_t"] + ] ] }, - { - "name": ["FPGAOPENCL"], + { + "name": ["FPGAOPENCL"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ] ] } ] @@ -2235,9 +3330,7 @@ "templateParams": [], "runtimeParams": [] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -2270,8 +3363,14 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -2313,9 +3412,21 @@ ] }, "instantiations": [ - [["DenseMatrix", "int64_t"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -2357,9 +3468,18 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -2591,8 +3711,14 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -2926,15 +4052,42 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"]], - [["DenseMatrix", "int8_t"], ["DenseMatrix", "int8_t"]], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"]], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"]], - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint8_t"]], - [["DenseMatrix", "bool"], ["DenseMatrix", "bool"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "int32_t"], + ["DenseMatrix", "int32_t"] + ], + [ + ["DenseMatrix", "int8_t"], + ["DenseMatrix", "int8_t"] + ], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"] + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"] + ], + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "uint8_t"] + ], + [ + ["DenseMatrix", "bool"], + ["DenseMatrix", "bool"] + ] ] }, { @@ -2975,10 +4128,7 @@ } ] }, - "instantiations": [ - ["int64_t"], - ["size_t"] - ] + "instantiations": [["int64_t"], ["size_t"]] }, { "kernelTemplate": { @@ -3005,9 +4155,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -3030,9 +4178,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -3062,19 +4208,42 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "size_t"]]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "size_t"] + ] + ] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["CSRMatrix", "double"], ["CSRMatrix", "double"]], - [["CSRMatrix", "float"], ["CSRMatrix", "float"]], - [["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"]]] + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["CSRMatrix", "float"], + ["CSRMatrix", "float"] + ], + [ + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ] + ] } ] }, @@ -3109,11 +4278,36 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"],["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"],["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ] ], - "opCodes": ["MINUS", "SIGN", "SQRT", "EXP", "ABS", "FLOOR", "CEIL", "ROUND", "LN", - "SIN", "COS", "TAN", "ASIN", "ACOS", "ATAN", "SINH", "COSH", "TANH", "ISNAN"] + "opCodes": [ + "MINUS", + "SIGN", + "SQRT", + "EXP", + "ABS", + "FLOOR", + "CEIL", + "ROUND", + "LN", + "SIN", + "COS", + "TAN", + "ASIN", + "ACOS", + "ATAN", + "SINH", + "COSH", + "TANH", + "ISNAN" + ] }, { "kernelTemplate": { @@ -3146,8 +4340,27 @@ ["float", "float"], ["int64_t", "int64_t"] ], - "opCodes": ["MINUS", "SIGN", "SQRT", "EXP", "ABS", "FLOOR", "CEIL", "ROUND", "LN", - "SIN", "COS", "TAN", "ASIN", "ACOS", "ATAN", "SINH", "COSH", "TANH", "ISNAN"] + "opCodes": [ + "MINUS", + "SIGN", + "SQRT", + "EXP", + "ABS", + "FLOOR", + "CEIL", + "ROUND", + "LN", + "SIN", + "COS", + "TAN", + "ASIN", + "ACOS", + "ATAN", + "SINH", + "COSH", + "TANH", + "ISNAN" + ] }, { "kernelTemplate": { @@ -3167,7 +4380,7 @@ } ] }, - "instantiations": [ + "instantiations": [ [["DenseMatrix", "double"]], [["DenseMatrix", "float"]], [["DenseMatrix", "int64_t"]], @@ -3197,9 +4410,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -3214,9 +4425,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -3231,9 +4440,7 @@ } ] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -3307,10 +4514,19 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] + ] } ] }, @@ -3342,17 +4558,27 @@ }, "api": [ { - "name": ["CUDA", "CPP"], - "instantiations": [[["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]]] + "name": ["CUDA", "CPP"], + "instantiations": [ + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] + ] }, - { - "name": ["FPGAOPENCL"], + { + "name": ["FPGAOPENCL"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ] ] } - ] }, { @@ -3391,19 +4617,31 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"], ["DenseMatrix", "float"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ] ] }, { - "name": ["CPP"], + "name": ["CPP"], "instantiations": [ - [["DenseMatrix", "double"], ["CSRMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "double"], + ["CSRMatrix", "double"], + ["DenseMatrix", "double"] + ] ] } - ] + ] }, { "kernelTemplate": { @@ -3531,10 +4769,7 @@ } ] }, - "instantiations": [ - ["Structure"], - ["char"] - ] + "instantiations": [["Structure"], ["char"]] }, { "kernelTemplate": { @@ -3554,10 +4789,7 @@ } ] }, - "instantiations": [ - ["Structure"], - ["char"] - ] + "instantiations": [["Structure"], ["char"]] }, { "kernelTemplate": { @@ -3698,8 +4930,16 @@ [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], "int64_t"], [["DenseMatrix", "int32_t"], ["DenseMatrix", "int32_t"], "int64_t"], [["DenseMatrix", "int8_t"], ["DenseMatrix", "int8_t"], "int64_t"], - [["DenseMatrix", "uint64_t"], ["DenseMatrix", "uint64_t"], "int64_t"], - [["DenseMatrix", "uint32_t"], ["DenseMatrix", "uint32_t"], "int64_t"], + [ + ["DenseMatrix", "uint64_t"], + ["DenseMatrix", "uint64_t"], + "int64_t" + ], + [ + ["DenseMatrix", "uint32_t"], + ["DenseMatrix", "uint32_t"], + "int64_t" + ], [["DenseMatrix", "uint8_t"], ["DenseMatrix", "uint8_t"], "int64_t"], [["DenseMatrix", "bool"], ["DenseMatrix", "bool"], "int64_t"] ] @@ -3784,23 +5024,29 @@ "name": "data" }, { "type": "size_t", "name": "batch_size" }, - { "type": "size_t", "name": "num_channls"}, - { "type": "size_t", "name": "img_h"}, - { "type": "size_t", "name": "img_w"}, - { "type": "size_t", "name": "pool_h"}, - { "type": "size_t", "name": "pool_w"}, - { "type": "size_t", "name": "stride_h"}, - { "type": "size_t", "name": "stride_w"}, - { "type": "size_t", "name": "pad_h"}, - { "type": "size_t", "name": "pad_w"} + { "type": "size_t", "name": "num_channls" }, + { "type": "size_t", "name": "img_h" }, + { "type": "size_t", "name": "img_w" }, + { "type": "size_t", "name": "pool_h" }, + { "type": "size_t", "name": "pool_w" }, + { "type": "size_t", "name": "stride_h" }, + { "type": "size_t", "name": "stride_w" }, + { "type": "size_t", "name": "pad_h" }, + { "type": "size_t", "name": "pad_w" } ] }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ], "opCodes": ["AVG", "MAX"] } @@ -3841,10 +5087,26 @@ ] }, "instantiations": [ - [["DenseMatrix", "double"], ["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["CSRMatrix", "double"], ["CSRMatrix", "double"], ["CSRMatrix", "double"]], - [["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"], ["CSRMatrix", "int64_t"]], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["CSRMatrix", "double"], + ["CSRMatrix", "double"], + ["CSRMatrix", "double"] + ], + [ + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"], + ["CSRMatrix", "int64_t"] + ], ["Frame", "Frame", "Frame"] ] }, @@ -3873,12 +5135,15 @@ "name": "arg" }, { "type": "float", "name": "min" }, - { "type": "float", "name": "max"} + { "type": "float", "name": "max" } ] - }, + }, "instantiations": [ - [["DenseMatrix", "uint8_t"], ["DenseMatrix", "float"]] - ] + [ + ["DenseMatrix", "uint8_t"], + ["DenseMatrix", "float"] + ] + ] }, { "kernelTemplate": { @@ -3913,10 +5178,16 @@ }, "api": [ { - "name": ["CUDA", "CPP"], + "name": ["CUDA", "CPP"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ], "opCodes": ["ReLU"] } @@ -3959,10 +5230,16 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ] } ] @@ -4016,10 +5293,16 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ] } ] @@ -4109,10 +5392,16 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ] } ] @@ -4146,10 +5435,16 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ] } ] @@ -4187,10 +5482,16 @@ }, "api": [ { - "name": ["CUDA"], + "name": ["CUDA"], "instantiations": [ - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]] + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ] ] } ] @@ -4246,12 +5547,30 @@ "instantiations": [ ["Frame", "Frame"], [["DenseMatrix", "size_t"], "Frame"], - [["DenseMatrix", "double"], ["DenseMatrix", "double"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "double"]], - [["DenseMatrix", "float"], ["DenseMatrix", "float"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "float"]], - [["DenseMatrix", "int64_t"], ["DenseMatrix", "int64_t"]], - [["DenseMatrix", "size_t"], ["DenseMatrix", "int64_t"]] + [ + ["DenseMatrix", "double"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "double"] + ], + [ + ["DenseMatrix", "float"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "float"] + ], + [ + ["DenseMatrix", "int64_t"], + ["DenseMatrix", "int64_t"] + ], + [ + ["DenseMatrix", "size_t"], + ["DenseMatrix", "int64_t"] + ] ] }, { @@ -4301,9 +5620,7 @@ } ] }, - "instantiations": [ - ["Frame"] - ] + "instantiations": [["Frame"]] }, { "kernelTemplate": { @@ -4357,10 +5674,8 @@ }, "api": [ { - "name": ["CPP"], - "instantiations": [ - [["DenseMatrix", "double"]] - ] + "name": ["CPP"], + "instantiations": [[["DenseMatrix", "double"]]] } ] }, @@ -4372,9 +5687,7 @@ "templateParams": [], "runtimeParams": [] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": { @@ -4384,9 +5697,7 @@ "templateParams": [], "runtimeParams": [] }, - "instantiations": [ - [] - ] + "instantiations": [[]] }, { "kernelTemplate": {