diff --git a/doc/DaphneLib/Overview.md b/doc/DaphneLib/Overview.md index 44fdb30a6..46efe6ea7 100644 --- a/doc/DaphneLib/Overview.md +++ b/doc/DaphneLib/Overview.md @@ -556,12 +556,80 @@ tensor([[[100.5474, 100.9653], [100.3148, 100.3607]]], dtype=torch.float64) ``` +## Command-Line Arguments to Influence DAPHNE Behavior +DAPHNE provides a range of command-line arguments that allow users to control its behavior and customize execution settings. +These arguments can also be passed from DaphneLib. + +*Example Array Format:* + +```python +from daphne.context.daphne_context import DaphneContext +import numpy as np + +m1 = np.array([1, 2, 3]) +dc = DaphneContext() +X = dc.from_numpy(m1) +X.print().compute(daphne_args=["--explain", "parsing_simplified, parsing", "--timing"]) +``` + +*Example String Format:* + +```python +from daphne.context.daphne_context import DaphneContext +import numpy as np + +m1 = np.array([1, 2, 3]) +dc = DaphneContext() +X = dc.from_numpy(m1) +X.print().compute(daphne_args="--explain=parsing_simplified,parsing --timing") +``` + +*Output (memory addresses may vary):* + +``` +IR after parsing: +module { + func.func @main() { + %0 = "daphne.constant"() {value = 0 : si64} : () -> si64 + %1 = "daphne.constant"() {value = 43781056 : si64} : () -> si64 + %2 = "daphne.constant"() {value = 3 : si64} : () -> si64 + %3 = "daphne.constant"() {value = 1 : si64} : () -> si64 + %4 = "daphne.constant"() {value = 2 : si64} : () -> si64 + %5 = "daphne.cast"(%0) : (si64) -> ui32 + %6 = "daphne.cast"(%1) : (si64) -> ui32 + %7 = "daphne.receiveFromNumpy"(%5, %6, %2, %3) : (ui32, ui32, si64, si64) -> !daphne.Matrix + %8 = "daphne.constant"() {value = true} : () -> i1 + %9 = "daphne.constant"() {value = false} : () -> i1 + "daphne.print"(%7, %8, %9) : (!daphne.Matrix, i1, i1) -> () + "daphne.return"() : () -> () + } +} +IR after parsing and some simplifications: +module { + func.func @main() { + %0 = "daphne.constant"() {value = 43781056 : ui32} : () -> ui32 + %1 = "daphne.constant"() {value = 0 : ui32} : () -> ui32 + %2 = "daphne.constant"() {value = false} : () -> i1 + %3 = "daphne.constant"() {value = true} : () -> i1 + %4 = "daphne.constant"() {value = 3 : si64} : () -> si64 + %5 = "daphne.constant"() {value = 1 : si64} : () -> si64 + %6 = "daphne.receiveFromNumpy"(%1, %0, %4, %5) : (ui32, ui32, si64, si64) -> !daphne.Matrix + "daphne.print"(%6, %3, %2) : (!daphne.Matrix, i1, i1) -> () + "daphne.return"() : () -> () + } +} +DenseMatrix(3x1, int64_t) +1 +2 +3 +{"startup_seconds": 0.0238036, "parsing_seconds": 0.00178526, "compilation_seconds": 0.0734249, "execution_seconds": 0.0232295, "total_seconds": 0.122243} +``` + ## Known Limitations DaphneLib is still in an early development stage. Thus, there are a few limitations that users should be aware of. We plan to fix all of these limitations in the future. -- Using DAPHNE's command-line arguments to influence its behavior is not supported yet. - Some DaphneDSL built-in functions are not represented by DaphneLib methods yet. - High-level primitives for integrated data analysis pipelines, which are implemented in DaphneDSL, cannot be called from DaphneLib yet. diff --git a/src/api/daphnelib/daphnelib.cpp b/src/api/daphnelib/daphnelib.cpp index 16c129e4c..a6f2c036b 100644 --- a/src/api/daphnelib/daphnelib.cpp +++ b/src/api/daphnelib/daphnelib.cpp @@ -31,9 +31,4 @@ extern "C" DaphneLibResult getResult() { return daphneLibRes; } * @brief Invokes DAPHNE with the specified DaphneDSL script and path to lib * dir. */ -extern "C" int daphne(const char *libDirPath, const char *scriptPath) { - const char *argv[] = {"daphne", "--libdir", libDirPath, scriptPath}; - int argc = 4; - - return mainInternal(argc, argv, &daphneLibRes); -} \ No newline at end of file +extern "C" int daphne(int argc, const char **argv) { return mainInternal(argc, argv, &daphneLibRes); } \ No newline at end of file diff --git a/src/api/python/daphne/operator/nodes/matrix.py b/src/api/python/daphne/operator/nodes/matrix.py index 1f9b79f00..03d340ec0 100644 --- a/src/api/python/daphne/operator/nodes/matrix.py +++ b/src/api/python/daphne/operator/nodes/matrix.py @@ -101,8 +101,8 @@ def getDType(self, d_type): def _is_numpy(self) -> bool: return self._np_array is not None - def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None) -> Union[np.array]: - return super().compute(type=type, verbose=verbose, asTensorFlow=asTensorFlow, asPyTorch=asPyTorch, shape=shape) + def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None, daphne_args=None) -> Union[np.array]: + return super().compute(type=type, verbose=verbose, asTensorFlow=asTensorFlow, asPyTorch=asPyTorch, shape=shape, daphne_args=daphne_args) def __add__(self, other: VALID_ARITHMETIC_TYPES) -> 'Matrix': return Matrix(self.daphne_context, '+', [self, other]) diff --git a/src/api/python/daphne/operator/operation_node.py b/src/api/python/daphne/operator/operation_node.py index 5b5e0a854..f6796cf82 100644 --- a/src/api/python/daphne/operator/operation_node.py +++ b/src/api/python/daphne/operator/operation_node.py @@ -91,7 +91,7 @@ def update_node_in_input_list(self, new_node, current_node): current_index = self._unnamed_input_nodes.index(current_node) self._unnamed_input_nodes[current_index] = new_node - def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None, useIndexColumn=False): + def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None, useIndexColumn=False, daphne_args=None): """ Compute function for processing the Daphne Object or operation node and returning the results. The function builds a DaphneDSL script from the node and its context, executes it, and processes the results @@ -103,6 +103,7 @@ def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyT :param asPyTorch: If True and the result is a matrix, the output will be converted to a PyTorch tensor. :param shape: If provided and the result is a matrix, it defines the shape to reshape the resulting tensor (either TensorFlow or PyTorch). :param useIndexColumn: If True and the result is a DataFrame, uses the column named "index" as the DataFrame's index. + :param daphne_args: Optional arguments specifically for Daphne DSL script execution, which can customize runtime behavior. :return: Depending on the parameters and the operation's output type, this function can return: - A pandas DataFrame for frame outputs. @@ -123,7 +124,7 @@ def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyT if verbose: exec_start_time = time.time() - self._script.execute() + self._script.execute(daphne_args) self._script.clear(self) if verbose: diff --git a/src/api/python/daphne/script_building/script.py b/src/api/python/daphne/script_building/script.py index f1d5d01b0..5e893fc0b 100644 --- a/src/api/python/daphne/script_building/script.py +++ b/src/api/python/daphne/script_building/script.py @@ -27,6 +27,7 @@ import ctypes import os +import shlex from typing import List, Dict, TYPE_CHECKING if TYPE_CHECKING: @@ -88,23 +89,43 @@ def clear(self, dag_root:DAGNode): self._dfs_clear_dag_nodes(dag_root) self._variable_counter = 0 - def execute(self): + def execute(self, daphne_args): temp_out_path = os.path.join(TMP_PATH, "tmpdaphne.daphne") - temp_out_file = open(temp_out_path, "w") - temp_out_file.writelines(self.daphnedsl_script) - temp_out_file.close() - - #os.environ['OPENBLAS_NUM_THREADS'] = '1' - res = DaphneLib.daphne(ctypes.c_char_p(str.encode(PROTOTYPE_PATH)), ctypes.c_char_p(str.encode(temp_out_path))) + with open(temp_out_path, "w") as temp_out_file: + temp_out_file.writelines(self.daphnedsl_script) + + # Construct argv + argv = ["daphne", "--libdir", PROTOTYPE_PATH] + + # Handle daphne_args + if isinstance(daphne_args, str): + argv.extend(shlex.split(daphne_args)) + elif isinstance(daphne_args, list): + argv.extend(daphne_args) + elif daphne_args is None: + pass + else: + raise TypeError("daphne_args must be a string or a list") + + # Add the script path + argv.append(temp_out_path) + + # Convert argv to ctypes + argc = len(argv) + argv_ctypes = (ctypes.c_char_p * argc)(* [arg.encode('utf-8') for arg in argv]) + + # Set argument and return types for the C function + DaphneLib.daphne.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] + DaphneLib.daphne.restype = ctypes.c_int + + res = DaphneLib.daphne(ctypes.c_int(argc), argv_ctypes) if res != 0: # Error message with DSL code line. error_message = DaphneLib.getResult().error_message.decode("utf-8") + raise RuntimeError(f"Error in DaphneDSL script: {error_message}") # Remove DSL code line from error message. # index_code_line = error_message.find("Source file ->") - 29 # error_message = error_message[:index_code_line] - - raise RuntimeError(f"Error in DaphneDSL script: {error_message}") - #os.environ['OPENBLAS_NUM_THREADS'] = '32' def _dfs_dag_nodes(self, dag_node: VALID_INPUT_TYPES)->str: """Uses Depth-First-Search to create code from DAG diff --git a/test/api/python/DaphneLibTest.cpp b/test/api/python/DaphneLibTest.cpp index 95179f171..8bcd27b43 100644 --- a/test/api/python/DaphneLibTest.cpp +++ b/test/api/python/DaphneLibTest.cpp @@ -26,6 +26,9 @@ #include #include +#include +#include + const std::string dirPath = "test/api/python/"; #define MAKE_TEST_CASE(name) \ @@ -65,6 +68,18 @@ const std::string dirPath = "test/api/python/"; const std::string prefix = dirPath + name; \ compareDaphneLibToStr(str, prefix + ".py"); \ } +#define MAKE_TEST_CASE_FILE(name) \ + TEST_CASE(name ".py", "[own]") { \ + const std::string prefix = dirPath + name; \ + std::ifstream fileStream(prefix + ".txt"); \ + std::stringstream buffer; \ + buffer << fileStream.rdbuf(); \ + std::stringstream out; \ + std::stringstream err; \ + int status = runDaphneLib(out, err, (prefix + ".py").c_str()); \ + CHECK(status == StatusCode::SUCCESS); \ + CHECK(err.str() != ""); \ + CHECK(out.str() == buffer.str()); MAKE_TEST_CASE("data_transfer_numpy_1") MAKE_TEST_CASE("data_transfer_numpy_2") @@ -158,3 +173,4 @@ MAKE_TEST_CASE("user_def_func_3_inputs") // MAKE_TEST_CASE_PARAMETRIZED("user_def_func_with_condition", "param=3.8") // MAKE_TEST_CASE("user_def_func_with_for_loop") // MAKE_TEST_CASE("user_def_func_with_while_loop") +MAKE_TEST_CASE_FILE("daphne_args") diff --git a/test/api/python/daphne_args.py b/test/api/python/daphne_args.py new file mode 100644 index 000000000..e0334648b --- /dev/null +++ b/test/api/python/daphne_args.py @@ -0,0 +1,10 @@ +import numpy as np +from daphne.context.daphne_context import DaphneContext + +m1 = np.array([3, 9, 12]) +dc = DaphneContext() +X = dc.from_numpy(m1) +X.print().compute(daphne_args=["--vec", "--pin-workers"]) +X.print().compute(daphne_args=["--explain", "parsing_simplified"]) +X.print().compute(daphne_args=["--explain", "parsing_simplified, parsing", "--timing"]) +X.print().compute(daphne_args="--explain=parsing_simplified,parsing --timing") \ No newline at end of file diff --git a/test/api/python/daphne_args.txt b/test/api/python/daphne_args.txt new file mode 100644 index 000000000..38e8310b1 --- /dev/null +++ b/test/api/python/daphne_args.txt @@ -0,0 +1,16 @@ +DenseMatrix(3x1, int64_t) +3 +9 +12 +DenseMatrix(3x1, int64_t) +3 +9 +12 +DenseMatrix(3x1, int64_t) +3 +9 +12 +DenseMatrix(3x1, int64_t) +3 +9 +12