diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c90948941..b71c5d984 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -69,6 +69,5 @@ jobs: '{ballista,docs}/**/*.md' \ '!ballista/CHANGELOG.md' \ README.md \ - CONTRIBUTING.md \ - 'ballista/**/*.{ts,tsx}' + CONTRIBUTING.md git diff --exit-code diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 454235617..db1b7fc92 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -72,44 +72,6 @@ jobs: /github/home/target/release/ballista-scheduler /github/home/target/release/ballista-executor - react-build: - name: React build - runs-on: ubuntu-latest # proprietary github image, not ubuntu:latest - container: - image: ubuntu:latest # actual ubuntu:latest that ubuntu publishes - env: - DEBIAN_FRONTEND: "noninteractive" - steps: - - uses: actions/checkout@v3 - - name: Install dependencies - run: | - cat /etc/*-release - apt-get -qq update - apt-get -qq upgrade - apt-get -qq install -y curl - curl -fsSL https://deb.nodesource.com/setup_18.x | bash - apt-get -qq update - apt-get -qq install -y nodejs - npm install -g yarn - which node - which npm - which yarn - node --version - npm --version - yarn --version - - name: Run yarn build - run: | - cd ballista/scheduler/ui - pwd - yarn install - yarn build - - name: Save artifacts - uses: actions/upload-artifact@v4 - with: - name: react-artifacts - path: | - ballista/scheduler/ui/build - # test the crate linux-test: name: Test Workspace on AMD64 Rust ${{ matrix.rust }} @@ -338,7 +300,7 @@ jobs: docker: name: Docker - needs: [linux-build-lib, react-build] + needs: [linux-build-lib] runs-on: ubuntu-latest permissions: contents: read @@ -350,11 +312,6 @@ jobs: with: name: rust-artifacts path: target/release - - name: Restore react artifacts - uses: actions/download-artifact@v4 - with: - name: react-artifacts - path: ballista/scheduler/ui/build - name: Build and push Docker image run: | echo "github user is $DOCKER_USER" @@ -468,7 +425,6 @@ jobs: env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" - # Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it # coverage: diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 11a7d70c2..000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -conda: - file: python/doc/environment.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3353fceec..741021619 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,6 @@ This section describes how you can get started with Ballista development. Ballista contains components implemented in the following programming languages: - Rust (Scheduler and Executor processes, Client library) -- Javascript (Scheduler Web UI) ### Rust Environment @@ -90,10 +89,6 @@ Environment variables are prefixed by `BALLISTA_EXECUTOR` or `BALLISTA_SCHEDULER respectively. Hyphens in command line arguments become underscores. For example, the `--scheduler-host` argument for the executor becomes `BALLISTA_EXECUTOR_SCHEDULER_HOST` -### Javascript Environment - -Refer to the instructions in the Scheduler Web UI [README](./ballista/scheduler/ui/README.md) - ## Integration Tests The integration tests can be executed by running the following command from the root of the repository. diff --git a/README.md b/README.md index 4b149f5d1..80c108178 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ See the [architecture guide](docs/source/contributors-guide/architecture.md) for - DataFrame and SQL APIs available from Python and Rust. - Clients can connect to a Ballista cluster using [Flight SQL][flight-sql]. - JDBC support via Arrow Flight SQL JDBC Driver -- Scheduler web interface and REST UI for monitoring query progress and viewing query plans and metrics. +- Scheduler REST UI for monitoring query progress and viewing query plans and metrics. - Support for Docker, Docker Compose, and Kubernetes deployment, as well as manual deployment on bare metal. ## Performance @@ -105,5 +105,4 @@ Please see the [Contribution Guide](CONTRIBUTING.md) for information about contr [datafusion]: https://github.com/apache/arrow-datafusion [flight]: https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/ [flight-sql]: https://arrow.apache.org/blog/2022/02/16/introducing-arrow-flight-sql/ -[ballista-talk]: https://www.youtube.com/watch?v=ZZHQaOap9pQ [user-guide]: https://datafusion.apache.org/ballista/ diff --git a/ballista/client/src/columnar_batch.rs b/ballista/client/src/columnar_batch.rs deleted file mode 100644 index 5e7fe89b0..000000000 --- a/ballista/client/src/columnar_batch.rs +++ /dev/null @@ -1,165 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::{collections::HashMap, sync::Arc}; - -use ballista_core::error::{ballista_error, Result}; - -use datafusion::arrow::{ - array::ArrayRef, - datatypes::{DataType, Schema}, - record_batch::RecordBatch, -}; -use datafusion::scalar::ScalarValue; - -pub type MaybeColumnarBatch = Result>; - -/// Batch of columnar data. -#[derive(Debug, Clone)] -pub struct ColumnarBatch { - schema: Arc, - columns: HashMap, -} - -impl ColumnarBatch { - pub fn from_arrow(batch: &RecordBatch) -> Self { - let columns = batch - .columns() - .iter() - .enumerate() - .map(|(i, array)| { - ( - batch.schema().field(i).name().clone(), - ColumnarValue::Columnar(array.clone()), - ) - }) - .collect(); - - Self { - schema: batch.schema(), - columns, - } - } - - pub fn from_values(values: &[ColumnarValue], schema: &Schema) -> Self { - let columns = schema - .fields() - .iter() - .enumerate() - .map(|(i, f)| (f.name().clone(), values[i].clone())) - .collect(); - - Self { - schema: Arc::new(schema.clone()), - columns, - } - } - - pub fn to_arrow(&self) -> Result { - let arrays = self - .schema - .fields() - .iter() - .map(|c| { - match self.column(c.name())? { - ColumnarValue::Columnar(array) => Ok(array.clone()), - ColumnarValue::Scalar(_, _) => { - // note that this can be implemented easily if needed - Err(ballista_error("Cannot convert scalar value to Arrow array")) - } - } - }) - .collect::>>()?; - - Ok(RecordBatch::try_new(self.schema.clone(), arrays)?) - } - - pub fn schema(&self) -> Arc { - self.schema.clone() - } - - pub fn num_columns(&self) -> usize { - self.columns.len() - } - - pub fn num_rows(&self) -> usize { - self.columns[self.schema.field(0).name()].len() - } - - pub fn column(&self, name: &str) -> Result<&ColumnarValue> { - Ok(&self.columns[name]) - } - - pub fn memory_size(&self) -> usize { - self.columns.values().map(|c| c.memory_size()).sum() - } -} - -/// A columnar value can either be a scalar value or an Arrow array. -#[derive(Debug, Clone)] -pub enum ColumnarValue { - Scalar(ScalarValue, usize), - Columnar(ArrayRef), -} - -impl ColumnarValue { - pub fn len(&self) -> usize { - match self { - ColumnarValue::Scalar(_, n) => *n, - ColumnarValue::Columnar(array) => array.len(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn data_type(&self) -> &DataType { - match self { - ColumnarValue::Columnar(array) => array.data_type(), - ColumnarValue::Scalar(value, _) => match value { - ScalarValue::UInt8(_) => &DataType::UInt8, - ScalarValue::UInt16(_) => &DataType::UInt16, - ScalarValue::UInt32(_) => &DataType::UInt32, - ScalarValue::UInt64(_) => &DataType::UInt64, - ScalarValue::Int8(_) => &DataType::Int8, - ScalarValue::Int16(_) => &DataType::Int16, - ScalarValue::Int32(_) => &DataType::Int32, - ScalarValue::Int64(_) => &DataType::Int64, - ScalarValue::Float32(_) => &DataType::Float32, - ScalarValue::Float64(_) => &DataType::Float64, - _ => unimplemented!(), - }, - } - } - - pub fn to_arrow(&self) -> Result { - match self { - ColumnarValue::Columnar(array) => Ok(array.clone()), - ColumnarValue::Scalar(value, n) => { - value.to_array_of_size(*n).map_err(|x| x.into()) - } - } - } - - pub fn memory_size(&self) -> usize { - match self { - ColumnarValue::Columnar(array) => array.get_array_memory_size(), - _ => 0, - } - } -} diff --git a/ballista/client/src/lib.rs b/ballista/client/src/lib.rs index 125278dcc..e61dfef28 100644 --- a/ballista/client/src/lib.rs +++ b/ballista/client/src/lib.rs @@ -17,6 +17,5 @@ #![doc = include_str!("../README.md")] -pub mod columnar_batch; pub mod context; pub mod prelude; diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml index 8a01f56fb..daabe8a27 100644 --- a/ballista/core/Cargo.toml +++ b/ballista/core/Cargo.toml @@ -60,7 +60,6 @@ futures = "0.3" hashbrown = "0.14" itertools = "0.12" -libloading = "0.8.0" log = "0.4" md-5 = { version = "^0.10.0" } object_store = { workspace = true } diff --git a/ballista/core/src/lib.rs b/ballista/core/src/lib.rs index 301bc0507..5306e8b98 100644 --- a/ballista/core/src/lib.rs +++ b/ballista/core/src/lib.rs @@ -31,8 +31,6 @@ pub mod error; pub mod event_loop; pub mod execution_plans; pub mod object_store_registry; -/// some plugins -pub mod plugin; pub mod utils; #[macro_use] diff --git a/ballista/core/src/plugin/mod.rs b/ballista/core/src/plugin/mod.rs deleted file mode 100644 index 8d2cd6b9b..000000000 --- a/ballista/core/src/plugin/mod.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::error::Result; -use crate::plugin::udf::UDFPluginManager; -use libloading::Library; -use std::any::Any; -use std::env; -use std::sync::Arc; - -/// plugin manager -pub mod plugin_manager; -/// udf plugin -pub mod udf; - -/// CARGO_PKG_VERSION -pub static CORE_VERSION: &str = env!("CARGO_PKG_VERSION"); -/// RUSTC_VERSION -pub static RUSTC_VERSION: &str = env!("RUSTC_VERSION"); - -/// Top plugin trait -pub trait Plugin { - /// Returns the plugin as [`Any`](std::any::Any) so that it can be - /// downcast to a specific implementation. - fn as_any(&self) -> &dyn Any; -} - -/// The enum of Plugin -#[repr(C)] -#[derive(PartialEq, std::cmp::Eq, std::hash::Hash, Copy, Clone)] -pub enum PluginEnum { - /// UDF/UDAF plugin - UDF, -} - -impl PluginEnum { - /// new a struct which impl the PluginRegistrar trait - pub fn init_plugin_manager(&self) -> Box { - match self { - PluginEnum::UDF => Box::::default(), - } - } -} - -/// Every plugin need a PluginDeclaration -#[derive(Copy, Clone)] -pub struct PluginDeclaration { - /// Rust doesn’t have a stable ABI, meaning different compiler versions can generate incompatible code. - /// For these reasons, the UDF plug-in must be compiled using the same version of rustc as datafusion. - pub rustc_version: &'static str, - - /// core version of the plugin. The plugin's core_version need same as plugin manager. - pub core_version: &'static str, - - /// One of PluginEnum - pub plugin_type: unsafe extern "C" fn() -> PluginEnum, -} - -/// Plugin Registrar , Every plugin need implement this trait -pub trait PluginRegistrar: Send + Sync + 'static { - /// # Safety - /// load plugin from library - unsafe fn load(&mut self, library: Arc) -> Result<()>; - - /// Returns the plugin as [`Any`](std::any::Any) so that it can be - /// downcast to a specific implementation. - fn as_any(&self) -> &dyn Any; -} - -/// Declare a plugin's PluginDeclaration. -/// -/// # Notes -/// -/// This works by automatically generating an `extern "C"` function named `get_plugin_type` with a -/// pre-defined signature and symbol name. And then generating a PluginDeclaration. -/// Therefore you will only be able to declare one plugin per library. -#[macro_export] -macro_rules! declare_plugin { - ($plugin_type:expr) => { - #[no_mangle] - pub extern "C" fn get_plugin_type() -> $crate::plugin::PluginEnum { - $plugin_type - } - - #[no_mangle] - pub static plugin_declaration: $crate::plugin::PluginDeclaration = - $crate::plugin::PluginDeclaration { - rustc_version: $crate::plugin::RUSTC_VERSION, - core_version: $crate::plugin::CORE_VERSION, - plugin_type: get_plugin_type, - }; - }; -} - -/// get the plugin dir -pub fn plugin_dir() -> String { - let current_exe_dir = match env::current_exe() { - Ok(exe_path) => exe_path.display().to_string(), - Err(_e) => "".to_string(), - }; - - // If current_exe_dir contain `deps` the root dir is the parent dir - // eg: /Users/xxx/workspace/rust_plugin_sty/target/debug/deps/plugins_app-067452b3ff2af70e - // the plugin dir is /Users/xxx/workspace/rust_plugin_sty/target/debug/deps - // else eg: /Users/xxx/workspace/rust_plugin_sty/target/debug/plugins_app - // the plugin dir is /Users/xxx/workspace/rust_plugin_sty/target/debug/ - if current_exe_dir.contains("/deps/") { - let i = current_exe_dir.find("/deps/").unwrap(); - String::from(¤t_exe_dir.as_str()[..i + 6]) - } else { - let i = current_exe_dir.rfind('/').unwrap(); - String::from(¤t_exe_dir.as_str()[..i]) - } -} diff --git a/ballista/core/src/plugin/plugin_manager.rs b/ballista/core/src/plugin/plugin_manager.rs deleted file mode 100644 index 6c19f0542..000000000 --- a/ballista/core/src/plugin/plugin_manager.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -use crate::error::{BallistaError, Result}; -use libloading::Library; -use log::info; -use std::collections::HashMap; -use std::io; -use std::sync::{Arc, Mutex}; -use walkdir::{DirEntry, WalkDir}; - -use crate::plugin::{ - PluginDeclaration, PluginEnum, PluginRegistrar, CORE_VERSION, RUSTC_VERSION, -}; -use once_cell::sync::OnceCell; - -/// To prevent the library from being loaded multiple times, we use once_cell defines a Arc> -/// Because datafusion is a library, not a service, users may not need to load all plug-ins in the process. -/// So fn global_plugin_manager return Arc>. In this way, users can load the required library through the load method of GlobalPluginManager when needed -static INSTANCE: OnceCell>> = OnceCell::new(); - -/// global_plugin_manager -pub fn global_plugin_manager( - plugin_path: &str, -) -> &'static Arc> { - INSTANCE.get_or_init(move || unsafe { - let mut gpm = GlobalPluginManager::default(); - gpm.load(plugin_path).unwrap(); - Arc::new(Mutex::new(gpm)) - }) -} - -#[derive(Default)] -/// manager all plugin_type's plugin_manager -pub struct GlobalPluginManager { - /// every plugin need a plugin registrar - pub plugin_managers: HashMap>, - - /// loaded plugin files - pub plugin_files: Vec, -} - -impl GlobalPluginManager { - /// # Safety - /// find plugin file from `plugin_path` and load it . - unsafe fn load(&mut self, plugin_path: &str) -> Result<()> { - if "".eq(plugin_path) { - return Ok(()); - } - // find library file from udaf_plugin_path - info!("load plugin from dir:{}", plugin_path); - - let plugin_files = self.get_all_plugin_files(plugin_path)?; - - for plugin_file in plugin_files { - let library = Library::new(plugin_file.path()).map_err(|e| { - BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("load library error: {e}"), - )) - })?; - - let library = Arc::new(library); - - let dec = library.get::<*mut PluginDeclaration>(b"plugin_declaration\0"); - if dec.is_err() { - info!( - "not found plugin_declaration in the library: {}", - plugin_file.path().to_str().unwrap() - ); - continue; - } - - let dec = dec.unwrap().read(); - - // ersion checks to prevent accidental ABI incompatibilities - if dec.rustc_version != RUSTC_VERSION || dec.core_version != CORE_VERSION { - return Err(BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - "Version mismatch", - ))); - } - - let plugin_enum = (dec.plugin_type)(); - let curr_plugin_manager = match self.plugin_managers.get_mut(&plugin_enum) { - None => { - let plugin_manager = plugin_enum.init_plugin_manager(); - self.plugin_managers.insert(plugin_enum, plugin_manager); - self.plugin_managers.get_mut(&plugin_enum).unwrap() - } - Some(manager) => manager, - }; - curr_plugin_manager.load(library)?; - self.plugin_files - .push(plugin_file.path().to_str().unwrap().to_string()); - } - - Ok(()) - } - - /// get all plugin file in the dir - fn get_all_plugin_files(&self, plugin_path: &str) -> io::Result> { - let mut plugin_files = Vec::new(); - for entry in WalkDir::new(plugin_path).into_iter().filter_map(|e| { - let item = e.unwrap(); - // every file only load once - if self - .plugin_files - .contains(&item.path().to_str().unwrap().to_string()) - { - return None; - } - - let file_type = item.file_type(); - if !file_type.is_file() { - return None; - } - - if let Some(path) = item.path().extension() { - if let Some(suffix) = path.to_str() { - if suffix == "dylib" || suffix == "so" || suffix == "dll" { - info!( - "load plugin from library file:{}", - item.path().to_str().unwrap() - ); - return Some(item); - } - } - } - - None - }) { - plugin_files.push(entry); - } - Ok(plugin_files) - } -} diff --git a/ballista/core/src/plugin/udf.rs b/ballista/core/src/plugin/udf.rs deleted file mode 100644 index 36ebd5e32..000000000 --- a/ballista/core/src/plugin/udf.rs +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -use crate::error::{BallistaError, Result}; -use crate::plugin::plugin_manager::global_plugin_manager; -use crate::plugin::{Plugin, PluginEnum, PluginRegistrar}; -use datafusion::logical_expr::{AggregateUDF, ScalarUDF}; -use libloading::{Library, Symbol}; -use std::any::Any; -use std::collections::HashMap; -use std::io; -use std::sync::Arc; - -/// UDF plugin trait -pub trait UDFPlugin: Plugin { - /// get a ScalarUDF by name - fn get_scalar_udf_by_name(&self, fun_name: &str) -> Result; - - /// return all udf names in the plugin - fn udf_names(&self) -> Result>; - - /// get a aggregate udf by name - fn get_aggregate_udf_by_name(&self, fun_name: &str) -> Result; - - /// return all udaf names - fn udaf_names(&self) -> Result>; -} - -/// UDFPluginManager -#[derive(Default, Clone)] -pub struct UDFPluginManager { - /// scalar udfs - pub scalar_udfs: HashMap>, - - /// aggregate udfs - pub aggregate_udfs: HashMap>, - - /// All libraries load from the plugin dir. - pub libraries: Vec>, -} - -impl PluginRegistrar for UDFPluginManager { - unsafe fn load(&mut self, library: Arc) -> Result<()> { - type PluginRegister = unsafe fn() -> Box; - let register_fun: Symbol = - library.get(b"registrar_udf_plugin\0").map_err(|e| { - BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("not found fn registrar_udf_plugin in the library: {e}"), - )) - })?; - - let udf_plugin: Box = register_fun(); - udf_plugin - .udf_names() - .unwrap() - .iter() - .try_for_each(|udf_name| { - if self.scalar_udfs.contains_key(udf_name) { - Err(BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("udf name: {udf_name} already exists"), - ))) - } else { - let scalar_udf = udf_plugin.get_scalar_udf_by_name(udf_name)?; - self.scalar_udfs - .insert(udf_name.to_string(), Arc::new(scalar_udf)); - Ok(()) - } - })?; - - udf_plugin - .udaf_names() - .unwrap() - .iter() - .try_for_each(|udaf_name| { - if self.aggregate_udfs.contains_key(udaf_name) { - Err(BallistaError::IoError(io::Error::new( - io::ErrorKind::Other, - format!("udaf name: {udaf_name} already exists"), - ))) - } else { - let aggregate_udf = - udf_plugin.get_aggregate_udf_by_name(udaf_name)?; - self.aggregate_udfs - .insert(udaf_name.to_string(), Arc::new(aggregate_udf)); - Ok(()) - } - })?; - self.libraries.push(library); - Ok(()) - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Declare a udf plugin registrar callback -/// -/// # Notes -/// -/// This works by automatically generating an `extern "C"` function named `registrar_udf_plugin` with a -/// pre-defined signature and symbol name. -/// Therefore you will only be able to declare one plugin per library. -#[macro_export] -macro_rules! declare_udf_plugin { - ($curr_plugin_type:ty, $constructor:path) => { - #[no_mangle] - pub extern "C" fn registrar_udf_plugin() -> Box { - // make sure the constructor is the correct type. - let constructor: fn() -> $curr_plugin_type = $constructor; - let object = constructor(); - Box::new(object) - } - - $crate::declare_plugin!($crate::plugin::PluginEnum::UDF); - }; -} - -/// get a Option of Immutable UDFPluginManager -pub fn get_udf_plugin_manager(path: &str) -> Option { - let udf_plugin_manager_opt = { - let gpm = global_plugin_manager(path).lock().unwrap(); - let plugin_registrar_opt = gpm.plugin_managers.get(&PluginEnum::UDF); - if let Some(plugin_registrar) = plugin_registrar_opt { - if let Some(udf_plugin_manager) = - plugin_registrar.as_any().downcast_ref::() - { - return Some(udf_plugin_manager.clone()); - } else { - return None; - } - } - None - }; - udf_plugin_manager_opt -} diff --git a/ballista/scheduler/ui/.dockerignore b/ballista/scheduler/ui/.dockerignore deleted file mode 100644 index dd87e2d73..000000000 --- a/ballista/scheduler/ui/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -build diff --git a/ballista/scheduler/ui/.gitignore b/ballista/scheduler/ui/.gitignore deleted file mode 100644 index 15a75a078..000000000 --- a/ballista/scheduler/ui/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -node_modules/ -/.pnp -.pnp.js - -# testing -/coverage - -# production -build/ - -# misc -.DS_Store -.env.local -.env.development.local -.env.test.local -.env.production.local - -npm-debug.log* -yarn-debug.log* -yarn-error.log* diff --git a/ballista/scheduler/ui/README.md b/ballista/scheduler/ui/README.md deleted file mode 100644 index 4e72825c5..000000000 --- a/ballista/scheduler/ui/README.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# Ballista UI - -## Start project from source - -### Run scheduler/executor - -First, run scheduler from project: - -```shell -$ cd ballista/scheduler -$ RUST_LOG=info cargo run --release -... - Finished release [optimized] target(s) in 11.92s - Running `/path-to-project/target/release/ballista-scheduler` -``` - -and run executor in new terminal: - -```shell -$ cd ballista/executor -$ RUST_LOG=info cargo run --release - Finished release [optimized] target(s) in 0.09s - Running `/path-to-project/target/release/ballista-executor` -``` - -### Run Client project - -```shell -$ cd ballista/scheduler/ui -$ yarn -Resolving packages... -$ yarn start -Starting the development server... -``` - -Now access to http://localhost:3000/ - -**NOTE**: If you get an error when running the `yarn start` command above, make sure -to use the `lts` version of npm. - -You can install it by running: `npm install --lts`. This should fix any errors that -may occur when trying to start the Ballista UI project. diff --git a/ballista/scheduler/ui/index.d.ts b/ballista/scheduler/ui/index.d.ts deleted file mode 100644 index be0f38ba0..000000000 --- a/ballista/scheduler/ui/index.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -declare module "@chakra-ui/icons"; diff --git a/ballista/scheduler/ui/package.json b/ballista/scheduler/ui/package.json deleted file mode 100644 index 1bf27a8a8..000000000 --- a/ballista/scheduler/ui/package.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "name": "scheduler-ui", - "version": "0.1.0", - "private": true, - "dependencies": { - "@chakra-ui/icons": "^2.1.1", - "@chakra-ui/react": "^2.8.2", - "@emotion/react": "^11.11.3", - "@emotion/styled": "^11.11.0", - "@testing-library/jest-dom": "^6.2.1", - "@testing-library/react": "^14.1.2", - "@testing-library/user-event": "^14.5.2", - "@types/jest": "^29.5.11", - "@types/node": "^20.11.5", - "@types/react": "^18.2.48", - "@types/react-dom": "^18.2.18", - "framer-motion": "^11.0.2", - "js-file-download": "^0.4.12", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-icons": "^5.0.1", - "react-inlinesvg": "^4.1.1", - "react-router-dom": "^6.21.3", - "react-scripts": "5.0.1", - "react-table": "^7.8.0", - "react-timeago": "^7.2.0", - "typescript": "^5.3.3", - "web-vitals": "^3.5.1" - }, - "scripts": { - "start": "react-scripts --openssl-legacy-provider start", - "build": "react-scripts --openssl-legacy-provider build", - "test": "react-scripts test", - "eject": "react-scripts eject" - }, - "eslintConfig": { - "extends": [ - "react-app", - "react-app/jest" - ] - }, - "browserslist": { - "production": [ - ">0.2%", - "not dead", - "not op_mini all" - ], - "development": [ - "last 1 chrome version", - "last 1 firefox version", - "last 1 safari version" - ] - }, - "devDependencies": { - "@types/react-table": "^7.7.19", - "@types/react-timeago": "^4.1.7", - "prettier": "^3.2.4", - "http-proxy-middleware": "^2.0.6" - } -} diff --git a/ballista/scheduler/ui/public/favicon.ico b/ballista/scheduler/ui/public/favicon.ico deleted file mode 100644 index a11777cc4..000000000 Binary files a/ballista/scheduler/ui/public/favicon.ico and /dev/null differ diff --git a/ballista/scheduler/ui/public/index.html b/ballista/scheduler/ui/public/index.html deleted file mode 100644 index d902333f0..000000000 --- a/ballista/scheduler/ui/public/index.html +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - Ballista UI - - - -
- - - diff --git a/ballista/scheduler/ui/public/logo192.png b/ballista/scheduler/ui/public/logo192.png deleted file mode 100644 index fc44b0a37..000000000 Binary files a/ballista/scheduler/ui/public/logo192.png and /dev/null differ diff --git a/ballista/scheduler/ui/public/logo512.png b/ballista/scheduler/ui/public/logo512.png deleted file mode 100644 index a4e47a654..000000000 Binary files a/ballista/scheduler/ui/public/logo512.png and /dev/null differ diff --git a/ballista/scheduler/ui/public/manifest.json b/ballista/scheduler/ui/public/manifest.json deleted file mode 100644 index 080d6c77a..000000000 --- a/ballista/scheduler/ui/public/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "short_name": "React App", - "name": "Create React App Sample", - "icons": [ - { - "src": "favicon.ico", - "sizes": "64x64 32x32 24x24 16x16", - "type": "image/x-icon" - }, - { - "src": "logo192.png", - "type": "image/png", - "sizes": "192x192" - }, - { - "src": "logo512.png", - "type": "image/png", - "sizes": "512x512" - } - ], - "start_url": ".", - "display": "standalone", - "theme_color": "#000000", - "background_color": "#ffffff" -} diff --git a/ballista/scheduler/ui/public/robots.txt b/ballista/scheduler/ui/public/robots.txt deleted file mode 100644 index dc045698d..000000000 --- a/ballista/scheduler/ui/public/robots.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# https://www.robotstxt.org/robotstxt.html -User-agent: * -Disallow: diff --git a/ballista/scheduler/ui/react-table-config.d.ts b/ballista/scheduler/ui/react-table-config.d.ts deleted file mode 100644 index 2c9994f91..000000000 --- a/ballista/scheduler/ui/react-table-config.d.ts +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { - UseColumnOrderInstanceProps, - UseColumnOrderState, - UseExpandedHooks, - UseExpandedInstanceProps, - UseExpandedOptions, - UseExpandedRowProps, - UseExpandedState, - UseFiltersColumnOptions, - UseFiltersColumnProps, - UseFiltersInstanceProps, - UseFiltersOptions, - UseFiltersState, - UseGlobalFiltersColumnOptions, - UseGlobalFiltersInstanceProps, - UseGlobalFiltersOptions, - UseGlobalFiltersState, - UseGroupByCellProps, - UseGroupByColumnOptions, - UseGroupByColumnProps, - UseGroupByHooks, - UseGroupByInstanceProps, - UseGroupByOptions, - UseGroupByRowProps, - UseGroupByState, - UsePaginationInstanceProps, - UsePaginationOptions, - UsePaginationState, - UseResizeColumnsColumnOptions, - UseResizeColumnsColumnProps, - UseResizeColumnsOptions, - UseResizeColumnsState, - UseRowSelectHooks, - UseRowSelectInstanceProps, - UseRowSelectOptions, - UseRowSelectRowProps, - UseRowSelectState, - UseRowStateCellProps, - UseRowStateInstanceProps, - UseRowStateOptions, - UseRowStateRowProps, - UseRowStateState, - UseSortByColumnOptions, - UseSortByColumnProps, - UseSortByHooks, - UseSortByInstanceProps, - UseSortByOptions, - UseSortByState, -} from "react-table"; - -declare module "react-table" { - // take this file as-is, or comment out the sections that don't apply to your plugin configuration - - export interface TableOptions< - D extends Record - > extends UseExpandedOptions, - UseFiltersOptions, - UseGlobalFiltersOptions, - UseGroupByOptions, - UsePaginationOptions, - UseResizeColumnsOptions, - UseRowSelectOptions, - UseRowStateOptions, - UseSortByOptions, - // note that having Record here allows you to add anything to the options, this matches the spirit of the - // underlying js library, but might be cleaner if it's replaced by a more specific type that matches your - // feature set, this is a safe default. - Record {} - - export interface Hooks< - D extends Record = Record - > extends UseExpandedHooks, - UseGroupByHooks, - UseRowSelectHooks, - UseSortByHooks {} - - export interface TableInstance< - D extends Record = Record - > extends UseColumnOrderInstanceProps, - UseExpandedInstanceProps, - UseFiltersInstanceProps, - UseGlobalFiltersInstanceProps, - UseGroupByInstanceProps, - UsePaginationInstanceProps, - UseRowSelectInstanceProps, - UseRowStateInstanceProps, - UseSortByInstanceProps {} - - export interface TableState< - D extends Record = Record - > extends UseColumnOrderState, - UseExpandedState, - UseFiltersState, - UseGlobalFiltersState, - UseGroupByState, - UsePaginationState, - UseResizeColumnsState, - UseRowSelectState, - UseRowStateState, - UseSortByState {} - - export interface ColumnInterface< - D extends Record = Record - > extends UseFiltersColumnOptions, - UseGlobalFiltersColumnOptions, - UseGroupByColumnOptions, - UseResizeColumnsColumnOptions, - UseSortByColumnOptions {} - - export interface ColumnInstance< - D extends Record = Record - > extends UseFiltersColumnProps, - UseGroupByColumnProps, - UseResizeColumnsColumnProps, - UseSortByColumnProps {} - - export interface Cell< - D extends Record = Record, - V = any - > extends UseGroupByCellProps, - UseRowStateCellProps {} - - export interface Row< - D extends Record = Record - > extends UseExpandedRowProps, - UseGroupByRowProps, - UseRowSelectRowProps, - UseRowStateRowProps {} -} diff --git a/ballista/scheduler/ui/src/App.css b/ballista/scheduler/ui/src/App.css deleted file mode 100644 index bea95535e..000000000 --- a/ballista/scheduler/ui/src/App.css +++ /dev/null @@ -1,18 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ diff --git a/ballista/scheduler/ui/src/App.test.tsx b/ballista/scheduler/ui/src/App.test.tsx deleted file mode 100644 index 20dca216e..000000000 --- a/ballista/scheduler/ui/src/App.test.tsx +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import React from "react"; -import { render, screen } from "@testing-library/react"; -import App from "./App"; - -test("renders learn react link", () => { - render(); - const linkElement = screen.getByText(/learn react/i); - expect(linkElement).toBeInTheDocument(); -}); diff --git a/ballista/scheduler/ui/src/App.tsx b/ballista/scheduler/ui/src/App.tsx deleted file mode 100644 index c66fb37f7..000000000 --- a/ballista/scheduler/ui/src/App.tsx +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import React, { useState, useEffect } from "react"; -import { - Box, - Grid, - Tab, - TabList, - TabPanel, - TabPanels, - Tabs, - VStack, -} from "@chakra-ui/react"; -import { Header } from "./components/Header"; -import { Summary } from "./components/Summary"; -import { ExecutorsList } from "./components/ExecutorsList"; -import { QueriesList } from "./components/QueriesList"; -import { Footer } from "./components/Footer"; -import "./App.css"; - -const App: React.FunctionComponent = () => { - const [schedulerState, setSchedulerState] = useState(undefined); - const [jobs, setJobs] = useState(undefined); - const [executors, setExecutors] = useState(undefined); - - function getSchedulerState() { - return fetch(`/api/state`, { - method: "POST", - headers: { - Accept: "application/json", - }, - }) - .then((res) => res.json()) - .then((res) => setSchedulerState(res)); - } - - function getJobs() { - return fetch(`/api/jobs`, { - method: "POST", - headers: { - Accept: "application/json", - }, - }) - .then((res) => res.json()) - .then((res) => setJobs(res)); - } - - function getExecutors() { - return fetch(`/api/executors`, { - method: "POST", - headers: { - Accept: "application/json", - }, - }) - .then((res) => res.json()) - .then((res) => setExecutors(res)); - } - - useEffect(() => { - getSchedulerState(); - getJobs(); - getExecutors(); - }, []); - - return ( - - - -
- - - - Jobs - Executors - - - - - - - - - - -