diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index e943922f838..882e2a4b62f 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -26,7 +26,10 @@ jobs: - api/** - bin/** - catalogs/** - - clients/** + - clients/client-java/** + - clients/client-java-runtime/** + - clients/filesystem-hadoop3/** + - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** @@ -88,7 +91,7 @@ jobs: - name: Backend Integration Test id: integrationTest run: | - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} + ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} -PskipPythonITs - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71facb817f6..93f9ec9af91 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,10 @@ jobs: - api/** - bin/** - catalogs/** - - clients/** + - clients/client-java/** + - clients/client-java-runtime/** + - clients/filesystem-hadoop3/** + - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index f05174d33b9..b323f0e3d82 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -26,7 +26,10 @@ jobs: - api/** - bin/** - catalogs/** - - clients/** + - clients/client-java/** + - clients/client-java-runtime/** + - clients/filesystem-hadoop3/** + - clients/filesystem-hadoop3-runtime/** - common/** - conf/** - core/** diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml new file mode 100644 index 00000000000..3dc686a916c --- /dev/null +++ b/.github/workflows/python-integration-test.yml @@ -0,0 +1,82 @@ +name: Python Client Integration Test + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the "main" branch + push: + branches: [ "main", "branch-*" ] + pull_request: + branches: [ "main", "branch-*" ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + source_changes: + - .github/** + - api/** + - clients/client-python/** + - common/** + - conf/** + - core/** + - meta/** + - server/** + - server-common/** + outputs: + source_changes: ${{ steps.filter.outputs.source_changes }} + + # Integration test for AMD64 architecture + test-amd64-arch: + needs: changes + if: needs.changes.outputs.source_changes == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + architecture: [linux/amd64] + java-version: [ 8 ] + env: + PLATFORM: ${{ matrix.architecture }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Python Client Integration Test + id: integrationTest + run: | + for pythonVersion in "3.8" "3.9" "3.10" "3.11" + do + ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :client:client-python:test + done + + - name: Upload integrate tests reports + uses: actions/upload-artifact@v3 + if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} + with: + name: integrate test reports + path: | + build/reports + integration-test/build/integration-test.log + distribution/package/logs/gravitino-server.out + distribution/package/logs/gravitino-server.log + catalogs/**/*.log \ No newline at end of file diff --git a/.gitignore b/.gitignore index e9956fee835..15b5f41582d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ docs/build dev/docker/tools/docker-connector include web/.gitignore +include clients/client-python/.gitignore # node env .node diff --git a/build.gradle.kts b/build.gradle.kts index deb6d8c9a53..b3abe5f3fb2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -93,6 +93,9 @@ project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) { ) } +val pythonVersion: String = project.properties["pythonVersion"] as? String ?: project.extra["pythonVersion"].toString() +project.extra["pythonVersion"] = pythonVersion + licenseReport { renderers = arrayOf(InventoryHtmlReportRenderer("report.html", "Backend")) filters = arrayOf(LicenseBundleNormalizer()) @@ -100,6 +103,11 @@ licenseReport { repositories { mavenCentral() } allprojects { + // Gravitino Python client project didn't need to apply the Spotless plugin + if (project.name == "client-python") { + return@allprojects + } + apply(plugin = "com.diffplug.spotless") repositories { mavenCentral() @@ -215,6 +223,11 @@ dependencies { } subprojects { + // Gravitino Python client project didn't need to apply the java plugin + if (project.name == "client-python") { + return@subprojects + } + apply(plugin = "jacoco") apply(plugin = "maven-publish") apply(plugin = "java") diff --git a/clients/client-python/.gitignore b/clients/client-python/.gitignore index 6769e21d99a..4399cbb1a8d 100644 --- a/clients/client-python/.gitignore +++ b/clients/client-python/.gitignore @@ -1,160 +1,9 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. + +**/.pytest_cache/** +**/__pycache__/** +gravitino.egg-info +vevn +.vevn +.idea diff --git a/clients/client-python/Makefile b/clients/client-python/Makefile deleted file mode 100644 index f769e5ac76a..00000000000 --- a/clients/client-python/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -.PHONY: mkvenv install test clean - -mkvenv: - python -m venv venv && source ./venv/bin/activate && pip install -U pip - -install: - source ./venv/bin/activate && pip install -e . - -test: - source ./venv/bin/activate && python -m unittest - -clean: - rm -rf venv diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts new file mode 100644 index 00000000000..9ee05355c80 --- /dev/null +++ b/clients/client-python/build.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +import io.github.piyushroshan.python.VenvTask + +plugins { + id("io.github.piyushroshan.python-gradle-miniforge-plugin") version "1.0.0" +} + +pythonPlugin { + pythonVersion.set(project.rootProject.extra["pythonVersion"].toString()) +} + +fun deleteCacheDir(targetDir: String) { + project.fileTree(project.projectDir).matching { + include("**/$targetDir/**") + }.forEach { file -> + val targetDirPath = file.path.substring(0, file.path.lastIndexOf(targetDir) + targetDir.length) + project.file(targetDirPath).deleteRecursively() + } +} + +tasks { + val pipInstall by registering(VenvTask::class) { + venvExec = "pip" + args = listOf("install", "-e", ".") + } + + val test by registering(VenvTask::class) { + dependsOn(pipInstall) + venvExec = "python" + args = listOf("-m", "unittest") + workingDir = projectDir.resolve(".") + } + + val build by registering(VenvTask::class) { + } + + val clean by registering(Delete::class) { + delete("build") + delete("gravitino.egg-info") + + doLast { + deleteCacheDir(".pytest_cache") + deleteCacheDir("__pycache__") + } + } +} diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index e130f2ed0fd..04dd35ffc91 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -8,8 +8,8 @@ setup( name="gravitino", - description="project description TBD", - version="0.0.1", + description="Python lib/client for Gravitino", + version="0.5.0", long_description=open("README.md").read(), long_description_content_type="text/markdown", url="https://github.com/datastrato/gravitino", diff --git a/docs/how-to-build.md b/docs/how-to-build.md index 50d42eac28b..76f10e65f6b 100644 --- a/docs/how-to-build.md +++ b/docs/how-to-build.md @@ -59,6 +59,28 @@ This software is licensed under the Apache License version 2." ./gradlew build -PjdkVersion=17 ``` +3. Build the Gravitino Python client. + + The default specified Python version is 3.8, if you want to use Python 3.9, 3.10 or 3.11 to build, + modify the property `pythonVersion` to 3.9, 3.10 or 3.11 in the `gradle.properties` file, or specify the version + with `-P`, like: + + ```shell + ./gradlew build -PpythonVersion=3.9 + ``` + + Or: + + ```shell + ./gradlew build -PpythonVersion=3.10 + ``` + + Or: + + ```shell + ./gradlew build -PpythonVersion=3.11 + ``` + :::note The first time you build the project, downloading the dependencies may take a while. You can add `-x test` to skip the tests, by using `./gradlew build -x test`. diff --git a/docs/how-to-test.md b/docs/how-to-test.md index affbcff8161..8a50360575d 100644 --- a/docs/how-to-test.md +++ b/docs/how-to-test.md @@ -72,6 +72,7 @@ To deploy the Gravitino server locally to run the integration tests, follow thes * Skip unit tests by using the `./gradlew build -PskipTests` command. * Skip integration tests by using the `./gradlew build -PskipITs` command. * Skip web frontend integration tests by using the `./gradlew build -PskipWebITs` command. +* Skip Python client integration tests by using the `./gradlew build -PskipPythonITs` command. * Skip both unit tests and integration tests by using the `./gradlew build -x test` or `./gradlew build -PskipTests -PskipITs` commands. ## Configuring parameters for integration tests diff --git a/gradle.properties b/gradle.properties index 3a2f7024bee..58ea449bb93 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,16 +8,19 @@ org.gradle.parallel=true org.gradle.caching=true org.gradle.jvmargs=-Xmx4g -#version that is going to be updated automatically by releases +# version that is going to be updated automatically by releases version = 0.5.0-SNAPSHOT -#sonatype credentials +# sonatype credentials SONATYPE_USER = admin SONATYPE_PASSWORD = password -#jdkVersion is used to specify the version of JDK to build and test Gravitino, current +# jdkVersion is used to specify the version of JDK to build and test Gravitino, current # supported version is 8, 11, and 17. jdkVersion = 8 # defaultScalaVersion is used to specify the version of Scala to build and test Gravitino defaultScalaVersion = 2.12 + +# pythonVersion is used to specify the version of Python to build and test Gravitino python client. +pythonVersion = 3.8 diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index d7253f5112a..291278cb830 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -135,10 +135,10 @@ dependencies { tasks.test { val skipITs = project.hasProperty("skipITs") - val skipWebITs = project.hasProperty("skipWebITs") if (skipITs) { exclude("**/integration/test/**") } else { + val skipWebITs = project.hasProperty("skipWebITs") if (skipWebITs) { exclude("**/integration/test/web/ui/**") } diff --git a/settings.gradle.kts b/settings.gradle.kts index a98a37c21e8..bf872bc0ef6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,7 +20,13 @@ include( ) include("catalogs:catalog-hadoop") include("catalogs:catalog-messaging-kafka") -include("clients:client-java", "clients:client-java-runtime", "clients:filesystem-hadoop3", "clients:filesystem-hadoop3-runtime") +include( + "clients:client-java", + "clients:client-java-runtime", + "clients:filesystem-hadoop3", + "clients:filesystem-hadoop3-runtime", + "clients:client-python" +) include("trino-connector") include("spark-connector") include("web")