diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 686421fe00..55faffc315 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -41,9 +41,14 @@ jobs: - name: Build wheel file run: | python pip_build.py --nightly - - name: Publish to PyPI + - name: Publish KerasHub Nightly to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: + password: ${{ secrets.PYPI_NIGHTLY_API_TOKEN_HUB }} + verbose: true + - name: Publish KerasNLP Nightly to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: keras_nlp/dist/ password: ${{ secrets.PYPI_NIGHTLY_API_TOKEN }} - packages-dir: dist/ verbose: true diff --git a/.github/workflows/publish-hub-to-pypi.yml b/.github/workflows/publish-hub-to-pypi.yml deleted file mode 100644 index 838ca9b698..0000000000 --- a/.github/workflows/publish-hub-to-pypi.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Publish Hub to PyPI - -on: - push: - branches: - - keras-hub - -permissions: - contents: read - -jobs: - build-and-publish: - name: Build and publish Hub to PyPI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Get pip cache dir - id: pip-cache - run: | - python -m pip install --upgrade pip setuptools - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v4 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install dependencies - run: | - pip install -r requirements.txt --progress-bar off - - name: Build a binary wheel and a source tarball - run: >- - python pip_build.py - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN_HUB }} diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 9d5ee35f78..9cf193731b 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -33,9 +33,15 @@ jobs: - name: Build a binary wheel and a source tarball run: >- python pip_build.py - - name: Publish distribution to PyPI + - name: Publish KerasHub to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ + password: ${{ secrets.PYPI_API_TOKEN_HUB }} + verbose: true + - name: Publish KerasNLP to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: keras_nlp/dist/ password: ${{ secrets.PYPI_API_TOKEN }} + verbose: true diff --git a/.gitignore b/.gitignore index 15d00c09cb..39317d3e29 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ __pycache__/ *.swp *.swo -keras_hub.egg-info/ +*.egg-info/ dist/ .coverage diff --git a/keras_hub/__init__.py b/keras_hub/__init__.py index 068cc28e72..50b3ee2eb8 100644 --- a/keras_hub/__init__.py +++ b/keras_hub/__init__.py @@ -11,12 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""DO NOT EDIT. - -This file was autogenerated. Do not edit it by hand, -since your modifications would be overwritten. -""" - import os # sentencepiece segfaults on some version of tensorflow if tf is imported first. diff --git a/keras_nlp/README.md b/keras_nlp/README.md new file mode 100644 index 0000000000..1ef858f088 --- /dev/null +++ b/keras_nlp/README.md @@ -0,0 +1,7 @@ +# KerasNLP: Multi-framework NLP Models + +KerasNLP has renamed to KerasHub! Read the announcement +[here](https://github.com/keras-team/keras-nlp/issues/1831). + +This directory contains a shim package for `keras-nlp` so that the old style +`pip install keras-nlp` and `import keras_nlp` continue to work. diff --git a/keras_nlp/keras_nlp/__init__.py b/keras_nlp/keras_nlp/__init__.py new file mode 100644 index 0000000000..47bd63622f --- /dev/null +++ b/keras_nlp/keras_nlp/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2024 The KerasHub Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +# Add everything in /api/ to the module search path. +import keras_hub + +# Import everything from /api/ into keras. +from keras_hub.api import * # noqa: F403 +from keras_hub.api import __version__ # Import * ignores names start with "_". + +__path__.extend(keras_hub.__path__) # noqa: F405 +# Don't pollute namespace. +del keras_hub +del os + + +# Never autocomplete `.src` or `.api` on an imported keras object. +def __dir__(): + keys = dict.fromkeys((globals().keys())) + keys.pop("src") + keys.pop("api") + return list(keys) + + +# Don't import `.src` or `.api` during `from keras import *`. +__all__ = [ + name + for name in globals().keys() + if not (name.startswith("_") or name in ("src", "api")) +] diff --git a/keras_nlp/setup.py b/keras_nlp/setup.py new file mode 100644 index 0000000000..6b2ef09ef7 --- /dev/null +++ b/keras_nlp/setup.py @@ -0,0 +1,75 @@ +# Copyright 2024 The KerasHub Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup script.""" + +import os +import pathlib + +from setuptools import find_packages +from setuptools import setup + + +def read(rel_path): + here = os.path.abspath(os.path.dirname(__file__)) + with open(os.path.join(here, rel_path)) as fp: + return fp.read() + + +def get_version(rel_path): + for line in read(rel_path).splitlines(): + if line.startswith("__version__"): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + raise RuntimeError("Unable to find version string.") + + +HERE = pathlib.Path(__file__).parent +README = (HERE / "README.md").read_text() +PARENT = HERE.parent +VERSION = get_version(PARENT / "keras_hub" / "src" / "version_utils.py") + +setup( + name="keras-nlp", + description=( + "Industry-strength Natural Language Processing extensions for Keras." + ), + long_description=README, + long_description_content_type="text/markdown", + version=VERSION, + url="https://github.com/keras-team/keras-nlp", + author="Keras team", + author_email="keras-nlp@google.com", + license="Apache License 2.0", + install_requires=[ + f"keras-hub=={VERSION}", + ], + # Supported Python versions + python_requires=">=3.9", + classifiers=[ + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Operating System :: Unix", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Topic :: Software Development", + ], + packages=find_packages(exclude=("*_test.py",)), +) diff --git a/pip_build.py b/pip_build.py index fdc3e97fdc..826162a5a7 100644 --- a/pip_build.py +++ b/pip_build.py @@ -30,13 +30,13 @@ import argparse import datetime -import glob import os import pathlib import re import shutil -package = "keras_hub" +hub_package = "keras_hub" +nlp_package = "keras_nlp" build_directory = "tmp_build_dir" dist_directory = "dist" to_copy = ["setup.py", "setup.cfg", "README.md"] @@ -46,90 +46,109 @@ def ignore_files(_, filenames): return [f for f in filenames if "_test" in f] -def export_version_string(version, is_nightly=False): +def update_version(build_path, package, version, is_nightly=False): """Export Version and Package Name.""" + package_name = package.replace("_", "-") if is_nightly: date = datetime.datetime.now() version += f".dev{date.strftime('%Y%m%d%H')}" - # Replaces `name="keras-hub"` in `setup.py` with `keras-hub-nightly` - with open("setup.py") as f: - setup_contents = f.read() - with open("setup.py", "w") as f: - setup_contents = setup_contents.replace( - 'name="keras-hub"', 'name="keras-hub-nightly"' - ) - f.write(setup_contents) + package_name = f"{package}-nightly" - # Make sure to export the __version__ string - with open(os.path.join(package, "src", "version_utils.py")) as f: - init_contents = f.read() - with open(os.path.join(package, "src", "version_utils.py"), "w") as f: - init_contents = re.sub( - "\n__version__ = .*\n", - f'\n__version__ = "{version}"\n', - init_contents, + with open(build_path / "setup.py") as f: + setup_contents = f.read() + with open(build_path / "setup.py", "w") as f: + setup_contents = setup_contents.replace( + "name=", f'name="{package_name}", # ' ) - f.write(init_contents) + setup_contents = setup_contents.replace( + "VERSION = ", f'VERSION = "{version}" # ' + ) + f.write(setup_contents) + + # Make sure to export the __version__ string + version_utils = build_path / package / "src" / "version_utils.py" + if os.path.exists(version_utils): + with open(version_utils) as f: + contents = f.read() + with open(version_utils, "w") as f: + contents = re.sub( + "\n__version__ = .*\n", + f'\n__version__ = "{version}"\n', + contents, + ) + f.write(contents) -def copy_source_to_build_directory(root_path): +def copy_source_to_build_directory(root_path, package): # Copy sources (`keras_hub/` directory and setup files) to build # directory - os.chdir(root_path) - os.mkdir(build_directory) shutil.copytree( - package, os.path.join(build_directory, package), ignore=ignore_files + root_path / package, + root_path / build_directory / package, + ignore=ignore_files, ) for fname in to_copy: - shutil.copy(fname, os.path.join(f"{build_directory}", fname)) - os.chdir(build_directory) + shutil.copy(root_path / fname, root_path / build_directory / fname) + + +def build_wheel(build_path, dist_path, __version__): + # Build the package + os.chdir(build_path) + os.system("python3 -m build") + + # Save the dist files generated by the build process + if not os.path.exists(dist_path): + os.mkdir(dist_path) + for fpath in (build_path / dist_directory).glob("*.*"): + shutil.copy(fpath, dist_path) + + # Find the .whl file path + for fname in os.listdir(dist_path): + if __version__ in fname and fname.endswith(".whl"): + whl_path = dist_path / fname + print(f"Build successful. Wheel file available at {whl_path}") + return whl_path + print("Build failed.") + return None -def build(root_path, is_nightly=False): +def build(root_path, is_nightly=False, keras_nlp=True): if os.path.exists(build_directory): raise ValueError(f"Directory already exists: {build_directory}") try: - copy_source_to_build_directory(root_path) - print(os.getcwd()) + whls = [] + build_path = root_path / build_directory + dist_path = root_path / dist_directory + os.mkdir(build_path) from keras_hub.src.version_utils import __version__ # noqa: E402 - export_version_string(__version__, is_nightly) - return build_and_save_output(root_path, __version__) - finally: - # Clean up: remove the build directory (no longer needed) - shutil.rmtree(build_directory) + copy_source_to_build_directory(root_path, hub_package) + update_version(build_path, hub_package, __version__, is_nightly) + whl = build_wheel(build_path, dist_path, __version__) + whls.append(whl) + if keras_nlp: + build_path = root_path / build_directory / nlp_package + dist_path = root_path / nlp_package / dist_directory -def build_and_save_output(root_path, __version__): - # Build the package - os.system("python3 -m build") - - # Save the dist files generated by the build process - os.chdir(root_path) - if not os.path.exists(dist_directory): - os.mkdir(dist_directory) - for fpath in glob.glob( - os.path.join(build_directory, dist_directory, "*.*") - ): - shutil.copy(fpath, dist_directory) + copy_source_to_build_directory(root_path, nlp_package) + update_version(build_path, nlp_package, __version__, is_nightly) + whl = build_wheel(build_path, dist_path, __version__) + whls.append(whl) - # Find the .whl file path - whl_path = None - for fname in os.listdir(dist_directory): - if __version__ in fname and fname.endswith(".whl"): - whl_path = os.path.abspath(os.path.join(dist_directory, fname)) - if whl_path: - print(f"Build successful. Wheel file available at {whl_path}") - else: - print("Build failed.") - return whl_path + return whls + finally: + # Clean up: remove the build directory (no longer needed) + os.chdir(root_path) + shutil.rmtree(root_path / build_directory) -def install_whl(whl_fpath): - print(f"Installing wheel file: {whl_fpath}") - os.system(f"pip3 install {whl_fpath} --force-reinstall --no-dependencies") +def install_whl(whls): + for path in whls: + print(f"Installing wheel file: {path}") + os.system(f"pip3 install {path} --force-reinstall --no-dependencies") if __name__ == "__main__": @@ -138,14 +157,22 @@ def install_whl(whl_fpath): "--install", action="store_true", help="Whether to install the generated wheel file.", + default=False, ) parser.add_argument( "--nightly", action="store_true", help="Whether to generate nightly wheel file.", + default=False, + ) + parser.add_argument( + "--keras_nlp", + action="store_true", + help="Whether to build the keras-nlp shim package.", + default=True, ) args = parser.parse_args() root_path = pathlib.Path(__file__).parent.resolve() - whl_path = build(root_path, args.nightly) - if whl_path and args.install: - install_whl(whl_path) + whls = build(root_path, args.nightly, args.keras_nlp) + if whls and args.install: + install_whl(whls) diff --git a/setup.py b/setup.py index 70e5ebfc4d..52f2e06e72 100644 --- a/setup.py +++ b/setup.py @@ -37,10 +37,7 @@ def get_version(rel_path): HERE = pathlib.Path(__file__).parent README = (HERE / "README.md").read_text() -if os.path.exists("keras_hub/version_utils.py"): - VERSION = get_version("keras_hub/version_utils.py") -else: - VERSION = get_version("keras_hub/src/version_utils.py") +VERSION = get_version("keras_hub/src/version_utils.py") setup( name="keras-hub",