diff --git a/.github/codecov.yaml b/.github/codecov.yaml new file mode 100644 index 0000000..5f55ada --- /dev/null +++ b/.github/codecov.yaml @@ -0,0 +1,10 @@ +comment: no +coverage: + range: 80..90 + status: + project: + default: + informational: true + patch: + default: + informational: true diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 0000000..ae5740c --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,3 @@ +# Overview + +Please replace this line with full information about your idea or problem. If it's a bug share as much as possible to reproduce it diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..b181a43 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,5 @@ +- fixes # + +--- + +Please make sure that all the checks pass. Please add here any additional information regarding this pull request. It's highly recommended that you link this PR to an issue (please create one if it doesn't exist for this PR) diff --git a/.github/stale.yaml b/.github/stale.yaml new file mode 100644 index 0000000..cc72780 --- /dev/null +++ b/.github/stale.yaml @@ -0,0 +1,23 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 90 + +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 30 + +# Issues with these labels will never be considered stale +exemptLabels: + - feature + - enhancement + - bug + +# Label to use when marking an issue as stale +staleLabel: wontfix + +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false diff --git a/.github/workflows/general.yaml b/.github/workflows/general.yaml new file mode 100644 index 0000000..852611e --- /dev/null +++ b/.github/workflows/general.yaml @@ -0,0 +1,131 @@ +name: general + +on: + push: + branches: + - main + tags: + - v*.*.* + pull_request: + branches: + - main + schedule: + - cron: "0 3 * * *" + +jobs: + + # Test (Linux) + + test-linux: + if: github.event_name != 'schedule' || github.repository_owner == 'frictionlessdata' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, '3.10'] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: make install + - name: Prepare environment + run: cp .env.example .env + - name: Test software + run: make test-ci + - name: Report coverage + uses: codecov/codecov-action@v2 + + # Test (MacOS) + + test-macos: + if: github.event_name != 'schedule' || github.repository_owner == 'frictionlessdata' + runs-on: macos-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + # https://stackoverflow.com/questions/9678408/cant-install-psycopg2-with-pip-in-virtualenv-on-mac-os-x-10-7 + run: LDFLAGS=`echo $(pg_config --ldflags)` make install + - name: Prepare environment + run: cp .env.example .env + - name: Test software + run: make test + + # Test (Windows) + + test-windows: + if: github.event_name != 'schedule' || github.repository_owner == 'frictionlessdata' + runs-on: windows-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: make install + - name: Prepare environment + run: cp .env.example .env + - name: Test software + run: make test + + # Deploy + + deploy: + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install and build site + run: | + echo '!**/*.html' >> .gitignore + make install + livemark build + - name: Publush to Github Pages + uses: stefanzweifel/git-auto-commit-action@v4 + with: + branch: site + create_branch: true + push_options: '--force' + + # Release + + release: + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + needs: [test-linux, test-macos, test-windows] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + - name: Build distribution + run: | + python setup.py sdist bdist_wheel + - name: Publish to PYPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_KEY }} + - name: Release to GitHub + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/project.yaml b/.github/workflows/project.yaml new file mode 100644 index 0000000..24c38d5 --- /dev/null +++ b/.github/workflows/project.yaml @@ -0,0 +1,24 @@ +name: project + +on: + issues: + types: + - opened + - reopened + pull_request: + types: + - opened + - reopened + +jobs: + project-assign: + runs-on: ubuntu-latest + steps: + - name: Assign to project + uses: leonsteinhaeuser/project-beta-automations@v2.0.1 + with: + gh_token: ${{ secrets.PROJECT_TOKEN }} + resource_node_id: ${{ github.event_name == 'issues' && github.event.issue.node_id || github.event.pull_request.node_id }} + organization: frictionlessdata + project_id: 16 + status_value: Inbox diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0c2bf02 --- /dev/null +++ b/.gitignore @@ -0,0 +1,96 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +.python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +pip-wheel-metadata/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask instance folder +instance/ + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# dotenv +.env + +# Spyder project settings +.spyderproject + +# Livemark +blog/**/*.html +docs/**/*.html +index.html +404.html + +# Extras +.frictionless/ +.google.json +coverage/ +.vscode/ +.server/ +site/ +tmp/ diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/CNAME b/CNAME new file mode 100644 index 0000000..64b40fc --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +standards.frictionlessdata.io diff --git a/LEAD.md b/LEAD.md new file mode 100644 index 0000000..e8b747f --- /dev/null +++ b/LEAD.md @@ -0,0 +1 @@ +roll diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..1b65832 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# The MIT License (MIT) + +Copyright © `2023` `Open Knowledge Foundation` + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1f3fa08 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +recursive-include standards * +include LICENSE.md +include Makefile +include pylama.ini +include pytest.ini +include README.md diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c5adefe --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +.PHONY: all coverage docker-setup docs install format github lint release test test-ci + + +PACKAGE := $(shell grep '^PACKAGE =' setup.py | cut -d '"' -f2) +VERSION := $(shell head -n 1 $(PACKAGE)/assets/VERSION) +LEAD := $(shell head -n 1 LEAD.md) + + +all: + @grep '^\.PHONY' Makefile | cut -d' ' -f2- | tr ' ' '\n' + +coverage: + sensible-browser coverage/index.html + +docs: + livemark build + +format: + black $(PACKAGE) tests + +install: + pip install --upgrade -e .[dev] + +lint: + black $(PACKAGE) tests --check + pylama $(PACKAGE) tests + pyright $(PACKAGE) tests + +release: + git checkout main && git pull origin && git fetch -p + @git log --pretty=format:"%C(yellow)%h%Creset %s%Cgreen%d" --reverse -20 + @echo "\nReleasing v$(VERSION) in 10 seconds. Press to abort\n" && sleep 10 + make test && git commit -a -m 'v$(VERSION)' && git tag -a v$(VERSION) -m 'v$(VERSION)' + git push --follow-tags + +test: + make lint + pytest --cov ${PACKAGE} --cov-report term-missing --cov-report html:coverage --cov-fail-under 70 --timeout=300 + +test-ci: + make lint + pytest --cov ${PACKAGE} --cov-report term-missing --cov-report xml --cov-fail-under 80 --timeout=300 --ci diff --git a/README.md b/README.md index 5880ca6..cc2824f 100644 --- a/README.md +++ b/README.md @@ -1 +1,6 @@ -# standards \ No newline at end of file +# Standards + +[![Build](https://img.shields.io/github/actions/workflow/status/frictionlessdata/standards/general.yaml?branch=main)](https://github.com/frictionlessdata/standards/actions) +[![Coverage](https://img.shields.io/codecov/c/github/frictionlessdata/standards/main)](https://codecov.io/gh/frictionlessdata/standards) +[![Codebase](https://img.shields.io/badge/codebase-github-brightgreen)](https://github.com/frictionlessdata/standards) +[![Support](https://img.shields.io/badge/support-slack-brightgreen)](https://join.slack.com/t/frictionlessdata/shared_invite/zt-17kpbffnm-tRfDW_wJgOw8tJVLvZTrBg) diff --git a/livemark.yaml b/livemark.yaml new file mode 100644 index 0000000..91fb372 --- /dev/null +++ b/livemark.yaml @@ -0,0 +1,29 @@ +brand: + text: Frictionless Standards +about: + text: Lightweight yet comprehensive data standards as Data Package and Table Schema +site: + favicon: assets/logo.png + styles: + - style.css +github: + user: frictionlessdata + repo: standards +counter: + type: plausible + code: standards.frictionlessdata.io +topics: + selector: h2 +links: + items: + - name: Open Knowledge + path: https://okfn.org + - name: Frictionless + path: https://frictionlessdata.io + - name: Support + path: https://join.slack.com/t/frictionlessdata/shared_invite/zt-17kpbffnm-tRfDW_wJgOw8tJVLvZTrBg +pages: + items: + - path: index + name: Introduction + from: README.md diff --git a/pylama.ini b/pylama.ini new file mode 100644 index 0000000..0161fd8 --- /dev/null +++ b/pylama.ini @@ -0,0 +1,9 @@ +[pylama] +linters = pyflakes,mccabe,pep8 +ignore = E128,E203,E301,E501,E731,C901 + +[pylama:mccabe] +complexity = 48 + +[pylama:*/__init__.py] +ignore = W0611,W0401 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..68b9bb6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 90 diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..ef66cc3 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,12 @@ +{ + "include": [ + "standards", + "tests" + ], + "strict": [ + "standards/**/*.*" + ], + "ignore": [ + "**/__init__.py" + ] +} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f60be90 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +testpaths = tests +markers = + ci: integrational tests (select with '--ci') +env_files = + .env +filterwarnings = + ignore::DeprecationWarning:boto.* + ignore::DeprecationWarning:moto.* + ignore::DeprecationWarning:grako.* + ignore::DeprecationWarning:savWriter.* diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f5864ae --- /dev/null +++ b/setup.py @@ -0,0 +1,94 @@ +import os +import io +from setuptools import setup, find_packages + + +# Helpers + + +def read(*paths): + """Read a text file.""" + basedir = os.path.dirname(__file__) + fullpath = os.path.join(basedir, *paths) + contents = io.open(fullpath, encoding="utf-8").read().strip() + return contents + + +# Prepare + + +PACKAGE = "standards" +NAME = 'frictionless-standards' +TESTS_REQUIRE = [ + "black", + "pylama", + "pytest", + "pyright", + "livemark", + "pytest-cov", + "pytest-vcr", + "pytest-mock", + "pytest-only", + "pytest-dotenv", + "pytest-timeout", + "pytest-lazy-fixture", +] +EXTRAS_REQUIRE = { + "dev": TESTS_REQUIRE, +} +INSTALL_REQUIRES = [ + "pyyaml>=5.3", + "attrs>=22.2.0", + "pydantic>=2.0a1", + "jsonschema>=2.5", +] +README = read("README.md") +VERSION = read(PACKAGE, "assets", "VERSION") +PACKAGES = find_packages(exclude=["tests"]) +ENTRY_POINTS = {"console_scripts": ["frictionless = frictionless.__main__:program"]} + + +# Run + + +setup( + name=NAME, + version=VERSION, + packages=PACKAGES, + include_package_data=True, + install_requires=INSTALL_REQUIRES, + tests_require=TESTS_REQUIRE, + extras_require=EXTRAS_REQUIRE, + entry_points=ENTRY_POINTS, + zip_safe=False, + long_description=README, + long_description_content_type="text/markdown", + description="Lightweight yet comprehensive data standards as Data Package and Table Schema", + author="Open Knowledge Foundation", + author_email="info@okfn.org", + url="https://github.com/frictionlessdata/standards", + license="MIT", + keywords=[ + "data validation", + "frictionless data", + "open data", + "json schema", + "json table schema", + "data package", + "tabular data package", + ], + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Software Development :: Libraries :: Python Modules", + ], +) diff --git a/standards/__init__.py b/standards/__init__.py new file mode 100644 index 0000000..b49ad62 --- /dev/null +++ b/standards/__init__.py @@ -0,0 +1,5 @@ +from .dialect import IDialect +from .package import IPackage +from .resource import IResource +from .schema import ISchema +from .standard import Standard diff --git a/standards/assets/VERSION b/standards/assets/VERSION new file mode 100644 index 0000000..009ff11 --- /dev/null +++ b/standards/assets/VERSION @@ -0,0 +1 @@ +2.0.0a1 diff --git a/standards/dialect.py b/standards/dialect.py new file mode 100644 index 0000000..13d3cc1 --- /dev/null +++ b/standards/dialect.py @@ -0,0 +1,51 @@ +from __future__ import annotations +from typing import List, Union +from typing_extensions import TypedDict + + +class IDialect(TypedDict, total=False): + name: str + type: str + title: str + description: str + header: bool + headerRows: List[int] + headerJoin: str + headerCase: bool + commentChar: str + commentRows: List[int] + skipBlankRows: bool + csv: ICsvControl + json: IJsonControl + excel: IExcelControl + + +class IControl(TypedDict, total=False): + name: str + # type: implicit + title: str + description: str + + +class ICsvControl(IControl, total=False): + delimiter: str + lineTerminator: str + quoteChar: str + doubleQuote: bool + escapeChar: str + nullSequence: str + skipInitialSpace: bool + + +class IJsonControl(IControl, total=False): + keys: List[str] + keyed: bool + property: str + + +class IExcelControl(IControl, total=False): + sheet: Union[str, int] + fillMergedCells: bool + preserveFormatting: bool + adjustFloatingPointError: bool + stringified: bool diff --git a/standards/formats/__init__.py b/standards/formats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/formats/ckan/__init__.py b/standards/formats/ckan/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/formats/ckan/mapper.py b/standards/formats/ckan/mapper.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/formats/ckan/package.py b/standards/formats/ckan/package.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/formats/ckan/resource.py b/standards/formats/ckan/resource.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/package.py b/standards/package.py new file mode 100644 index 0000000..51d55e3 --- /dev/null +++ b/standards/package.py @@ -0,0 +1,12 @@ +from __future__ import annotations +from typing import List +from typing_extensions import TypedDict, Required +from .resource import IResource + + +class IPackage(TypedDict, total=False): + name: str + type: str + title: str + description: str + resources: Required[List[IResource]] diff --git a/standards/resource.py b/standards/resource.py new file mode 100644 index 0000000..e682885 --- /dev/null +++ b/standards/resource.py @@ -0,0 +1,39 @@ +from __future__ import annotations +from typing import List, Any, Dict +from typing_extensions import Required, TypedDict, Literal +from .dialect import IDialect +from .schema import ISchema + + +class IResource(TypedDict, total=False): + name: Required[str] + # type: required + title: str + description: str + path: str + data: Any + scheme: str + format: str + compression: str + extrapaths: List[str] + innerpath: str + encoding: str + dialect: IDialect + + +class IFileResource(IResource, total=False): + type: Required[Literal["file"]] + + +class ITextResource(IResource, total=False): + type: Required[Literal["text"]] + + +class IJsonResource(IResource, total=False): + type: Required[Literal["json"]] + schema: Dict[str, Any] # Json Schema + + +class ITableResource(IResource, total=False): + type: Required[Literal["table"]] + schema: ISchema # Table Schema diff --git a/standards/schema.py b/standards/schema.py new file mode 100644 index 0000000..112c7d6 --- /dev/null +++ b/standards/schema.py @@ -0,0 +1,102 @@ +from __future__ import annotations +from typing import Dict, List, Literal, Any +from typing_extensions import Required, TypedDict + + +class ISchema(TypedDict, total=False): + name: str + type: str + title: str + description: str + fields: Required[List[IField]] + missingValues: List[str] + primary_key: List[str] + foreign_keys: List[IForeignKey] + + +class IField(TypedDict, total=False): + name: Required[str] + # type: required + title: str + description: str + format: str + missingValues: List[str] + + +class IAnyField(IField, total=False): + type: Required[Literal["any"]] + + +class IArrayField(IField, total=False): + type: Required[Literal["array"]] + # support json/csv format + arrayItem: Dict[str, Any] + + +class IBooleanField(IField, total=False): + type: Required[Literal["boolean"]] + trueValues: List[str] + falseValues: List[str] + + +class IDateField(IField, total=False): + type: Required[Literal["date"]] + + +class IDatetimeField(IField, total=False): + type: Required[Literal["datetime"]] + + +class IDurationField(IField, total=False): + type: Required[Literal["duration"]] + + +class IGeojsonField(IField, total=False): + type: Required[Literal["geojson"]] + + +class IGeopointField(IField, total=False): + type: Required[Literal["geopoint"]] + + +class IIntegerField(IField, total=False): + type: Required[Literal["integer"]] + bareNumber: bool + groupChar: str + + +class INumberField(IField, total=False): + type: Required[Literal["number"]] + bareNumber: bool + groupChar: str + decimalChar: str + + +class IObjectField(IField, total=False): + type: Required[Literal["object"]] + + +class IStringField(IField, total=False): + type: Required[Literal["string"]] + + +class ITimeField(IField, total=False): + type: Required[Literal["time"]] + + +class IYearField(IField, total=False): + type: Required[Literal["year"]] + + +class IYearmonthField(IField, total=False): + type: Required[Literal["yearmonth"]] + + +class IForeignKey(TypedDict, total=False): + fields: Required[List[str]] + reference: IForeignKeyReference + + +class IForeignKeyReference(TypedDict, total=False): + fields: Required[List[str]] + resource: Required[str] diff --git a/standards/standard.py b/standards/standard.py new file mode 100644 index 0000000..3740f15 --- /dev/null +++ b/standards/standard.py @@ -0,0 +1,14 @@ +from __future__ import annotations +import attrs +import pydantic +from typing import Type, Dict, Any +from typing_extensions import TypedDict + + +@attrs.define(kw_only=True) +class Standard: + definition: Type[TypedDict] + + def to_jsonschema(self) -> Dict[str, Any]: + atype = pydantic.AnalyzedType(self.definition) + return atype.json_schema() diff --git a/style.css b/style.css new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_standard.py b/tests/test_standard.py new file mode 100644 index 0000000..9cffd1c --- /dev/null +++ b/tests/test_standard.py @@ -0,0 +1,6 @@ +from standards import Standard, ISchema + + +def test_standard(): + schema = Standard(definition=ISchema) + assert schema.to_jsonschema()