diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 9a951da496..d490fe6f2f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,9 +6,8 @@ What kind of issue is this? use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support) - [ ] **PlatformIO IDE**. - All issues related to PlatformIO IDE should be reported to appropriate repository: - [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or - [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) + All issues related to PlatformIO IDE should be reported to the + [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository - [ ] **Development Platform or Board**. All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index ff353c31b8..3f0a4529f4 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -35,7 +35,7 @@ jobs: tox -e testcore - name: Build Python source tarball - run: python setup.py sdist + run: python setup.py sdist bdist_wheel - name: Publish package to PyPI if: ${{ github.ref == 'refs/heads/master' }} diff --git a/.github/workflows/projects.yml b/.github/workflows/projects.yml index 080b8162a1..1d37c5a38a 100644 --- a/.github/workflows/projects.yml +++ b/.github/workflows/projects.yml @@ -13,11 +13,11 @@ jobs: folder: "Marlin" config_dir: "Marlin" env_name: "mega2560" - # - esphome: - # repository: "esphome/esphome" - # folder: "esphome" - # config_dir: "esphome" - # env_name: "esp32-arduino" + - esphome: + repository: "esphome/esphome" + folder: "esphome" + config_dir: "esphome" + env_name: "esp32-arduino" - smartknob: repository: "scottbez1/smartknob" folder: "smartknob" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f2f4188ab..8deafb9c70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,12 +6,13 @@ To get started, si 1. Fork the repository on GitHub 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` 3. Run `pip install tox` -4. Go to the root of project where is located `tox.ini` and run `tox -e py37` +4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`. + You can replace `py39` with your own Python version. For example, `py311` means Python 3.11. 5. Activate current development environment: - * Windows: `.tox\py37\Scripts\activate` - * Bash/ZSH: `source .tox/py37/bin/activate` - * Fish: `source .tox/py37/bin/activate.fish` + * Windows: `.tox\py39\Scripts\activate` + * Bash/ZSH: `source .tox/py39/bin/activate` + * Fish: `source .tox/py39/bin/activate.fish` 6. Make changes to code, documentation, etc. 7. Lint source code `make before-commit` diff --git a/HISTORY.rst b/HISTORY.rst index 3aa710ff90..2aa6774484 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,19 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.8 (2023-07-05) +~~~~~~~~~~~~~~~~~~ + +* Added a new ``--lint`` option to the `pio project config `__ command, enabling users to efficiently perform linting on the |PIOCONF| +* Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information +* Expanded the functionality of the |LIBRARYJSON| manifest by allowing the use of the underscore symbol in the `keywords `__ field +* Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) +* Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) +* Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) +* Updated the `pio project metadata `__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata +* Resolved a critical issue related to the usage of the ``-include`` flag within the `build_flags `__ option, specifically when employing dynamic variables (`issue #4682 `_) +* Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated `__ + 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 98609771ba..3f462c9ae6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 98609771ba8f78505adad20e66c6505c262f2650 +Subproject commit 3f462c9ae63623710d48c85e16241bbb0a2b6553 diff --git a/examples b/examples index 3e23b5ac43..4b572ec9fe 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 3e23b5ac43ab7ec277ce7d68618458f3980f8089 +Subproject commit 4b572ec9fef6df5aacb1e113177f5ec28dda1cde diff --git a/platformio/__init__.py b/platformio/__init__.py index 7d3c8f8abf..75b51791c2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, 7) +VERSION = (6, 1, 8) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/__main__.py b/platformio/__main__.py index c0ad38aca8..58cabe8b66 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,7 +14,7 @@ import os import sys -from traceback import format_exc +import traceback import click @@ -53,13 +53,13 @@ def cli(ctx, force, caller, no_ansi): # pylint: disable=unused-argument except: # pylint: disable=bare-except pass - maintenance.on_platformio_start(ctx, caller) + maintenance.on_cmd_start(ctx, caller) @cli.result_callback() @click.pass_context -def process_result(ctx, result, *_, **__): - maintenance.on_platformio_end(ctx, result) +def process_result(*_, **__): + maintenance.on_cmd_end() def configure(): @@ -96,6 +96,7 @@ def main(argv=None): if argv: assert isinstance(argv, list) sys.argv = argv + try: ensure_python3(raise_exception=True) configure() @@ -106,18 +107,18 @@ def main(argv=None): except Exception as exc: # pylint: disable=broad-except if not isinstance(exc, exception.ReturnErrorCode): maintenance.on_platformio_exception(exc) - error_str = "Error: " + error_str = f"{exc.__class__.__name__}: " if isinstance(exc, exception.PlatformioException): error_str += str(exc) else: - error_str += format_exc() + error_str += traceback.format_exc() error_str += """ ============================================================ An unexpected error occurred. Further steps: * Verify that you have the latest version of PlatformIO using - `pip install -U platformio` command + `python -m pip install -U platformio` command * Try to find answer in FAQ Troubleshooting section https://docs.platformio.org/page/faq/index.html @@ -129,6 +130,8 @@ def main(argv=None): """ click.secho(error_str, fg="red", err=True) exit_code = int(str(exc)) if str(exc).isdigit() else 1 + + maintenance.on_platformio_exit() sys.argv = prev_sys_argv return exit_code diff --git a/platformio/account/client.py b/platformio/account/client.py index a898de481d..064321448c 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -16,7 +16,7 @@ import time from platformio import __accounts_api__, app -from platformio.exception import PlatformioException +from platformio.exception import PlatformioException, UserSideException from platformio.http import HTTPClient, HTTPClientError @@ -24,11 +24,11 @@ class AccountError(PlatformioException): MESSAGE = "{0}" -class AccountNotAuthorized(AccountError): +class AccountNotAuthorized(AccountError, UserSideException): MESSAGE = "You are not authorized! Please log in to PlatformIO Account." -class AccountAlreadyAuthorized(AccountError): +class AccountAlreadyAuthorized(AccountError, UserSideException): MESSAGE = "You are already authorized with {0} account." diff --git a/platformio/app.py b/platformio/app.py index 48e90b97f9..d94cdfefc7 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -18,6 +18,7 @@ import os import platform import socket +import time import uuid from platformio import __version__, exception, fs, proc @@ -68,18 +69,23 @@ def projects_dir_validate(projects_dir): "command_ctx": None, "caller_id": None, "custom_project_conf": None, + "pause_telemetry": False, } +def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True): + state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir) + if ensure_dir_exists and not os.path.isdir(state_dir): + os.makedirs(state_dir) + return os.path.join(state_dir, file_name) + + class State: def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: - core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") - if not os.path.isdir(core_dir): - os.makedirs(core_dir) - self.path = os.path.join(core_dir, "appstate.json") + self.path = resolve_state_path("core_dir", "appstate.json") self._storage = {} self._lockfile = None self.modified = False @@ -248,6 +254,7 @@ def get_cid(): cid = str(cid) if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) + set_state_item("created_at", int(time.time())) return cid diff --git a/platformio/assets/templates/ide-projects/atom/.clang_complete.tpl b/platformio/assets/templates/ide-projects/atom/.clang_complete.tpl deleted file mode 100644 index 6d8e70ed23..0000000000 --- a/platformio/assets/templates/ide-projects/atom/.clang_complete.tpl +++ /dev/null @@ -1,6 +0,0 @@ -% for include in filter_includes(includes): --I{{include}} -% end -% for define in defines: --D{{!define}} -% end diff --git a/platformio/assets/templates/ide-projects/atom/.gcc-flags.json.tpl b/platformio/assets/templates/ide-projects/atom/.gcc-flags.json.tpl deleted file mode 100644 index 85b4e9dabc..0000000000 --- a/platformio/assets/templates/ide-projects/atom/.gcc-flags.json.tpl +++ /dev/null @@ -1,9 +0,0 @@ -% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines]) -{ - "execPath": "{{ cxx_path }}", - "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccErrorLimit": 15, - "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}", - "gccSuppressWarnings": false -} diff --git a/platformio/assets/templates/ide-projects/atom/.gitignore.tpl b/platformio/assets/templates/ide-projects/atom/.gitignore.tpl deleted file mode 100644 index bbdd36c798..0000000000 --- a/platformio/assets/templates/ide-projects/atom/.gitignore.tpl +++ /dev/null @@ -1,3 +0,0 @@ -.pio -.clang_complete -.gcc-flags.json diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.cflags.tpl b/platformio/assets/templates/ide-projects/qtcreator/platformio.cflags.tpl deleted file mode 100644 index f09a94f906..0000000000 --- a/platformio/assets/templates/ide-projects/qtcreator/platformio.cflags.tpl +++ /dev/null @@ -1 +0,0 @@ -{{cc_flags.replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.cxxflags.tpl b/platformio/assets/templates/ide-projects/qtcreator/platformio.cxxflags.tpl deleted file mode 100644 index c5b3051128..0000000000 --- a/platformio/assets/templates/ide-projects/qtcreator/platformio.cxxflags.tpl +++ /dev/null @@ -1 +0,0 @@ -{{cxx_flags.replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8670278345..3aa460cdff 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -72,7 +72,8 @@ variables=clivars, # Propagating External Environment ENV=os.environ, - UNIX_TIME=int(time()), + TIMESTAMP=int(time()), + UNIX_TIME="$TIMESTAMP", # deprecated BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index ba194eb076..1646dc95f5 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -200,13 +200,16 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): + p = env.subst(p) if os.path.isdir(p): result[k][i] = os.path.abspath(p) # fix relative path for "-include" for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": - result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path()))) + p = env.subst(f[1].get_path()) + if os.path.exists(p): + result["CCFLAGS"][i] = (f[0], os.path.abspath(p)) return result diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index 3e27dd365c..25ca1eb070 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -16,6 +16,7 @@ import glob import os +import click import SCons.Defaults # pylint: disable=import-error import SCons.Subst # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error @@ -154,8 +155,12 @@ def DumpIntegrationData(*args): ], "defines": dump_defines(projenv), "includes": projenv.DumpIntegrationIncludes(), - "cc_flags": _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"), - "cxx_flags": _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), + "cc_flags": click.parser.split_arg_string( + _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS") + ), + "cxx_flags": click.parser.split_arg_string( + _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS") + ), "cc_path": where_is_program( globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}") ), diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index 77f0e55c16..ccf331f453 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -60,8 +60,8 @@ def _load_cpp_data(self): data = load_build_metadata(self.project_dir, self.envname) if not data: return - self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", "")) - self.cxx_flags = click.parser.split_arg_string(data.get("cxx_flags", "")) + self.cc_flags = data.get("cc_flags", []) + self.cxx_flags = data.get("cxx_flags", []) self.cpp_includes = self._dump_includes(data.get("includes", {})) self.cpp_defines = data.get("defines", []) self.cc_path = data.get("cc_path") diff --git a/platformio/cli.py b/platformio/cli.py index 41aab522e1..a2a707ad70 100644 --- a/platformio/cli.py +++ b/platformio/cli.py @@ -63,6 +63,21 @@ def in_silence(): ] ) + @classmethod + def reveal_cmd_path_args(cls, ctx): + result = [] + group = ctx.command + args = cls.leftover_args[::] + while args: + cmd_name = args.pop(0) + next_group = group.get_command(ctx, cmd_name) + if next_group: + group = next_group + result.append(cmd_name) + if not hasattr(group, "get_command"): + break + return result + def invoke(self, ctx): PlatformioCLI.leftover_args = ctx.args if hasattr(ctx, "protected_args"): diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index da8a2b3e55..9e4d61d2eb 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -53,16 +53,15 @@ def cli(dev, verbose): subprocess.run( [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], check=True, - capture_output=not verbose, + stdout=subprocess.PIPE if not verbose else None, ) - r = subprocess.run( + output = subprocess.run( [python_exe, "-m", "platformio", "--version"], check=True, - capture_output=True, - text=True, - ) - assert "version" in r.stdout - actual_version = r.stdout.split("version", 1)[1].strip() + stdout=subprocess.PIPE, + ).stdout.decode() + assert "version" in output + actual_version = output.split("version", 1)[1].strip() click.secho( "PlatformIO has been successfully upgraded to %s" % actual_version, fg="green", diff --git a/platformio/compat.py b/platformio/compat.py index 17b4cc0aff..008dc7ae01 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -17,6 +17,7 @@ import importlib.util import inspect import locale +import shlex import sys from platformio.exception import UserSideException @@ -29,6 +30,20 @@ from asyncio import get_event_loop as aio_get_running_loop +if sys.version_info >= (3, 8): + from shlex import join as shlex_join +else: + + def shlex_join(split_command): + return " ".join(shlex.quote(arg) for arg in split_command) + + +if sys.version_info >= (3, 9): + from asyncio import to_thread as aio_to_thread +else: + from starlette.concurrency import run_in_threadpool as aio_to_thread + + PY2 = sys.version_info[0] == 2 # DO NOT REMOVE IT. ESP8266/ESP32 depend on it IS_CYGWIN = sys.platform.startswith("cygwin") IS_WINDOWS = WINDOWS = sys.platform.startswith("win") diff --git a/platformio/debug/exception.py b/platformio/debug/exception.py index 7f4d0f4cf1..5147815d3c 100644 --- a/platformio/debug/exception.py +++ b/platformio/debug/exception.py @@ -30,3 +30,7 @@ class DebugSupportError(DebugError, UserSideException): class DebugInvalidOptionsError(DebugError, UserSideException): pass + + +class DebugInitError(DebugError, UserSideException): + pass diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index ce7e82c8bf..29a8ba3a23 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -13,13 +13,13 @@ # limitations under the License. import os -import re import signal import time from platformio import telemetry from platformio.compat import aio_get_running_loop, is_bytes from platformio.debug import helpers +from platformio.debug.exception import DebugInitError from platformio.debug.process.client import DebugClientProcess @@ -130,11 +130,7 @@ def stdout_data_received(self, data): self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: - telemetry.send_event( - "Debug", - "Started", - telemetry.dump_run_environment(self.debug_config.env_options), - ) + telemetry.log_debug_started(self.debug_config) self._auto_exec_continue() def console_log(self, msg): @@ -179,14 +175,7 @@ def _handle_error(self, data): and b"Error in sourced" in self._errors_buffer ): return - - last_erros = self._errors_buffer.decode() - last_erros = " ".join(reversed(last_erros.split("\n"))) - last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) - - err = "%s -> %s" % ( - telemetry.dump_run_environment(self.debug_config.env_options), - last_erros, + telemetry.log_debug_exception( + DebugInitError(self._errors_buffer.decode()), self.debug_config ) - telemetry.send_exception("DebugInitError: %s" % err) self.transport.close() diff --git a/platformio/device/finder.py b/platformio/device/finder.py index fc6ad3c48b..621adaece6 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -163,7 +163,7 @@ def _reveal_device_port(self, device): for item in list_serial_ports(as_objects=True): if item.vid == device.vid and item.pid == device.pid: candidates.append(item) - if len(candidates) == 1: + if len(candidates) <= 1: return device.device for item in candidates: if ("GDB" if self.prefer_gdb_port else "UART") in item.description: diff --git a/platformio/exception.py b/platformio/exception.py index bf2cec8c90..80ffb496bc 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -81,7 +81,7 @@ class InvalidSettingValue(UserSideException): MESSAGE = "Invalid value '{0}' for the setting '{1}'" -class InvalidJSONFile(PlatformioException): +class InvalidJSONFile(ValueError, UserSideException): MESSAGE = "Could not load broken JSON: {0}" diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index 9c79e3147c..4b6195e4b1 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from pathlib import Path from platformio import __version__, app, fs, util from platformio.home.rpc.handlers.base import BaseRPCHandler -from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project @@ -32,16 +30,11 @@ class AppRPC(BaseRPCHandler): "projectsDir", ] - @staticmethod - def get_state_path(): - core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") - if not os.path.isdir(core_dir): - os.makedirs(core_dir) - return os.path.join(core_dir, "homestate.json") - @staticmethod def load_state(): - with app.State(AppRPC.get_state_path(), lock=True) as state: + with app.State( + app.resolve_state_path("core_dir", "homestate.json"), lock=True + ) as state: storage = state.get("storage", {}) # base data @@ -81,7 +74,9 @@ def get_state(): @staticmethod def save_state(state): - with app.State(AppRPC.get_state_path(), lock=True) as s: + with app.State( + app.resolve_state_path("core_dir", "homestate.json"), lock=True + ) as s: s.clear() s.update(state) storage = s.get("storage", {}) diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 0d6b9f48ad..b8dcfc9bfc 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -19,10 +19,10 @@ from functools import cmp_to_key import click -from starlette.concurrency import run_in_threadpool from platformio import fs from platformio.cache import ContentCache +from platformio.compat import aio_to_thread from platformio.device.list.util import list_logical_devices from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.http import HTTPSession, ensure_internet_on @@ -33,12 +33,14 @@ async def request( # pylint: disable=signature-differs,invalid-overridden-metho self, *args, **kwargs ): func = super().request - return await run_in_threadpool(func, *args, **kwargs) + return await aio_to_thread(func, *args, **kwargs) class OSRPC(BaseRPCHandler): - @staticmethod - async def fetch_content(url, data=None, headers=None, cache_valid=None): + _http_session = None + + @classmethod + async def fetch_content(cls, url, data=None, headers=None, cache_valid=None): if not headers: headers = { "User-Agent": ( @@ -57,11 +59,13 @@ async def fetch_content(url, data=None, headers=None, cache_valid=None): # check internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) - session = HTTPAsyncSession() + if not cls._http_session: + cls._http_session = HTTPAsyncSession() + if data: - r = await session.post(url, data=data, headers=headers) + r = await cls._http_session.post(url, data=data, headers=headers) else: - r = await session.get(url, headers=headers) + r = await cls._http_session.get(url, headers=headers) r.raise_for_status() result = r.text @@ -73,9 +77,9 @@ async def fetch_content(url, data=None, headers=None, cache_valid=None): async def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): return await self.fetch_content(uri, data, headers, cache_valid) - if os.path.isfile(uri): - with io.open(uri, encoding="utf-8") as fp: - return fp.read() + local_path = uri[7:] if uri.startswith("file://") else uri + with io.open(local_path, encoding="utf-8") as fp: + return fp.read() return None @staticmethod diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index 080ba96c7f..b09338e51b 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -22,13 +22,13 @@ import click from ajsonrpc.core import JSONRPC20DispatchException -from starlette.concurrency import run_in_threadpool from platformio import __main__, __version__, app, fs, proc, util from platformio.compat import ( IS_WINDOWS, aio_create_task, aio_get_running_loop, + aio_to_thread, get_locale_encoding, is_bytes, ) @@ -177,7 +177,7 @@ async def call(args, options=None): @staticmethod async def _call_subprocess(args, options): - result = await run_in_threadpool( + result = await aio_to_thread( proc.exec_command, [get_core_fullpath()] + args, cwd=options.get("cwd") or os.getcwd(), @@ -197,7 +197,7 @@ def _thread_safe_call(args, cwd): exit_code, ) - return await run_in_threadpool( + return await aio_to_thread( _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd() ) diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py index 2cc959df03..34daf5405a 100644 --- a/platformio/home/rpc/handlers/platform.py +++ b/platformio/home/rpc/handlers/platform.py @@ -12,21 +12,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path + +from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory class PlatformRPC(BaseRPCHandler): + async def fetch_platforms(self, search_query=None, page=0, force_installed=False): + if force_installed: + return { + "items": await aio_to_thread( + self._load_installed_platforms, search_query + ) + } + + search_result = await self.factory.manager.dispatcher["registry.call_client"]( + method="list_packages", + query=search_query, + qualifiers={ + "types": ["platform"], + }, + page=page, + ) + return { + "page": search_result["page"], + "limit": search_result["limit"], + "total": search_result["total"], + "items": [ + { + "id": item["id"], + "ownername": item["owner"]["username"], + "name": item["name"], + "version": item["version"]["name"], + "description": item["description"], + "tier": item["tier"], + } + for item in search_result["items"] + ], + } + @staticmethod - def list_installed(options=None): - result = [] - options = options or {} + def _load_installed_platforms(search_query=None): + search_query = (search_query or "").strip() def _matchSearchQuery(p): - searchQuery = options.get("searchQuery") - if not searchQuery: - return True content_blocks = [p.name, p.title, p.description] if p.frameworks: content_blocks.append(" ".join(p.frameworks.keys())) @@ -34,28 +68,73 @@ def _matchSearchQuery(p): board_data = board.get_brief_data() for key in ("id", "mcu", "vendor"): content_blocks.append(board_data.get(key)) - return searchQuery.strip() in " ".join(content_blocks) + return search_query in " ".join(content_blocks) + items = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): p = PlatformFactory.new(pkg) - if not _matchSearchQuery(p): + if search_query and not _matchSearchQuery(p): continue - result.append( - dict( - __pkg_path=pkg.path, - __pkg_meta=pkg.metadata.as_dict(), - name=p.name, - title=p.title, - description=p.description, - ) + items.append( + { + "__pkg_path": pkg.path, + "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None, + "name": p.name, + "version": str(pkg.metadata.version), + "title": p.title, + "description": p.description, + } ) - return result + return items + + async def fetch_boards(self, platform_spec): + spec = PackageSpec(platform_spec) + if spec.owner: + return await self.factory.manager.dispatcher["registry.call_client"]( + method="get_package", + typex="platform", + owner=spec.owner, + name=spec.name, + extra_path="/boards", + ) + return await aio_to_thread(self._load_installed_boards, spec) @staticmethod - def get_boards(spec): - p = PlatformFactory.new(spec) + def _load_installed_boards(platform_spec): + p = PlatformFactory.new(platform_spec) return sorted( [b.get_brief_data() for b in p.get_boards().values()], key=lambda item: item["name"], ) + + async def fetch_examples(self, platform_spec): + spec = PackageSpec(platform_spec) + if spec.owner: + return await self.factory.manager.dispatcher["registry.call_client"]( + method="get_package", + typex="platform", + owner=spec.owner, + name=spec.name, + extra_path="/examples", + ) + return await aio_to_thread(self._load_installed_examples, spec) + + @staticmethod + def _load_installed_examples(platform_spec): + platform = PlatformFactory.new(platform_spec) + platform_dir = platform.get_dir() + parser = ManifestParserFactory.new_from_dir(platform_dir) + result = parser.as_dict().get("examples") or [] + for example in result: + example["files"] = [ + { + "path": item, + "url": ( + "file://%s" + + os.path.join(platform_dir, "examples", example["name"], item) + ), + } + for item in example["files"] + ] + return result diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 904d00e4d3..65d05063a0 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -15,6 +15,7 @@ import os import shutil import time +from pathlib import Path import semantic_version from ajsonrpc.core import JSONRPC20DispatchException @@ -267,15 +268,39 @@ async def import_pio(project_dir): ) return new_project_dir - async def create_empty(self, configuration, options=None): + async def init_v2(self, configuration, options=None): project_dir = os.path.join(configuration["location"], configuration["name"]) if not os.path.isdir(project_dir): os.makedirs(project_dir) + envclone = os.environ.copy() + envclone["PLATFORMIO_FORCE_ANSI"] = "true" + options = options or {} + options["spawn"] = {"env": envclone, "cwd": project_dir} + + args = ["project", "init"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + + if configuration.get("example"): + await self.factory.notify_clients( + method=options.get("stdoutNotificationMethod"), + params=["Copying example files...\n"], + actor="frontend", + ) + await self._pre_init_example(configuration, project_dir) + else: + args.extend(self._pre_init_empty(configuration)) + + return await self.factory.manager.dispatcher["core.exec"](args, options=options) + + @staticmethod + def _pre_init_empty(configuration): project_options = [] platform = configuration["platform"] - board = configuration.get("board", {}).get("id") - env_name = board or platform["name"] + board_id = configuration.get("board", {}).get("id") + env_name = board_id or platform["name"] if configuration.get("description"): project_options.append(("description", configuration.get("description"))) try: @@ -288,20 +313,25 @@ async def create_empty(self, configuration, options=None): project_options.append( ("platform", "{name} @ {version}".format(**platform)) ) - if board: - project_options.append(("board", board)) + if board_id: + project_options.append(("board", board_id)) if configuration.get("framework"): project_options.append(("framework", configuration["framework"]["name"])) - args = ["project", "init", "-e", env_name, "--sample-code"] - ide = app.get_session_var("caller_id") - if ide in ProjectGenerator.get_supported_ides(): - args.extend(["--ide", ide]) + args = ["-e", env_name, "--sample-code"] for name, value in project_options: args.extend(["-O", f"{name}={value}"]) + return args - envclone = os.environ.copy() - envclone["PLATFORMIO_FORCE_ANSI"] = "true" - options = options or {} - options["spawn"] = {"env": envclone, "cwd": project_dir} - return await self.factory.manager.dispatcher["core.exec"](args, options=options) + async def _pre_init_example(self, configuration, project_dir): + for item in configuration["example"]["files"]: + p = Path(project_dir).joinpath(item["path"]) + if not p.parent.is_dir(): + p.parent.mkdir(parents=True) + p.write_text( + await self.factory.manager.dispatcher["os.request_content"]( + item["url"] + ), + encoding="utf-8", + ) + return [] diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index 1370f32898..84a374b611 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -13,8 +13,8 @@ # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException -from starlette.concurrency import run_in_threadpool +from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.registry.client import RegistryClient @@ -24,7 +24,7 @@ class RegistryRPC(BaseRPCHandler): async def call_client(method, *args, **kwargs): try: client = RegistryClient() - return await run_in_threadpool(getattr(client, method), *args, **kwargs) + return await aio_to_thread(getattr(client, method), *args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( code=5000, message="Registry Call Error", data=str(exc) diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index 2437e40ec0..ab7c13dad2 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -14,6 +14,7 @@ from urllib.parse import parse_qs +import ajsonrpc.utils import click from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request from ajsonrpc.dispatcher import Dispatcher @@ -24,6 +25,10 @@ from platformio.http import InternetConnectionError from platformio.proc import force_exit +# Remove this line when PR is merged +# https://github.com/pavlov99/ajsonrpc/pull/22 +ajsonrpc.utils.is_invalid_params = lambda: False + class JSONRPCServerFactoryBase: connection_nums = 0 diff --git a/platformio/http.py b/platformio/http.py index 48235f7422..ee67b631e9 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -18,7 +18,7 @@ from urllib.parse import urljoin import requests.adapters -from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error +from urllib3.util.retry import Retry from platformio import __check_internet_hosts__, app, util from platformio.cache import ContentCache, cleanup_content_cache @@ -27,7 +27,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) -class HTTPClientError(PlatformioException): +class HTTPClientError(UserSideException): def __init__(self, message, response=None): super().__init__() self.message = message @@ -50,7 +50,10 @@ def __init__(self, *args, **kwargs): self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None super().__init__(*args, **kwargs) self.headers.update({"User-Agent": app.get_user_agent()}) - self.verify = app.get_setting("enable_proxy_strict_ssl") + try: + self.verify = app.get_setting("enable_proxy_strict_ssl") + except PlatformioException: + self.verify = True def request( # pylint: disable=signature-differs,arguments-differ self, method, url, *args, **kwargs @@ -154,7 +157,10 @@ def fetch_json_data(self, method, path, **kwargs): with ContentCache("http") as cc: result = cc.get(cache_key) if result is not None: - return json.loads(result) + try: + return json.loads(result) + except json.JSONDecodeError: + pass response = self.send_request(method, path, **kwargs) data = self._parse_json_response(response) cc.set(cache_key, response.text, cache_valid) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 3110b2c192..45b4e5c95c 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,23 +25,20 @@ from platformio.commands.upgrade import get_latest_version from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages -from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageSpec from platformio.package.version import pepver_to_semver from platformio.system.prune import calculate_unnecessary_system_data -def on_platformio_start(ctx, caller): +def on_cmd_start(ctx, caller): app.set_session_var("command_ctx", ctx) set_caller(caller) - telemetry.on_command() - + telemetry.on_cmd_start(ctx) if PlatformioCLI.in_silence(): return after_upgrade(ctx) -def on_platformio_end(ctx, result): # pylint: disable=unused-argument +def on_cmd_end(): if PlatformioCLI.in_silence(): return @@ -60,8 +57,12 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument ) -def on_platformio_exception(e): - telemetry.on_exception(e) +def on_platformio_exception(exc): + telemetry.log_exception(exc) + + +def on_platformio_exit(): + telemetry.on_exit() def set_caller(caller=None): @@ -79,11 +80,10 @@ def set_caller(caller=None): class Upgrader: def __init__(self, from_version, to_version): - self.from_version = pepver_to_semver(from_version) - self.to_version = pepver_to_semver(to_version) - + self.from_version = from_version + self.to_version = to_version self._upgraders = [ - (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), + (semantic_version.Version("6.1.8-a.1"), self._appstate_migration), ] def run(self, ctx): @@ -99,37 +99,43 @@ def run(self, ctx): return all(result) @staticmethod - def _update_pkg_metadata(_): - pm = ToolPackageManager() - for pkg in pm.get_installed(): - if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: - continue - result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) - if len(result) != 1: - continue - result = result[0] - pkg.metadata.spec = PackageSpec( - id=result["id"], - owner=result["owner"]["username"], - name=result["name"], + def _appstate_migration(_): + state_path = app.resolve_state_path("core_dir", "appstate.json") + if not os.path.isfile(state_path): + return True + app.delete_state_item("telemetry") + created_at = app.get_state_item("created_at", None) + if not created_at: + state_stat = os.stat(state_path) + app.set_state_item( + "created_at", + int( + state_stat.st_birthtime + if hasattr(state_stat, "st_birthtime") + else state_stat.st_ctime + ), ) - pkg.dump_meta() return True def after_upgrade(ctx): terminal_width = shutil.get_terminal_size().columns - last_version = app.get_state_item("last_version", "0.0.0") - if last_version == __version__: - return + last_version_str = app.get_state_item("last_version", "0.0.0") + if last_version_str == __version__: + return None - if last_version == "0.0.0": + if last_version_str == "0.0.0": app.set_state_item("last_version", __version__) - elif pepver_to_semver(last_version) > pepver_to_semver(__version__): + return print_welcome_banner() + + last_version = pepver_to_semver(last_version_str) + current_version = pepver_to_semver(__version__) + + if last_version > current_version and not last_version.prerelease: click.secho("*" * terminal_width, fg="yellow") click.secho( "Obsolete PIO Core v%s is used (previous was %s)" - % (__version__, last_version), + % (__version__, last_version_str), fg="yellow", ) click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") @@ -139,43 +145,50 @@ def after_upgrade(ctx): fg="cyan", ) click.secho("*" * terminal_width, fg="yellow") - return - else: - click.secho("Please wait while upgrading PlatformIO...", fg="yellow") - - # Update PlatformIO's Core packages - cleanup_content_cache("http") - update_core_packages() - - u = Upgrader(last_version, __version__) - if u.run(ctx): - app.set_state_item("last_version", __version__) - click.secho( - "PlatformIO has been successfully upgraded to %s!\n" % __version__, - fg="green", - ) - telemetry.send_event( - category="Auto", - action="Upgrade", - label="%s > %s" % (last_version, __version__), - ) + return None + + click.secho("Please wait while upgrading PlatformIO...", fg="yellow") + + # Update PlatformIO's Core packages + cleanup_content_cache("http") + update_core_packages() - # PlatformIO banner + u = Upgrader(last_version, current_version) + if u.run(ctx): + app.set_state_item("last_version", __version__) + click.secho( + "PlatformIO has been successfully upgraded to %s!\n" % __version__, + fg="green", + ) + telemetry.log_event( + "pio_upgrade_core", + { + "label": "%s > %s" % (last_version_str, __version__), + "from_version": last_version_str, + "to_version": __version__, + }, + ) + + return print_welcome_banner() + + +def print_welcome_banner(): + terminal_width = shutil.get_terminal_size().columns click.echo("*" * terminal_width) click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) click.echo( - "- %s us on Twitter to stay up-to-date " - "on the latest project news > %s" + "- %s it on GitHub > %s" % ( - click.style("follow", fg="cyan"), - click.style("https://twitter.com/PlatformIO_Org", fg="cyan"), + click.style("star", fg="cyan"), + click.style("https://github.com/platformio/platformio-core", fg="cyan"), ) ) click.echo( - "- %s it on GitHub > %s" + "- %s us on LinkedIn to stay up-to-date " + "on the latest project news > %s" % ( - click.style("star", fg="cyan"), - click.style("https://github.com/platformio/platformio", fg="cyan"), + click.style("follow", fg="cyan"), + click.style("https://www.linkedin.com/company/platformio/", fg="cyan"), ) ) if not os.getenv("PLATFORMIO_IDE"): @@ -228,7 +241,7 @@ def check_platformio_upgrade(): else: click.secho("platformio upgrade", fg="cyan", nl=False) click.secho("` or `", fg="yellow", nl=False) - click.secho("pip install -U platformio", fg="cyan", nl=False) + click.secho("python -m pip install -U platformio", fg="cyan", nl=False) click.secho("` command.", fg="yellow") click.secho("Changes: ", fg="yellow", nl=False) click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index c2426c9c6c..9d5fee4046 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -59,7 +59,8 @@ def humanize_package(pkg, spec=None, verbose=False): if spec and not isinstance(spec, PackageSpec): spec = PackageSpec(spec) data = [ - click.style("{name} @ {version}".format(**pkg.metadata.as_dict()), fg="cyan") + click.style(pkg.metadata.name, fg="cyan"), + click.style(f"@ {str(pkg.metadata.version)}", bold=True), ] extra_data = ["required: %s" % (spec.humanize() if spec else "Any")] if verbose: @@ -135,20 +136,20 @@ def list_global_packages(options): ("libraries", LibraryPackageManager(options.get("storage_dir"))), ] only_packages = any( - options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data + options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data ) - for type_, pm in data: + for typex, pm in data: skip_conds = [ only_packages - and not options.get(type_) - and not options.get(f"only_{type_}"), + and not options.get(typex) + and not options.get(f"only_{typex}"), not pm.get_installed(), ] if any(skip_conds): continue - click.secho(type_.capitalize(), bold=True) + click.secho(typex.capitalize(), bold=True) print_dependency_tree( - pm, filter_specs=options.get(type_), verbose=options.get("verbose") + pm, filter_specs=options.get(typex), verbose=options.get("verbose") ) click.echo() @@ -156,12 +157,12 @@ def list_global_packages(options): def list_project_packages(options): environments = options["environments"] only_packages = any( - options.get(type_) or options.get(f"only_{type_}") - for type_ in ("platforms", "tools", "libraries") + options.get(typex) or options.get(f"only_{typex}") + for typex in ("platforms", "tools", "libraries") ) only_platform_packages = any( - options.get(type_) or options.get(f"only_{type_}") - for type_ in ("platforms", "tools") + options.get(typex) or options.get(f"only_{typex}") + for typex in ("platforms", "tools") ) only_library_packages = options.get("libraries") or options.get("only_libraries") diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index 228c1411ba..c8b4ea7774 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -56,7 +56,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument ) @click.option( "--type", - "type_", + "typex", type=click.Choice(list(PackageType.items().values())), help="Custom package type", ) @@ -83,7 +83,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument hidden=True, ) def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals - package, owner, type_, released_at, private, notify, no_interactive, non_interactive + package, owner, typex, released_at, private, notify, no_interactive, non_interactive ): click.secho("Preparing a package...", fg="cyan") no_interactive = no_interactive or non_interactive @@ -103,14 +103,14 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals p = PackagePacker(package) archive_path = p.pack() - type_ = type_ or PackageType.from_archive(archive_path) + typex = typex or PackageType.from_archive(archive_path) manifest = ManifestSchema().load_manifest( ManifestParserFactory.new_from_archive(archive_path).as_dict() ) name = manifest.get("name") version = manifest.get("version") data = [ - ("Type:", type_), + ("Type:", typex), ("Owner:", owner), ("Name:", name), ("Version:", version), @@ -124,13 +124,13 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals check_archive_file_names(archive_path) # look for duplicates - check_package_duplicates(owner, type_, name, version, manifest.get("system")) + check_package_duplicates(owner, typex, name, version, manifest.get("system")) if not no_interactive: click.confirm( "Are you sure you want to publish the %s %s to the registry?\n" % ( - type_, + typex, click.style( "%s/%s@%s" % (owner, name, version), fg="cyan", @@ -146,7 +146,7 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals ) click.echo("Publishing...") response = RegistryClient().publish_package( - owner, type_, archive_path, released_at, private, notify + owner, typex, archive_path, released_at, private, notify ) if not do_not_pack: os.remove(archive_path) diff --git a/platformio/package/exception.py b/platformio/package/exception.py index d38c0e602b..ba896faef8 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -13,10 +13,10 @@ # limitations under the License. from platformio import util -from platformio.exception import PlatformioException, UserSideException +from platformio.exception import UserSideException -class PackageException(PlatformioException): +class PackageException(UserSideException): pass @@ -51,14 +51,14 @@ class MissingPackageManifestError(ManifestException): MESSAGE = "Could not find one of '{0}' manifest files in the package" -class UnknownPackageError(UserSideException): +class UnknownPackageError(PackageException): MESSAGE = ( "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() ) -class NotGlobalLibDir(UserSideException): +class NotGlobalLibDir(PackageException): MESSAGE = ( "The `{0}` is not a PlatformIO project.\n\n" "To manage libraries in global storage `{1}`,\n" diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 3c6b2047d5..bc2a4347ca 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -15,7 +15,7 @@ import os from time import sleep, time -from platformio.exception import PlatformioException +from platformio.exception import UserSideException LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour LOCKFILE_DELAY = 0.2 @@ -36,11 +36,11 @@ LOCKFILE_CURRENT_INTERFACE = None -class LockFileExists(PlatformioException): +class LockFileExists(UserSideException): pass -class LockFileTimeoutError(PlatformioException): +class LockFileTimeoutError(UserSideException): pass diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 494e2d2412..4c339be8a7 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -25,12 +25,13 @@ class PackageManagerRegistryMixin: def install_from_registry(self, spec, search_qualifiers=None): + package = version = None if spec.owner and spec.name and not search_qualifiers: package = self.fetch_registry_package(spec) if not package: raise UnknownPackageError(spec.humanize()) version = self.pick_best_registry_version(package["versions"], spec) - else: + elif spec.id or spec.name: packages = self.search_registry_packages(spec, search_qualifiers) if not packages: raise UnknownPackageError(spec.humanize()) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 2075ac0381..95e081080b 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -183,7 +183,7 @@ class ManifestSchema(BaseSchema): validate=[ validate.Length(min=1, max=50), validate.Regexp( - r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed" + r"^[a-z\d\-_\+\. ]+$", error="Only [a-z0-9+_-. ] chars are allowed" ), ] ) @@ -276,9 +276,9 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.20" + version = "3.21" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" - "v%s/json/licenses.json" % version + f"v{version}/json/licenses.json" ) return json.loads(fetch_remote_content(spdx_data_url)) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 4db5310607..36578a3726 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -24,7 +24,7 @@ from platformio import fs from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType -from platformio.package.version import cast_version_to_semver +from platformio.package.version import SemanticVersionError, cast_version_to_semver from platformio.util import items_in_list @@ -175,7 +175,7 @@ def __init__( # pylint: disable=redefined-builtin,too-many-arguments if requirements: try: self.requirements = requirements - except ValueError as exc: + except SemanticVersionError as exc: if not self.name or self.uri or self.raw: raise exc self.raw = "%s=%s" % (self.name, requirements) @@ -224,11 +224,14 @@ def requirements(self, value): if not value: self._requirements = None return - self._requirements = ( - value - if isinstance(value, semantic_version.SimpleSpec) - else semantic_version.SimpleSpec(str(value)) - ) + try: + self._requirements = ( + value + if isinstance(value, semantic_version.SimpleSpec) + else semantic_version.SimpleSpec(str(value)) + ) + except ValueError as exc: + raise SemanticVersionError(exc) from exc def humanize(self): result = "" diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index 058a4983f7..67348391c5 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -18,14 +18,10 @@ from urllib.parse import urlparse from platformio import proc -from platformio.package.exception import ( - PackageException, - PlatformioException, - UserSideException, -) +from platformio.exception import UserSideException -class VCSBaseException(PackageException): +class VCSBaseException(UserSideException): pass @@ -74,8 +70,8 @@ def check_client(self): self.get_cmd_output(["--version"]) else: assert self.run_cmd(["--version"]) - except (AssertionError, OSError, PlatformioException) as exc: - raise UserSideException( + except (AssertionError, OSError) as exc: + raise VCSBaseException( "VCS: `%s` client is not installed in your system" % self.command ) from exc return True diff --git a/platformio/package/version.py b/platformio/package/version.py index 770be9e4b2..909d83e134 100644 --- a/platformio/package/version.py +++ b/platformio/package/version.py @@ -16,6 +16,12 @@ import semantic_version +from platformio.exception import UserSideException + + +class SemanticVersionError(UserSideException): + pass + def cast_version_to_semver(value, force=True, raise_exception=False): assert value @@ -29,7 +35,7 @@ def cast_version_to_semver(value, force=True, raise_exception=False): except ValueError: pass if raise_exception: - raise ValueError("Invalid SemVer version %s" % value) + raise SemanticVersionError("Invalid SemVer version %s" % value) # parse commit hash if re.match(r"^[\da-f]+$", value, flags=re.I): return semantic_version.Version("0.0.0+sha." + value) diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index dfd7c40675..daca77c311 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -52,7 +52,6 @@ def run( # pylint: disable=too-many-arguments self.ensure_engine_compatible() self.configure_project_packages(variables["pioenv"], targets) - self._report_non_sensitive_data(variables["pioenv"], targets) self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") @@ -64,20 +63,13 @@ def run( # pylint: disable=too-many-arguments if not os.path.isfile(variables["build_script"]): raise BuildScriptNotFound(variables["build_script"]) + telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) result = self._run_scons(variables, targets, jobs) + assert "returncode" in result return result - def _report_non_sensitive_data(self, env, targets): - options = self.config.items(env=env, as_dict=True) - options["platform_packages"] = [ - dict(name=item["name"], version=item["version"]) - for item in self.dump_used_packages() - ] - options["platform"] = {"name": self.name, "version": self.version} - telemetry.send_run_environment(options, targets) - def _run_scons(self, variables, targets, jobs): scons_dir = get_core_package_dir("tool-scons") args = [ diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 73a3ebd778..78d48943e6 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -14,10 +14,10 @@ import os -from platformio import fs, telemetry, util +from platformio import fs, util from platformio.compat import MISSING from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError -from platformio.exception import UserSideException +from platformio.exception import InvalidJSONFile, UserSideException from platformio.platform.exception import InvalidBoardManifest @@ -28,7 +28,7 @@ def __init__(self, manifest_path): self.manifest_path = manifest_path try: self._manifest = fs.load_json(manifest_path) - except ValueError as exc: + except InvalidJSONFile as exc: raise InvalidBoardManifest(manifest_path) from exc if not set(["name", "url", "vendor"]) <= set(self._manifest): raise UserSideException( @@ -119,7 +119,6 @@ def get_debug_tool_name(self, custom=None): if tool_name == "custom": return tool_name if not debug_tools: - telemetry.send_event("Debug", "Request", self.id) raise DebugSupportError(self._manifest["name"]) if tool_name: if tool_name in debug_tools: diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index f044d3b7a7..18ace0dee0 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import PlatformioException +from platformio.exception import UserSideException -class PlatformException(PlatformioException): +class PlatformException(UserSideException): pass diff --git a/platformio/project/commands/config.py b/platformio/project/commands/config.py index 9f218e8024..5e4ed21fd6 100644 --- a/platformio/project/commands/config.py +++ b/platformio/project/commands/config.py @@ -30,16 +30,23 @@ default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) +@click.option("--lint", is_flag=True) @click.option("--json-output", is_flag=True) -def project_config_cmd(project_dir, json_output): +def project_config_cmd(project_dir, lint, json_output): if not is_platformio_project(project_dir): raise NotPlatformIOProjectError(project_dir) with fs.cd(project_dir): - config = ProjectConfig.get_instance() + if lint: + return lint_configuration(json_output) + return print_configuration(json_output) + + +def print_configuration(json_output=False): + config = ProjectConfig.get_instance() if json_output: return click.echo(config.to_json()) click.echo( - "Computed project configuration for %s" % click.style(project_dir, fg="cyan") + "Computed project configuration for %s" % click.style(os.getcwd(), fg="cyan") ) for section, options in config.as_tuple(): click.secho(section, fg="cyan") @@ -55,3 +62,43 @@ def project_config_cmd(project_dir, json_output): ) click.echo() return None + + +def lint_configuration(json_output=False): + result = ProjectConfig.lint() + errors = result["errors"] + warnings = result["warnings"] + if json_output: + return click.echo(result) + if not errors and not warnings: + return click.secho( + 'The "platformio.ini" configuration file is free from linting errors.', + fg="green", + ) + if errors: + click.echo( + tabulate( + [ + ( + click.style(error["type"], fg="red"), + error["message"], + error.get("source", "") + (f":{error.get('lineno')}") + if "lineno" in error + else "", + ) + for error in errors + ], + tablefmt="plain", + ) + ) + if warnings: + click.echo( + tabulate( + [ + (click.style("Warning", fg="yellow"), warning) + for warning in warnings + ], + tablefmt="plain", + ) + ) + return None diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py index fbf71b8bbd..ee9f5b194c 100644 --- a/platformio/project/commands/metadata.py +++ b/platformio/project/commands/metadata.py @@ -37,6 +37,7 @@ @click.option("--json-output", is_flag=True) @click.option("--json-output-path", type=click.Path()) def project_metadata_cmd(project_dir, environments, json_output, json_output_path): + project_dir = os.path.abspath(project_dir) with fs.cd(project_dir): config = ProjectConfig.get_instance() config.validate(environments) diff --git a/platformio/project/config.py b/platformio/project/config.py index c2b072ef16..8020fdd305 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -86,7 +86,7 @@ def __init__(self, path=None, parse_extra=True, expand_interpolations=True): if path and os.path.isfile(path): self.read(path, parse_extra) - self._maintain_renaimed_options() + self._maintain_renamed_options() def __getattr__(self, name): return getattr(self._parser, name) @@ -98,7 +98,7 @@ def read(self, path, parse_extra=True): try: self._parser.read(path, "utf-8") except configparser.Error as exc: - raise exception.InvalidProjectConfError(path, str(exc)) + raise exception.InvalidProjectConfError(path, str(exc)) from exc if not parse_extra: return @@ -110,7 +110,7 @@ def read(self, path, parse_extra=True): for item in glob.glob(pattern, recursive=True): self.read(item) - def _maintain_renaimed_options(self): + def _maintain_renamed_options(self): renamed_options = {} for option in ProjectOptions.values(): if option.oldnames: @@ -324,6 +324,7 @@ def _re_interpolation_handler(self, parent_section, match): f"`${{this.__env__}}` is called from the `{parent_section}` " "section that is not valid PlatformIO environment, see", option, + " ", section, ) return parent_section[4:] @@ -332,7 +333,10 @@ def _re_interpolation_handler(self, parent_section, match): value = self.get(section, option) except RecursionError as exc: raise exception.ProjectOptionValueError( - "Infinite recursion has been detected", option, section + "Infinite recursion has been detected", + option, + " ", + section, ) from exc if isinstance(value, list): return "\n".join(value) @@ -359,7 +363,10 @@ def get(self, section, option, default=MISSING): if not self.expand_interpolations: return value raise exception.ProjectOptionValueError( - exc.format_message(), option, section + exc.format_message(), + option, + " (%s) " % option_meta.description, + section, ) @staticmethod @@ -424,16 +431,51 @@ def validate(self, envs=None, silent=False): return True +class ProjectConfigLintMixin: + @classmethod + def lint(cls, path=None): + errors = [] + warnings = [] + try: + config = cls.get_instance(path) + config.validate(silent=True) + warnings = config.warnings + config.as_tuple() + except Exception as exc: # pylint: disable=broad-exception-caught + if exc.__cause__ is not None: + exc = exc.__cause__ + + item = {"type": exc.__class__.__name__, "message": str(exc)} + for attr in ("lineno", "source"): + if hasattr(exc, attr): + item[attr] = getattr(exc, attr) + + if item["type"] == "ParsingError" and hasattr(exc, "errors"): + for lineno, line in getattr(exc, "errors"): + errors.append( + { + "type": item["type"], + "message": f"Parsing error: {line}", + "lineno": lineno, + "source": item["source"], + } + ) + else: + errors.append(item) + return {"errors": errors, "warnings": warnings} + + class ProjectConfigDirsMixin: def get_optional_dir(self, name): """ Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0 + PIO Home 3.0 Project Inspection depends on it """ return self.get("platformio", f"{name}_dir") -class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): +class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin, ProjectConfigDirsMixin): _instances = {} @staticmethod diff --git a/platformio/project/exception.py b/platformio/project/exception.py index 95681bc0df..3821c865b6 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -51,4 +51,4 @@ class InvalidEnvNameError(ProjectError, UserSideException): class ProjectOptionValueError(ProjectError, UserSideException): - MESSAGE = "{0} for option `{1}` in section [{2}]" + MESSAGE = "{0} for option `{1}`{2}in section [{3}]" diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index aec12dd98a..cb004d23ca 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -164,6 +164,7 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False): def _load_build_metadata(project_dir, env_names, debug=False): # pylint: disable=import-outside-toplevel + from platformio import app from platformio.run.cli import cli as cmd_run args = ["--project-dir", project_dir, "--target", "__idedata"] @@ -171,13 +172,15 @@ def _load_build_metadata(project_dir, env_names, debug=False): args.extend(["--target", "__debug"]) for name in env_names: args.extend(["-e", name]) + app.set_session_var("pause_telemetry", True) result = CliRunner().invoke(cmd_run, args) + app.set_session_var("pause_telemetry", False) if result.exit_code != 0 and not isinstance( result.exception, exception.ReturnErrorCode ): raise result.exception if '"includes":' not in result.output: - raise exception.PlatformioException(result.output) + raise exception.UserSideException(result.output) return _get_cached_build_metadata(project_dir, env_names) diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 654e5ac353..d55f9aa7dc 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -52,7 +52,7 @@ def get_best_envname(self, boards=None): @staticmethod def get_ide_tpls_dir(): - return os.path.join(fs.get_assets_dir(), "templates", "ide-projects") + return os.path.join(os.path.dirname(__file__), "tpls") @classmethod def get_supported_ides(cls): diff --git a/platformio/assets/templates/ide-projects/clion/.gitignore.tpl b/platformio/project/integration/tpls/clion/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/clion/.gitignore.tpl rename to platformio/project/integration/tpls/clion/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/clion/CMakeLists.txt.tpl b/platformio/project/integration/tpls/clion/CMakeLists.txt.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/clion/CMakeLists.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeLists.txt.tpl diff --git a/platformio/assets/templates/ide-projects/clion/CMakeListsPrivate.txt.tpl b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl similarity index 86% rename from platformio/assets/templates/ide-projects/clion/CMakeListsPrivate.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl index 6297706f9c..c65848f6b9 100644 --- a/platformio/assets/templates/ide-projects/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl @@ -8,6 +8,7 @@ % import os % import re % +% from platformio.compat import shlex_join % from platformio.project.helpers import load_build_metadata % % def _normalize_path(path): @@ -64,17 +65,16 @@ set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Periph SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") -SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}") -SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}") +SET(CMAKE_CXX_FLAGS {{ _normalize_path(to_unix_path(shlex_join(cxx_flags))) }}) +SET(CMAKE_C_FLAGS {{ _normalize_path(to_unix_path(shlex_join(cc_flags))) }}) -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) +% cc_stds = [arg for arg in cc_flags if arg.startswith("-std=")] +% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")] % if cc_stds: -SET(CMAKE_C_STANDARD {{ cc_stds[-1] }}) +SET(CMAKE_C_STANDARD {{ cc_stds[-1][-2:] }}) % end % if cxx_stds: -set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }}) +set(CMAKE_CXX_STANDARD {{ cxx_stds[-1][-2:] }}) % end if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}") diff --git a/platformio/assets/templates/ide-projects/codeblocks/platformio.cbp.tpl b/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/codeblocks/platformio.cbp.tpl rename to platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.cproject.tpl b/platformio/project/integration/tpls/eclipse/.cproject.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.cproject.tpl rename to platformio/project/integration/tpls/eclipse/.cproject.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.project.tpl b/platformio/project/integration/tpls/eclipse/.project.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.project.tpl rename to platformio/project/integration/tpls/eclipse/.project.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.settings/PlatformIO Debugger.launch.tpl rename to platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.settings/language.settings.xml.tpl b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl similarity index 95% rename from platformio/assets/templates/ide-projects/eclipse/.settings/language.settings.xml.tpl rename to platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl index d8826fca80..f2123be679 100644 --- a/platformio/assets/templates/ide-projects/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl @@ -1,6 +1,4 @@ -% import re -% STD_RE = re.compile(r"(\-std=[a-z\+]+\w+)") -% cxx_stds = STD_RE.findall(cxx_flags) +% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")] % cxx_std = cxx_stds[-1] if cxx_stds else "" % % if cxx_path.startswith(user_home_dir): diff --git a/platformio/assets/templates/ide-projects/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl rename to platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl diff --git a/platformio/assets/templates/ide-projects/sublimetext/.ccls.tpl b/platformio/project/integration/tpls/emacs/.ccls.tpl similarity index 50% rename from platformio/assets/templates/ide-projects/sublimetext/.ccls.tpl rename to platformio/project/integration/tpls/emacs/.ccls.tpl index a747bb61b2..94d376802a 100644 --- a/platformio/assets/templates/ide-projects/sublimetext/.ccls.tpl +++ b/platformio/project/integration/tpls/emacs/.ccls.tpl @@ -1,7 +1,9 @@ +% from platformio.compat import shlex_join +% clang -{{"%c"}} {{ !cc_flags }} -{{"%cpp"}} {{ !cxx_flags }} +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} diff --git a/platformio/assets/templates/ide-projects/emacs/.gitignore.tpl b/platformio/project/integration/tpls/emacs/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/emacs/.gitignore.tpl rename to platformio/project/integration/tpls/emacs/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/private/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/private/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/private/launcher.properties.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/private/launcher.properties.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/private/private.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/private/private.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/project.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/project.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/.gitignore.tpl b/platformio/project/integration/tpls/qtcreator/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/.gitignore.tpl rename to platformio/project/integration/tpls/qtcreator/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/Makefile.tpl b/platformio/project/integration/tpls/qtcreator/Makefile.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/Makefile.tpl rename to platformio/project/integration/tpls/qtcreator/Makefile.tpl diff --git a/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl new file mode 100644 index 0000000000..024c813ddc --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl @@ -0,0 +1,3 @@ +% from platformio.compat import shlex_join +% +{{shlex_join(cc_flags).replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.config.tpl b/platformio/project/integration/tpls/qtcreator/platformio.config.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.config.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.config.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.creator.tpl b/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.creator.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.creator.tpl diff --git a/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl new file mode 100644 index 0000000000..551f02e1ab --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl @@ -0,0 +1,3 @@ +% from platformio.compat import shlex_join +% +{{shlex_join(cxx_flags).replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.files.tpl b/platformio/project/integration/tpls/qtcreator/platformio.files.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.files.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.files.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.includes.tpl b/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.includes.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.includes.tpl diff --git a/platformio/assets/templates/ide-projects/vim/.ccls.tpl b/platformio/project/integration/tpls/sublimetext/.ccls.tpl similarity index 50% rename from platformio/assets/templates/ide-projects/vim/.ccls.tpl rename to platformio/project/integration/tpls/sublimetext/.ccls.tpl index a747bb61b2..94d376802a 100644 --- a/platformio/assets/templates/ide-projects/vim/.ccls.tpl +++ b/platformio/project/integration/tpls/sublimetext/.ccls.tpl @@ -1,7 +1,9 @@ +% from platformio.compat import shlex_join +% clang -{{"%c"}} {{ !cc_flags }} -{{"%cpp"}} {{ !cxx_flags }} +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} diff --git a/platformio/assets/templates/ide-projects/sublimetext/platformio.sublime-project.tpl b/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/sublimetext/platformio.sublime-project.tpl rename to platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl diff --git a/platformio/assets/templates/ide-projects/emacs/.ccls.tpl b/platformio/project/integration/tpls/vim/.ccls.tpl similarity index 50% rename from platformio/assets/templates/ide-projects/emacs/.ccls.tpl rename to platformio/project/integration/tpls/vim/.ccls.tpl index a747bb61b2..94d376802a 100644 --- a/platformio/assets/templates/ide-projects/emacs/.ccls.tpl +++ b/platformio/project/integration/tpls/vim/.ccls.tpl @@ -1,7 +1,9 @@ +% from platformio.compat import shlex_join +% clang -{{"%c"}} {{ !cc_flags }} -{{"%cpp"}} {{ !cxx_flags }} +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} diff --git a/platformio/assets/templates/ide-projects/vim/.gitignore.tpl b/platformio/project/integration/tpls/vim/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vim/.gitignore.tpl rename to platformio/project/integration/tpls/vim/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.filters.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.filters.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl diff --git a/platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.gitignore.tpl b/platformio/project/integration/tpls/vscode/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.gitignore.tpl rename to platformio/project/integration/tpls/vscode/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl similarity index 69% rename from platformio/assets/templates/ide-projects/vscode/.vscode/c_cpp_properties.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl index ac525e56cd..0be8a3b480 100644 --- a/platformio/assets/templates/ide-projects/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -1,27 +1,25 @@ % import os % import platform -% import re -% -% import click % % systype = platform.system().lower() % % cpp_standards_remap = { -% "0x": "11", -% "1y": "14", -% "1z": "17", -% "2a": "20", -% "2b": "23" +% "c++0x": "c++11", +% "c++1y": "c++14", +% "c++1z": "c++17", +% "c++2a": "c++20", +% "c++2b": "c++23", +% "gnu++0x": "gnu++11", +% "gnu++1y": "gnu++14", +% "gnu++1z": "gnu++17", +% "gnu++2a": "gnu++20", +% "gnu++2b": "gnu++23" % } % % def _escape(text): % return to_unix_path(text).replace('"', '\\"') % end % -% def split_args(args_string): -% return click.parser.split_arg_string(to_unix_path(args_string)) -% end -% % def filter_args(args, allowed, ignore=None): % if not allowed: % return [] @@ -31,18 +29,18 @@ % result = [] % i = 0 % length = len(args) -% while(i < length): -% if any(args[i].startswith(f) for f in allowed) and not any( -% args[i].startswith(f) for f in ignore): -% result.append(args[i]) -% if i + 1 < length and not args[i + 1].startswith("-"): -% i += 1 -% result.append(args[i]) -% end +% while(i < length): +% if any(args[i].startswith(f) for f in allowed) and not any( +% args[i].startswith(f) for f in ignore): +% result.append(args[i]) +% if i + 1 < length and not args[i + 1].startswith("-"): +% i += 1 +% result.append(args[i]) % end -% i += 1 -% end -% return result +% end +% i += 1 +% end +% return result % end % % def _find_abs_path(inc, inc_paths): @@ -76,12 +74,10 @@ % % cleaned_includes = filter_includes(includes, ["toolchain"]) % -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) -% cc_m_flags = split_args(cc_flags) +% cc_stds = [arg[5:] for arg in cc_flags if arg.startswith("-std=")] +% cxx_stds = [arg[5:] for arg in cxx_flags if arg.startswith("-std=")] % forced_includes = _find_forced_includes( -% filter_args(cc_m_flags, ["-include", "-imacros"]), cleaned_includes) +% filter_args(cc_flags, ["-include", "-imacros"]), cleaned_includes) % // // !!! WARNING !!! AUTO-GENERATED FILE! @@ -114,10 +110,10 @@ "" ], % if cc_stds: - "cStandard": "c{{ cc_stds[-1] }}", + "cStandard": "{{ cc_stds[-1] }}", % end % if cxx_stds: - "cppStandard": "c++{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", + "cppStandard": "{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", % end % if forced_includes: "forcedInclude": [ @@ -130,7 +126,7 @@ "compilerPath": "{{ cc_path }}", "compilerArgs": [ % for flag in [ -% f for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) +% f for f in filter_args(cc_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) % ]: "{{ flag }}", % end diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/extensions.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.vscode/extensions.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl diff --git a/platformio/registry/client.py b/platformio/registry/client.py index 1a89e345a2..6173f7dfe6 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -142,12 +142,12 @@ def list_packages(self, query=None, qualifiers=None, page=None, sort=None): x_with_authorization=self.allowed_private_packages(), ) - def get_package(self, type_, owner, name, version=None, extra_path=None): + def get_package(self, typex, owner, name, version=None, extra_path=None): try: return self.fetch_json_data( "get", "/v3/packages/{owner}/{type}/{name}{extra_path}".format( - type=type_, + type=typex, owner=owner.lower(), name=name.lower(), extra_path=extra_path or "", diff --git a/platformio/system/commands/info.py b/platformio/system/commands/info.py index 52a066362d..4db1f42895 100644 --- a/platformio/system/commands/info.py +++ b/platformio/system/commands/info.py @@ -77,7 +77,6 @@ def system_info_cmd(json_output): ).get_installed() ), } - click.echo( json.dumps(data) if json_output diff --git a/platformio/system/completion.py b/platformio/system/completion.py index 57e2f1d325..100d1a1d2b 100644 --- a/platformio/system/completion.py +++ b/platformio/system/completion.py @@ -29,8 +29,10 @@ class ShellType(Enum): def get_bash_version(): - result = subprocess.run(["bash", "--version"], capture_output=True, check=True) - match = re.search(r"version\s+(\d+)\.(\d+)", result.stdout.decode(), re.IGNORECASE) + output = subprocess.run( + ["bash", "--version"], check=True, stdout=subprocess.PIPE + ).stdout.decode() + match = re.search(r"version\s+(\d+)\.(\d+)", output, re.IGNORECASE) if match: return (int(match.group(1)), int(match.group(2))) return (0, 0) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index e9cbcd583d..820b1507a6 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -14,299 +14,181 @@ import atexit import hashlib -import json import os import queue import re -import shutil import sys import threading +import time +import traceback from collections import deque -from time import sleep, time -from traceback import format_exc import requests -from platformio import __version__, app, exception, util +from platformio import __title__, __version__, app, exception, fs, util from platformio.cli import PlatformioCLI -from platformio.compat import hashlib_encode_data, string_types -from platformio.http import HTTPSession -from platformio.proc import is_ci, is_container -from platformio.project.helpers import is_platformio_project +from platformio.compat import hashlib_encode_data +from platformio.debug.config.base import DebugConfigBase +from platformio.http import HTTPSession, ensure_internet_on +from platformio.proc import is_ci +KEEP_MAX_REPORTS = 100 +SEND_MAX_EVENTS = 25 -class TelemetryBase: - def __init__(self): - self._params = {} - - def __getitem__(self, name): - return self._params.get(name, None) - def __setitem__(self, name, value): - self._params[name] = value +class MeasurementProtocol: + def __init__(self, events=None): + self.client_id = app.get_cid() + self._events = events or [] + self._user_properties = {} - def __delitem__(self, name): - if name in self._params: - del self._params[name] + self.set_user_property("systype", util.get_systype()) + created_at = app.get_state_item("created_at", None) + if created_at: + self.set_user_property("created_at", int(created_at)) - def send(self, hittype): - raise NotImplementedError() + @staticmethod + def event_to_dict(name, params, timestamp=None): + event = {"name": name, "params": params} + if timestamp is not None: + event["timestamp"] = timestamp + return event + def set_user_property(self, name, value): + self._user_properties[name] = value -class MeasurementProtocol(TelemetryBase): - TID = "UA-1768265-9" - PARAMS_MAP = { - "screen_name": "cd", - "event_category": "ec", - "event_action": "ea", - "event_label": "el", - "event_value": "ev", - } - - def __init__(self): - super().__init__() - self["v"] = 1 - self["tid"] = self.TID - self["cid"] = app.get_cid() - - try: - self["sr"] = "%dx%d" % shutil.get_terminal_size() - except ValueError: - pass + def add_event(self, name, params): + self._events.append(self.event_to_dict(name, params)) - self._prefill_screen_name() - self._prefill_appinfo() - self._prefill_sysargs() - self._prefill_custom_data() - - def __getitem__(self, name): - if name in self.PARAMS_MAP: - name = self.PARAMS_MAP[name] - return super().__getitem__(name) - - def __setitem__(self, name, value): - if name in self.PARAMS_MAP: - name = self.PARAMS_MAP[name] - super().__setitem__(name, value) - - def _prefill_appinfo(self): - self["av"] = __version__ - self["an"] = app.get_user_agent() - - def _prefill_sysargs(self): - args = [] - for arg in sys.argv[1:]: - arg = str(arg) - if arg == "account": # ignore account cmd which can contain username - return - if any(("@" in arg, "/" in arg, "\\" in arg)): - arg = "***" - args.append(arg.lower()) - self["cd3"] = " ".join(args) - - def _prefill_custom_data(self): - caller_id = str(app.get_session_var("caller_id")) - self["cd1"] = util.get_systype() - self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 - if caller_id: - self["cd5"] = caller_id.lower() - - def _prefill_screen_name(self): - def _first_arg_from_list(args_, list_): - for _arg in args_: - if _arg in list_: - return _arg - return None - - args = [] - for arg in PlatformioCLI.leftover_args: - if not isinstance(arg, string_types): - arg = str(arg) - if not arg.startswith("-"): - args.append(arg.lower()) - if not args: - return - - cmd_path = args[:1] - if args[0] in ( - "access", - "account", - "device", - "org", - "package", - "pkg", - "platform", - "project", - "settings", - "system", - "team", - ): - cmd_path = args[:2] - if args[0] == "lib" and len(args) > 1: - lib_subcmds = ( - "builtin", - "install", - "list", - "register", - "search", - "show", - "stats", - "uninstall", - "update", - ) - sub_cmd = _first_arg_from_list(args[1:], lib_subcmds) - if sub_cmd: - cmd_path.append(sub_cmd) - elif args[0] == "remote" and len(args) > 1: - remote_subcmds = ("agent", "device", "run", "test") - sub_cmd = _first_arg_from_list(args[1:], remote_subcmds) - if sub_cmd: - cmd_path.append(sub_cmd) - if len(args) > 2 and sub_cmd in ("agent", "device"): - remote2_subcmds = ("list", "start", "monitor") - sub_cmd = _first_arg_from_list(args[2:], remote2_subcmds) - if sub_cmd: - cmd_path.append(sub_cmd) - self["screen_name"] = " ".join([p.title() for p in cmd_path]) - - def _ignore_hit(self): - if not app.get_setting("enable_telemetry"): - return True - if self["ea"] in ("Idedata", "__Idedata"): - return True - return False - - def send(self, hittype): - if self._ignore_hit(): - return - self["t"] = hittype - # correct queue time - if "qt" in self._params and isinstance(self["qt"], float): - self["qt"] = int((time() - self["qt"]) * 1000) - MPDataPusher().push(self._params) + def to_payload(self): + return { + "client_id": self.client_id, + "user_properties": self._user_properties, + "events": self._events, + } @util.singleton -class MPDataPusher: - MAX_WORKERS = 5 - +class TelemetryLogger: def __init__(self): - self._queue = queue.LifoQueue() - self._failedque = deque() - self._http_session = HTTPSession() - self._http_offline = False - self._workers = [] - - def push(self, item): - # if network is off-line - if self._http_offline: - if "qt" not in item: - item["qt"] = time() - self._failedque.append(item) - return + self._events = deque() - self._queue.put(item) - self._tune_workers() + self._sender_thread = None + self._sender_queue = queue.Queue() + self._sender_terminated = False - def in_wait(self): - return self._queue.unfinished_tasks + self._http_session = HTTPSession() + self._http_offline = False - def get_items(self): - items = list(self._failedque) - try: - while True: - items.append(self._queue.get_nowait()) - except queue.Empty: - pass - return items + def close(self): + self._http_session.close() - def _tune_workers(self): - for i, w in enumerate(self._workers): - if not w.is_alive(): - del self._workers[i] + def log_event(self, name, params, timestamp=None, instant_sending=False): + if not app.get_setting("enable_telemetry") or app.get_session_var( + "pause_telemetry" + ): + return None + timestamp = timestamp or int(time.time()) + self._events.append( + MeasurementProtocol.event_to_dict(name, params, timestamp=timestamp) + ) + if self._http_offline: # if network is off-line + return False + if instant_sending: + self.send() + return True - need_nums = min(self._queue.qsize(), self.MAX_WORKERS) - active_nums = len(self._workers) - if need_nums <= active_nums: + def send(self): + if not self._events or self._sender_terminated: return + if not self._sender_thread: + self._sender_thread = threading.Thread( + target=self._sender_worker, daemon=True + ) + self._sender_thread.start() + while self._events: + events = [] + try: + while len(events) < SEND_MAX_EVENTS: + events.append(self._events.popleft()) + except IndexError: + pass + self._sender_queue.put(events) - for i in range(need_nums - active_nums): - t = threading.Thread(target=self._worker) - t.daemon = True - t.start() - self._workers.append(t) - - def _worker(self): + def _sender_worker(self): while True: + if self._sender_terminated: + return try: - item = self._queue.get() - _item = item.copy() - if "qt" not in _item: - _item["qt"] = time() - self._failedque.append(_item) - if self._send_data(item): - self._failedque.remove(_item) - self._queue.task_done() - except: # pylint: disable=W0702 + events = self._sender_queue.get() + if not self._commit_events(events): + self._events.extend(events) + self._sender_queue.task_done() + except (queue.Empty, ValueError): pass - def _send_data(self, data): + def _commit_events(self, events): if self._http_offline: return False + mp = MeasurementProtocol(events) + payload = mp.to_payload() + # print("_commit_payload", payload) try: r = self._http_session.post( - "https://ssl.google-analytics.com/collect", - data=data, - timeout=1, + "https://telemetry.platformio.org/collect", + json=payload, + timeout=(2, 5), # connect, read ) r.raise_for_status() return True except requests.exceptions.HTTPError as exc: # skip Bad Request - if 400 >= exc.response.status_code < 500: + if exc.response.status_code >= 400 and exc.response.status_code < 500: return True except: # pylint: disable=bare-except pass self._http_offline = True return False + def terminate_sender(self): + self._sender_terminated = True -def on_command(): - resend_backuped_reports() + def is_sending(self): + return self._sender_queue.unfinished_tasks - mp = MeasurementProtocol() - mp.send("screenview") + def get_unsent_events(self): + result = list(self._events) + try: + while True: + result.extend(self._sender_queue.get_nowait()) + except queue.Empty: + pass + return result - if is_ci(): - measure_ci() +def log_event(name, params, instant_sending=False): + TelemetryLogger().log_event(name, params, instant_sending=instant_sending) -def on_exception(e): - skip_conditions = [ - isinstance(e, cls) - for cls in ( - IOError, - exception.ReturnErrorCode, - exception.UserSideException, - ) - ] - if any(skip_conditions): - return - is_fatal = any( - [ - not isinstance(e, exception.PlatformioException), - "Error" in e.__class__.__name__, - ] - ) - description = "%s: %s" % ( - type(e).__name__, - " ".join(reversed(format_exc().split("\n"))) if is_fatal else str(e), - ) - send_exception(description, is_fatal) + +def on_cmd_start(cmd_ctx): + process_postponed_logs() + log_command(cmd_ctx) + + +def on_exit(): + TelemetryLogger().send() + + +def log_command(ctx): + params = { + "path_args": PlatformioCLI.reveal_cmd_path_args(ctx), + } + if is_ci(): + params["ci_actor"] = resolve_ci_actor() or "Unknown" + log_event("cmd_run", params) -def measure_ci(): - event = {"category": "CI", "action": "NoName", "label": None} +def resolve_ci_actor(): known_cis = ( "GITHUB_ACTIONS", "TRAVIS", @@ -318,123 +200,184 @@ def measure_ci(): ) for name in known_cis: if os.getenv(name, "false").lower() == "true": - event["action"] = name - break - send_event(**event) + return name + return None -def dump_run_environment(options): +def dump_project_env_params(config, env, platform): non_sensitive_data = [ "platform", - "platform_packages", "framework", "board", "upload_protocol", "check_tool", "debug_tool", - "monitor_filters", "test_framework", ] - safe_options = {k: v for k, v in options.items() if k in non_sensitive_data} - if is_platformio_project(os.getcwd()): - phash = hashlib.sha1(hashlib_encode_data(app.get_cid())) - safe_options["pid"] = phash.hexdigest() - return json.dumps(safe_options, sort_keys=True, ensure_ascii=False) - - -def send_run_environment(options, targets): - send_event( - "Env", - " ".join([t.title() for t in targets or ["run"]]), - dump_run_environment(options), + section = f"env:{env}" + params = { + option: config.get(section, option) + for option in non_sensitive_data + if config.has_option(section, option) + } + params["pid"] = hashlib.sha1(hashlib_encode_data(config.path)).hexdigest() + params["platform_name"] = platform.name + params["platform_version"] = platform.version + return params + + +def log_platform_run(platform, project_config, project_env, targets=None): + params = dump_project_env_params(project_config, project_env, platform) + if targets: + params["targets"] = targets + log_event("platform_run", params, instant_sending=True) + + +def log_exception(exc): + skip_conditions = [ + isinstance(exc, cls) + for cls in ( + IOError, + exception.ReturnErrorCode, + exception.UserSideException, + ) + ] + skip_conditions.append(not isinstance(exc, Exception)) + if any(skip_conditions): + return + is_fatal = any( + [ + not isinstance(exc, exception.PlatformioException), + "Error" in exc.__class__.__name__, + ] ) + def _strip_module_path(match): + module_path = match.group(1).replace(fs.get_source_dir() + os.sep, "") + sp_folder_name = "site-packages" + sp_pos = module_path.find(sp_folder_name) + if sp_pos != -1: + module_path = module_path[sp_pos + len(sp_folder_name) + 1 :] + module_path = fs.to_unix_path(module_path) + return f'File "{module_path}",' + + trace = re.sub( + r'File "([^"]+)",', + _strip_module_path, + traceback.format_exc(), + flags=re.MULTILINE, + ) -def send_event(category, action, label=None, value=None, screen_name=None): - mp = MeasurementProtocol() - mp["event_category"] = category[:150] - mp["event_action"] = action[:500] - if label: - mp["event_label"] = label[:500] - if value: - mp["event_value"] = int(value) - if screen_name: - mp["screen_name"] = screen_name[:2048] - mp.send("event") + params = { + "name": exc.__class__.__name__, + "description": str(exc), + "traceback": trace, + "cmd_args": sys.argv[1:], + "is_fatal": is_fatal, + } + log_event("exception", params) -def send_exception(description, is_fatal=False): +def log_debug_started(debug_config: DebugConfigBase): + log_event( + "debug_started", + dump_project_env_params( + debug_config.project_config, debug_config.env_name, debug_config.platform + ), + ) + + +def log_debug_exception(exc, debug_config: DebugConfigBase): # cleanup sensitive information, such as paths - description = description.replace("Traceback (most recent call last):", "") - description = description.replace("\\", "/") + description = fs.to_unix_path(str(exc)) description = re.sub( r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)', lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]), description, re.I | re.M, ) - description = re.sub(r"\s+", " ", description, flags=re.M) - - mp = MeasurementProtocol() - mp["exd"] = description[:8192].strip() - mp["exf"] = 1 if is_fatal else 0 - mp.send("exception") + params = { + "name": exc.__class__.__name__, + "description": description.strip(), + } + params.update( + dump_project_env_params( + debug_config.project_config, debug_config.env_name, debug_config.platform + ) + ) + log_event("debug_exception", params) @atexit.register def _finalize(): timeout = 1000 # msec elapsed = 0 + telemetry = TelemetryLogger() + telemetry.terminate_sender() try: while elapsed < timeout: - if not MPDataPusher().in_wait(): + if not telemetry.is_sending(): break - sleep(0.2) + time.sleep(0.2) elapsed += 200 - backup_reports(MPDataPusher().get_items()) except KeyboardInterrupt: pass + postpone_events(telemetry.get_unsent_events()) + telemetry.close() -def backup_reports(items): - if not items: - return - - KEEP_MAX_REPORTS = 100 - tm = app.get_state_item("telemetry", {}) - if "backup" not in tm: - tm["backup"] = [] - - for params in items: - # skip static options - for key in list(params.keys()): - if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): - del params[key] - - # store time in UNIX format - if "qt" not in params: - params["qt"] = time() - elif not isinstance(params["qt"], float): - params["qt"] = time() - (params["qt"] / 1000) +def load_postponed_events(): + state_path = app.resolve_state_path( + "cache_dir", "telemetry.json", ensure_dir_exists=False + ) + if not os.path.isfile(state_path): + return [] + with app.State(state_path) as state: + return state.get("events", []) - tm["backup"].append(params) - tm["backup"] = tm["backup"][KEEP_MAX_REPORTS * -1 :] - app.set_state_item("telemetry", tm) +def save_postponed_events(events): + state_path = app.resolve_state_path("cache_dir", "telemetry.json") + if not events: + try: + if os.path.isfile(state_path): + os.remove(state_path) + except: # pylint: disable=bare-except + pass + return None + with app.State(state_path, lock=True) as state: + state["events"] = events + state.modified = True + return True -def resend_backuped_reports(): - tm = app.get_state_item("telemetry", {}) - if "backup" not in tm or not tm["backup"]: - return False +def postpone_events(events): + if not events: + return None + postponed_events = load_postponed_events() or [] + timestamp = int(time.time()) + for event in events: + if "timestamp" not in event: + event["timestamp"] = timestamp + postponed_events.append(event) + save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :]) + return True - for report in tm["backup"]: - mp = MeasurementProtocol() - for key, value in report.items(): - mp[key] = value - mp.send(report["t"]) - # clean - tm["backup"] = [] - app.set_state_item("telemetry", tm) +def process_postponed_logs(): + if not ensure_internet_on(): + return None + events = load_postponed_events() + if not events: + return None + save_postponed_events([]) # clean + telemetry = TelemetryLogger() + for event in events: + if set(["name", "params", "timestamp"]) <= set(event.keys()): + telemetry.log_event( + event["name"], + event["params"], + timestamp=event["timestamp"], + instant_sending=False, + ) return True diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index 25fea0ab62..935a1328c9 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -55,8 +55,8 @@ class UnityTestRunner(TestRunnerBase): void unityOutputStart(unsigned long); void unityOutputChar(unsigned int); -void unityOutputFlush(); -void unityOutputComplete(); +void unityOutputFlush(void); +void unityOutputComplete(void); #define UNITY_OUTPUT_START() unityOutputStart((unsigned long) $baudrate) #define UNITY_OUTPUT_CHAR(c) unityOutputChar(c) @@ -246,18 +246,20 @@ def generate_unity_extras(self, dst_dir): unity_h = dst_dir / "unity_config.h" if not unity_h.is_file(): unity_h.write_text( - string.Template(self.UNITY_CONFIG_H).substitute( - baudrate=self.get_test_speed() - ), + string.Template(self.UNITY_CONFIG_H) + .substitute(baudrate=self.get_test_speed()) + .strip() + + "\n", encoding="utf8", ) framework_config = self.get_unity_framework_config() unity_c = dst_dir / ("unity_config.%s" % framework_config.get("language", "c")) if not unity_c.is_file(): unity_c.write_text( - string.Template(self.UNITY_CONFIG_C).substitute( - framework_config_code=framework_config["code"] - ), + string.Template(self.UNITY_CONFIG_C) + .substitute(framework_config_code=framework_config["code"]) + .strip() + + "\n", encoding="utf8", ) diff --git a/platformio/util.py b/platformio/util.py index c981384ad8..004ac5c9f1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import functools import math import platform import re import shutil import time -from datetime import datetime import click @@ -169,8 +169,8 @@ def items_in_list(needle, haystack): def parse_datetime(datestr): if "T" in datestr and "Z" in datestr: - return datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") - return datetime.strptime(datestr) + return datetime.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") + return datetime.datetime.strptime(datestr) def merge_dicts(d1, d2, path=None): diff --git a/setup.py b/setup.py index 42427a85c0..f37da675a6 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys from setuptools import find_packages, setup from platformio import ( @@ -25,31 +24,48 @@ __version__, ) -PY36 = sys.version_info < (3, 7) - +env_marker_below_37 = "python_version < '3.7'" +env_marker_gte_37 = "python_version >= '3.7'" minimal_requirements = [ "bottle==0.12.*", - "click%s" % ("==8.0.4" if PY36 else ">=8.0.4,<9"), + "click==8.0.4; " + env_marker_below_37, + "click==8.1.*; " + env_marker_gte_37, "colorama", - "marshmallow==%s" % ("3.14.1" if PY36 else "3.*"), - "pyelftools>=0.27,<1", + "marshmallow==3.14.1; " + env_marker_below_37, + "marshmallow==3.19.*; " + env_marker_gte_37, + "pyelftools==0.29", "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" "requests==2.*", - "urllib3<2", # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ - "requests==%s" % ("2.27.1" if PY36 else "2.*"), "semantic_version==2.10.*", - "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"), + "tabulate==0.*", ] home_requirements = [ - "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), - "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"), - "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), - "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), + "aiofiles>=0.8.0", + "ajsonrpc==1.2.*", + "starlette==0.19.1; " + env_marker_below_37, + "starlette==0.28.*; " + env_marker_gte_37, + "uvicorn==0.16.0; " + env_marker_below_37, + "uvicorn==0.22.*; " + env_marker_gte_37, + "wsproto==1.0.0; " + env_marker_below_37, + "wsproto==1.2.*; " + env_marker_gte_37, ] +# issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ +try: + import ssl + + if ssl.OPENSSL_VERSION.startswith("OpenSSL ") and ssl.OPENSSL_VERSION_INFO < ( + 1, + 1, + 1, + ): + minimal_requirements.append("urllib3<2") +except ImportError: + pass + + setup( name=__title__, version=__version__, @@ -65,11 +81,11 @@ package_data={ "platformio": [ "assets/system/99-platformio-udev.rules", - "assets/templates/ide-projects/*/*.tpl", - "assets/templates/ide-projects/*/.*.tpl", # include hidden files - "assets/templates/ide-projects/*/.*/*.tpl", # include hidden folders - "assets/templates/ide-projects/*/*/*.tpl", # NetBeans - "assets/templates/ide-projects/*/*/*/*.tpl", # NetBeans + "project/integration/tpls/*/*.tpl", + "project/integration/tpls/*/.*.tpl", # include hidden files + "project/integration/tpls/*/.*/*.tpl", # include hidden folders + "project/integration/tpls/*/*/*.tpl", # NetBeans + "project/integration/tpls/*/*/*/*.tpl", # NetBeans ] }, entry_points={ diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 61ccde0da7..ea995c6810 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -469,7 +469,7 @@ def test_custom_project_tools( project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) - spec = "platformio/tool-openocd" + spec = "platformio/tool-openocd @ ^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], @@ -503,7 +503,7 @@ def test_custom_project_tools( # check saved deps assert config.get("env:devkit", "platform_packages") == [ - "platformio/tool-openocd@^2.1100.211028", + "platformio/tool-openocd@^2", ] # install tool without saving to config @@ -518,7 +518,7 @@ def test_custom_project_tools( PackageSpec("tool-openocd@2.1100.211028"), ] assert config.get("env:devkit", "platform_packages") == [ - "platformio/tool-openocd@^2.1100.211028", + "platformio/tool-openocd@^2", ] # unknown tool diff --git a/tests/commands/pkg/test_uninstall.py b/tests/commands/pkg/test_uninstall.py index 1dae981c7a..d776b142e8 100644 --- a/tests/commands/pkg/test_uninstall.py +++ b/tests/commands/pkg/test_uninstall.py @@ -313,7 +313,7 @@ def test_custom_project_tools( project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) - spec = "platformio/tool-openocd" + spec = "platformio/tool-openocd@^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], @@ -329,7 +329,7 @@ def test_custom_project_tools( assert not os.path.exists(config.get("platformio", "platforms_dir")) # check saved deps assert config.get("env:devkit", "platform_packages") == [ - "platformio/tool-openocd@^2.1100.211028", + "platformio/tool-openocd@^2", ] # uninstall result = clirunner.invoke( diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index b3d9f0133a..80ef0d9194 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -62,7 +62,7 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): - result = clirunner.invoke(project_init_cmd, ["--ide", "atom"]) + result = clirunner.invoke(project_init_cmd, ["--ide", "vscode"]) assert result.exit_code != 0 assert isinstance(result.exception, ProjectEnvsNotAvailableError) diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index cb9e3d859e..de13a5a8fc 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -23,6 +23,10 @@ def test_generic_build(clirunner, validate_cliresult, tmpdir): ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), ("-Iextra_inc", "-Iextra_inc"), + ( + "-include $PROJECT_DIR/lib/component/component-forced-include.h", + "component-forced-include.h", + ), ] tmpdir.join("platformio.ini").write( @@ -95,6 +99,10 @@ def post_prog_action(source, target, env): #error "POST_SCRIPT_MACRO" #endif +#ifndef I_AM_FORCED_COMPONENT_INCLUDE +#error "I_AM_FORCED_COMPONENT_INCLUDE" +#endif + #ifdef COMMENTED_MACRO #error "COMMENTED_MACRO" #endif @@ -124,6 +132,11 @@ def post_prog_action(source, target, env): void dummy(void ) {}; """ ) + component_dir.join("component-forced-include.h").write( + """ +#define I_AM_FORCED_COMPONENT_INCLUDE + """ + ) result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py index 349e9c7920..d37cb71b3f 100644 --- a/tests/misc/test_misc.py +++ b/tests/misc/test_misc.py @@ -36,13 +36,13 @@ def test_ping_internet_ips(): def test_api_internet_offline(without_internet, isolated_pio_core): regclient = RegistryClient() with pytest.raises(http.InternetConnectionError): - regclient.fetch_json_data("get", "/v2/stats") + regclient.fetch_json_data("get", "/v3/search") def test_api_cache(monkeypatch, isolated_pio_core): regclient = RegistryClient() - api_kwargs = {"method": "get", "path": "/v2/stats", "x_cache_valid": "10s"} + api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"} result = regclient.fetch_json_data(**api_kwargs) - assert result and "boards" in result + assert result and "total" in result monkeypatch.setattr(http, "_internet_on", lambda: False) assert regclient.fetch_json_data(**api_kwargs) == result diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index a3279e051d..bcc7e00a6f 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -28,7 +28,7 @@ def test_library_json_parser(): contents = """ { "name": "TestPackage", - "keywords": "kw1, KW2, kw3, KW2", + "keywords": "kw1, KW2, kw3, KW2, kw 4, kw_5, kw-6", "headers": "include1.h, Include2.hpp", "platforms": ["atmelavr", "espressif"], "repository": { @@ -62,7 +62,7 @@ def test_library_json_parser(): "url": "https://github.com/username/repo.git", }, "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, - "keywords": ["kw1", "kw2", "kw3"], + "keywords": ["kw1", "kw2", "kw3", "kw 4", "kw_5", "kw-6"], "headers": ["include1.h", "Include2.hpp"], "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, diff --git a/tests/project/test_config.py b/tests/project/test_config.py index f559502ed3..6b74285538 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -684,3 +684,34 @@ def test_invalid_env_names(tmp_path: Path): config = ProjectConfig(str(project_conf)) with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): config.validate() + + +def test_linting_errors(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app1] +lib_use = 1 +broken_line + """ + ) + result = ProjectConfig.lint(str(project_conf)) + assert not result["warnings"] + assert result["errors"] and len(result["errors"]) == 1 + error = result["errors"][0] + assert error["type"] == "ParsingError" + assert error["lineno"] == 4 + + +def test_linting_warnings(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app1] +lib_use = 1 + """ + ) + result = ProjectConfig.lint(str(project_conf)) + assert not result["errors"] + assert result["warnings"] and len(result["warnings"]) == 1 + assert "deprecated" in result["warnings"][0] diff --git a/tox.ini b/tox.ini index 608328066a..f8a15f3bc5 100644 --- a/tox.ini +++ b/tox.ini @@ -54,16 +54,17 @@ commands = [testenv:docs] deps = sphinx - sphinx-rtd-theme==1.2.0 + sphinx-rtd-theme==1.2.2 sphinx-notfound-page sphinx-copybutton restructuredtext-lint +change_dir = docs commands = - sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html + sphinx-build -b html . _build/html [testenv:docslinkcheck] deps = - sphinx - sphinx-rtd-theme + {[testenv:docs]deps} +change_dir = docs commands = - sphinx-build -W -b linkcheck docs docs/_build/html + sphinx-build -b linkcheck . _build