From 148ce1a897d72edfe78bc43f6573bd5b4948488f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 May 2023 18:03:34 +0300 Subject: [PATCH 01/69] Bump version to 6.1.8a1 --- HISTORY.rst | 3 +++ platformio/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3aa710ff90..e92896b941 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,9 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.8 (2023-??-??) +~~~~~~~~~~~~~~~~~~ + 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 7d3c8f8abf..8294a86c4f 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, 7) +VERSION = (6, 1, "8a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 89ffd822750378de0bc01b97d6067c55b7334f1a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 May 2023 18:05:48 +0300 Subject: [PATCH 02/69] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 9a951da496..d490fe6f2f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,9 +6,8 @@ What kind of issue is this? use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support) - [ ] **PlatformIO IDE**. - All issues related to PlatformIO IDE should be reported to appropriate repository: - [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or - [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) + All issues related to PlatformIO IDE should be reported to the + [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository - [ ] **Development Platform or Board**. All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms From 71afa639e2eb4ed4368ab0ec0cd3a2d790e4a297 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 May 2023 17:59:47 +0300 Subject: [PATCH 03/69] Fix tox's "docs" environment // Resolve #4624 --- tox.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 608328066a..1db3d5c6d1 100644 --- a/tox.ini +++ b/tox.ini @@ -58,12 +58,13 @@ deps = sphinx-notfound-page sphinx-copybutton restructuredtext-lint +change_dir = docs commands = - sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html + sphinx-build . _build [testenv:docslinkcheck] deps = - sphinx - sphinx-rtd-theme + {[testenv:docs]deps} +change_dir = docs commands = - sphinx-build -W -b linkcheck docs docs/_build/html + sphinx-build -b linkcheck . _build From 4388cd43212f13ccd81294114f67c0dcb9d3cf96 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 May 2023 18:12:25 +0300 Subject: [PATCH 04/69] Improve "CONTRIBUTING" guide // Resolve #4622 --- CONTRIBUTING.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f2f4188ab..8deafb9c70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,12 +6,13 @@ To get started, si 1. Fork the repository on GitHub 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` 3. Run `pip install tox` -4. Go to the root of project where is located `tox.ini` and run `tox -e py37` +4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`. + You can replace `py39` with your own Python version. For example, `py311` means Python 3.11. 5. Activate current development environment: - * Windows: `.tox\py37\Scripts\activate` - * Bash/ZSH: `source .tox/py37/bin/activate` - * Fish: `source .tox/py37/bin/activate.fish` + * Windows: `.tox\py39\Scripts\activate` + * Bash/ZSH: `source .tox/py39/bin/activate` + * Fish: `source .tox/py39/bin/activate.fish` 6. Make changes to code, documentation, etc. 7. Lint source code `make before-commit` From 4fc6b26db54e6931a96cfa3a863d06361912cd0e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 May 2023 18:46:38 +0300 Subject: [PATCH 05/69] Fix docs builder --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1db3d5c6d1..fe24f9c6a6 100644 --- a/tox.ini +++ b/tox.ini @@ -60,7 +60,7 @@ deps = restructuredtext-lint change_dir = docs commands = - sphinx-build . _build + sphinx-build -b html . _build/html [testenv:docslinkcheck] deps = From e0f839a37228256a843eeae3c47e08878a5d4a7a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 31 May 2023 20:15:08 +0300 Subject: [PATCH 06/69] Update deps --- docs | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 98609771ba..bc8ee6e923 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 98609771ba8f78505adad20e66c6505c262f2650 +Subproject commit bc8ee6e923c7f7f0cef09dc035763e2fa343977e diff --git a/setup.py b/setup.py index 42427a85c0..c463302e61 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ home_requirements = [ "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.27.*"), "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] From 63ca19541f0a61955373f9807d3febd56cf4a69d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 31 May 2023 20:15:36 +0300 Subject: [PATCH 07/69] Enhance the parsing of the platformio.ini to provide comprehensive diagnostic information --- HISTORY.rst | 2 ++ platformio/project/config.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index e92896b941..a29f23d1dc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,8 @@ PlatformIO Core 6 6.1.8 (2023-??-??) ~~~~~~~~~~~~~~~~~~ +* Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information + 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/config.py b/platformio/project/config.py index c2b072ef16..08ab422cb5 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -98,7 +98,7 @@ def read(self, path, parse_extra=True): try: self._parser.read(path, "utf-8") except configparser.Error as exc: - raise exception.InvalidProjectConfError(path, str(exc)) + raise exception.InvalidProjectConfError(path, str(exc)) from exc if not parse_extra: return From eeb0116f280422f1653987835f875a60472b5933 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 1 Jun 2023 18:41:50 +0300 Subject: [PATCH 08/69] Skip exception when fetching SSL settings --- platformio/http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/http.py b/platformio/http.py index 48235f7422..5f3021bf63 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -50,7 +50,10 @@ def __init__(self, *args, **kwargs): self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None super().__init__(*args, **kwargs) self.headers.update({"User-Agent": app.get_user_agent()}) - self.verify = app.get_setting("enable_proxy_strict_ssl") + try: + self.verify = app.get_setting("enable_proxy_strict_ssl") + except PlatformioException: + self.verify = True def request( # pylint: disable=signature-differs,arguments-differ self, method, url, *args, **kwargs From 6ea7ded483220b2dff87c0d63b2624fcbd69b2fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jun 2023 18:15:30 +0300 Subject: [PATCH 09/69] Run tests against API v3 --- tests/misc/test_misc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py index 349e9c7920..d37cb71b3f 100644 --- a/tests/misc/test_misc.py +++ b/tests/misc/test_misc.py @@ -36,13 +36,13 @@ def test_ping_internet_ips(): def test_api_internet_offline(without_internet, isolated_pio_core): regclient = RegistryClient() with pytest.raises(http.InternetConnectionError): - regclient.fetch_json_data("get", "/v2/stats") + regclient.fetch_json_data("get", "/v3/search") def test_api_cache(monkeypatch, isolated_pio_core): regclient = RegistryClient() - api_kwargs = {"method": "get", "path": "/v2/stats", "x_cache_valid": "10s"} + api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"} result = regclient.fetch_json_data(**api_kwargs) - assert result and "boards" in result + assert result and "total" in result monkeypatch.setattr(http, "_internet_on", lambda: False) assert regclient.fetch_json_data(**api_kwargs) == result From cb65bdf22f2013cee5352a6d8f5ecc982fd9a1e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jun 2023 18:24:42 +0300 Subject: [PATCH 10/69] Enhance user privacy protection through refined telemetry implementation --- platformio/app.py | 14 +- platformio/cli.py | 15 + platformio/debug/process/gdb.py | 20 +- platformio/home/rpc/handlers/app.py | 17 +- platformio/maintenance.py | 51 ++-- platformio/platform/_run.py | 11 +- platformio/platform/board.py | 3 +- platformio/telemetry.py | 433 +++++++++++----------------- platformio/util.py | 10 + 9 files changed, 253 insertions(+), 321 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 48e90b97f9..c01e13c39b 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -18,6 +18,7 @@ import os import platform import socket +import time import uuid from platformio import __version__, exception, fs, proc @@ -71,15 +72,19 @@ def projects_dir_validate(projects_dir): } +def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True): + state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir) + if ensure_dir_exists and not os.path.isdir(state_dir): + os.makedirs(state_dir) + return os.path.join(state_dir, file_name) + + class State: def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: - core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") - if not os.path.isdir(core_dir): - os.makedirs(core_dir) - self.path = os.path.join(core_dir, "appstate.json") + self.path = resolve_state_path("core_dir", "appstate.json") self._storage = {} self._lockfile = None self.modified = False @@ -248,6 +253,7 @@ def get_cid(): cid = str(cid) if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) + set_state_item("created_at", int(time.time())) return cid diff --git a/platformio/cli.py b/platformio/cli.py index 41aab522e1..a2a707ad70 100644 --- a/platformio/cli.py +++ b/platformio/cli.py @@ -63,6 +63,21 @@ def in_silence(): ] ) + @classmethod + def reveal_cmd_path_args(cls, ctx): + result = [] + group = ctx.command + args = cls.leftover_args[::] + while args: + cmd_name = args.pop(0) + next_group = group.get_command(ctx, cmd_name) + if next_group: + group = next_group + result.append(cmd_name) + if not hasattr(group, "get_command"): + break + return result + def invoke(self, ctx): PlatformioCLI.leftover_args = ctx.args if hasattr(ctx, "protected_args"): diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index ce7e82c8bf..2a1f2f7255 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -130,11 +130,7 @@ def stdout_data_received(self, data): self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: - telemetry.send_event( - "Debug", - "Started", - telemetry.dump_run_environment(self.debug_config.env_options), - ) + telemetry.log_debug_started(self.debug_config) self._auto_exec_continue() def console_log(self, msg): @@ -180,13 +176,11 @@ def _handle_error(self, data): ): return - last_erros = self._errors_buffer.decode() - last_erros = " ".join(reversed(last_erros.split("\n"))) - last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) - - err = "%s -> %s" % ( - telemetry.dump_run_environment(self.debug_config.env_options), - last_erros, + last_errors = self._errors_buffer.decode() + last_errors = " ".join(reversed(last_errors.split("\n"))) + last_errors = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_errors, flags=re.M) + telemetry.log_debug_exception( + "DebugInitError: %s" % last_errors, self.debug_config ) - telemetry.send_exception("DebugInitError: %s" % err) + self.transport.close() diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index 9c79e3147c..4b6195e4b1 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from pathlib import Path from platformio import __version__, app, fs, util from platformio.home.rpc.handlers.base import BaseRPCHandler -from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project @@ -32,16 +30,11 @@ class AppRPC(BaseRPCHandler): "projectsDir", ] - @staticmethod - def get_state_path(): - core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") - if not os.path.isdir(core_dir): - os.makedirs(core_dir) - return os.path.join(core_dir, "homestate.json") - @staticmethod def load_state(): - with app.State(AppRPC.get_state_path(), lock=True) as state: + with app.State( + app.resolve_state_path("core_dir", "homestate.json"), lock=True + ) as state: storage = state.get("storage", {}) # base data @@ -81,7 +74,9 @@ def get_state(): @staticmethod def save_state(state): - with app.State(AppRPC.get_state_path(), lock=True) as s: + with app.State( + app.resolve_state_path("core_dir", "homestate.json"), lock=True + ) as s: s.clear() s.update(state) storage = s.get("storage", {}) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 3110b2c192..c7a3d3b753 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,8 +25,6 @@ from platformio.commands.upgrade import get_latest_version from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages -from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageSpec from platformio.package.version import pepver_to_semver from platformio.system.prune import calculate_unnecessary_system_data @@ -34,7 +32,8 @@ def on_platformio_start(ctx, caller): app.set_session_var("command_ctx", ctx) set_caller(caller) - telemetry.on_command() + telemetry.log_command(ctx) + telemetry.resend_postponed_logs() if PlatformioCLI.in_silence(): return @@ -60,8 +59,8 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument ) -def on_platformio_exception(e): - telemetry.on_exception(e) +def on_platformio_exception(exc): + telemetry.log_exception(exc) def set_caller(caller=None): @@ -83,7 +82,7 @@ def __init__(self, from_version, to_version): self.to_version = pepver_to_semver(to_version) self._upgraders = [ - (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), + (semantic_version.Version("6.1.8-a.1"), self._appstate_migration), ] def run(self, ctx): @@ -99,21 +98,22 @@ def run(self, ctx): return all(result) @staticmethod - def _update_pkg_metadata(_): - pm = ToolPackageManager() - for pkg in pm.get_installed(): - if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: - continue - result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) - if len(result) != 1: - continue - result = result[0] - pkg.metadata.spec = PackageSpec( - id=result["id"], - owner=result["owner"]["username"], - name=result["name"], + def _appstate_migration(_): + state_path = app.resolve_state_path("core_dir", "appstate.json") + if not os.path.isfile(state_path): + return True + app.delete_state_item("telemetry") + created_at = app.get_state_item("created_at", None) + if not created_at: + state_stat = os.stat(state_path) + app.set_state_item( + "created_at", + int( + state_stat.st_birthtime + if hasattr(state_stat, "st_birthtime") + else state_stat.st_ctime + ), ) - pkg.dump_meta() return True @@ -154,10 +154,13 @@ def after_upgrade(ctx): "PlatformIO has been successfully upgraded to %s!\n" % __version__, fg="green", ) - telemetry.send_event( - category="Auto", - action="Upgrade", - label="%s > %s" % (last_version, __version__), + telemetry.log_event( + "pio_upgrade_core", + { + "label": "%s > %s" % (last_version, __version__), + "from_version": last_version, + "to_version": __version__, + }, ) # PlatformIO banner diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index dfd7c40675..6a29d955eb 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -52,7 +52,6 @@ def run( # pylint: disable=too-many-arguments self.ensure_engine_compatible() self.configure_project_packages(variables["pioenv"], targets) - self._report_non_sensitive_data(variables["pioenv"], targets) self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") @@ -64,20 +63,12 @@ def run( # pylint: disable=too-many-arguments if not os.path.isfile(variables["build_script"]): raise BuildScriptNotFound(variables["build_script"]) + telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) result = self._run_scons(variables, targets, jobs) assert "returncode" in result return result - def _report_non_sensitive_data(self, env, targets): - options = self.config.items(env=env, as_dict=True) - options["platform_packages"] = [ - dict(name=item["name"], version=item["version"]) - for item in self.dump_used_packages() - ] - options["platform"] = {"name": self.name, "version": self.version} - telemetry.send_run_environment(options, targets) - def _run_scons(self, variables, targets, jobs): scons_dir = get_core_package_dir("tool-scons") args = [ diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 73a3ebd778..4cd102d1b2 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -14,7 +14,7 @@ import os -from platformio import fs, telemetry, util +from platformio import fs, util from platformio.compat import MISSING from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError from platformio.exception import UserSideException @@ -119,7 +119,6 @@ def get_debug_tool_name(self, custom=None): if tool_name == "custom": return tool_name if not debug_tools: - telemetry.send_event("Debug", "Request", self.id) raise DebugSupportError(self._manifest["name"]) if tool_name: if tool_name in debug_tools: diff --git a/platformio/telemetry.py b/platformio/telemetry.py index e9cbcd583d..56aa173ada 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -14,12 +14,10 @@ import atexit import hashlib -import json import os +import platform as python_platform import queue import re -import shutil -import sys import threading from collections import deque from time import sleep, time @@ -27,167 +25,50 @@ import requests -from platformio import __version__, app, exception, util +from platformio import __title__, __version__, app, exception, util from platformio.cli import PlatformioCLI -from platformio.compat import hashlib_encode_data, string_types -from platformio.http import HTTPSession +from platformio.compat import hashlib_encode_data +from platformio.debug.config.base import DebugConfigBase +from platformio.http import HTTPSession, ensure_internet_on from platformio.proc import is_ci, is_container -from platformio.project.helpers import is_platformio_project +KEEP_MAX_REPORTS = 100 +SEND_MAX_EVENTS = 25 -class TelemetryBase: - def __init__(self): - self._params = {} - - def __getitem__(self, name): - return self._params.get(name, None) - - def __setitem__(self, name, value): - self._params[name] = value - - def __delitem__(self, name): - if name in self._params: - del self._params[name] - - def send(self, hittype): - raise NotImplementedError() - - -class MeasurementProtocol(TelemetryBase): - TID = "UA-1768265-9" - PARAMS_MAP = { - "screen_name": "cd", - "event_category": "ec", - "event_action": "ea", - "event_label": "el", - "event_value": "ev", - } +class MeasurementProtocol: def __init__(self): - super().__init__() - self["v"] = 1 - self["tid"] = self.TID - self["cid"] = app.get_cid() - - try: - self["sr"] = "%dx%d" % shutil.get_terminal_size() - except ValueError: - pass + self._user_properties = {} + self._events = [] - self._prefill_screen_name() - self._prefill_appinfo() - self._prefill_sysargs() - self._prefill_custom_data() - - def __getitem__(self, name): - if name in self.PARAMS_MAP: - name = self.PARAMS_MAP[name] - return super().__getitem__(name) - - def __setitem__(self, name, value): - if name in self.PARAMS_MAP: - name = self.PARAMS_MAP[name] - super().__setitem__(name, value) - - def _prefill_appinfo(self): - self["av"] = __version__ - self["an"] = app.get_user_agent() - - def _prefill_sysargs(self): - args = [] - for arg in sys.argv[1:]: - arg = str(arg) - if arg == "account": # ignore account cmd which can contain username - return - if any(("@" in arg, "/" in arg, "\\" in arg)): - arg = "***" - args.append(arg.lower()) - self["cd3"] = " ".join(args) - - def _prefill_custom_data(self): - caller_id = str(app.get_session_var("caller_id")) - self["cd1"] = util.get_systype() - self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 + caller_id = app.get_session_var("caller_id") if caller_id: - self["cd5"] = caller_id.lower() + self.set_user_property("pio_caller_id", caller_id) + self.set_user_property("pio_core_version", __version__) + self.set_user_property( + "pio_human_actor", int(bool(caller_id or not (is_ci() or is_container()))) + ) + self.set_user_property("pio_systype", util.get_systype()) + created_at = app.get_state_item("created_at", None) + if created_at: + self.set_user_property("pio_created_at", int(created_at)) - def _prefill_screen_name(self): - def _first_arg_from_list(args_, list_): - for _arg in args_: - if _arg in list_: - return _arg - return None + def set_user_property(self, name, value): + self._user_properties[name] = {"value": value} - args = [] - for arg in PlatformioCLI.leftover_args: - if not isinstance(arg, string_types): - arg = str(arg) - if not arg.startswith("-"): - args.append(arg.lower()) - if not args: - return + def add_event(self, name, params): + self._events.append({"name": name, "params": params}) - cmd_path = args[:1] - if args[0] in ( - "access", - "account", - "device", - "org", - "package", - "pkg", - "platform", - "project", - "settings", - "system", - "team", - ): - cmd_path = args[:2] - if args[0] == "lib" and len(args) > 1: - lib_subcmds = ( - "builtin", - "install", - "list", - "register", - "search", - "show", - "stats", - "uninstall", - "update", - ) - sub_cmd = _first_arg_from_list(args[1:], lib_subcmds) - if sub_cmd: - cmd_path.append(sub_cmd) - elif args[0] == "remote" and len(args) > 1: - remote_subcmds = ("agent", "device", "run", "test") - sub_cmd = _first_arg_from_list(args[1:], remote_subcmds) - if sub_cmd: - cmd_path.append(sub_cmd) - if len(args) > 2 and sub_cmd in ("agent", "device"): - remote2_subcmds = ("list", "start", "monitor") - sub_cmd = _first_arg_from_list(args[2:], remote2_subcmds) - if sub_cmd: - cmd_path.append(sub_cmd) - self["screen_name"] = " ".join([p.title() for p in cmd_path]) - - def _ignore_hit(self): - if not app.get_setting("enable_telemetry"): - return True - if self["ea"] in ("Idedata", "__Idedata"): - return True - return False - - def send(self, hittype): - if self._ignore_hit(): - return - self["t"] = hittype - # correct queue time - if "qt" in self._params and isinstance(self["qt"], float): - self["qt"] = int((time() - self["qt"]) * 1000) - MPDataPusher().push(self._params) + def to_payload(self): + return { + "client_id": app.get_cid(), + "user_properties": self._user_properties, + "events": self._events, + } @util.singleton -class MPDataPusher: +class TelemetryLogger: MAX_WORKERS = 5 def __init__(self): @@ -197,21 +78,23 @@ def __init__(self): self._http_offline = False self._workers = [] - def push(self, item): + def log(self, payload): + if not app.get_setting("enable_telemetry"): + return None + # if network is off-line if self._http_offline: - if "qt" not in item: - item["qt"] = time() - self._failedque.append(item) - return + self._failedque.append(payload) + return False - self._queue.put(item) + self._queue.put(payload) self._tune_workers() + return True def in_wait(self): return self._queue.unfinished_tasks - def get_items(self): + def get_unprocessed(self): items = list(self._failedque) try: while True: @@ -244,19 +127,27 @@ def _worker(self): if "qt" not in _item: _item["qt"] = time() self._failedque.append(_item) - if self._send_data(item): + if self._send(item): self._failedque.remove(_item) self._queue.task_done() except: # pylint: disable=W0702 pass - def _send_data(self, data): + def _send(self, payload): if self._http_offline: return False try: r = self._http_session.post( - "https://ssl.google-analytics.com/collect", - data=data, + "https://www.google-analytics.com/mp/collect", + params={ + "measurement_id": util.decrypt_message( + __title__, "t5m7rKu6tbqwx8Cw" + ), + "api_secret": util.decrypt_message( + __title__, "48SRy5rmut28ptm7zLjS5sa7tdmhrQ==" + ), + }, + json=payload, timeout=1, ) r.raise_for_status() @@ -271,17 +162,42 @@ def _send_data(self, data): return False -def on_command(): - resend_backuped_reports() - +def log_event(name, params): mp = MeasurementProtocol() - mp.send("screenview") + mp.add_event(name, params) + TelemetryLogger().log(mp.to_payload()) + +def log_command(ctx): + path_args = PlatformioCLI.reveal_cmd_path_args(ctx) + params = { + "page_title": " ".join([arg.title() for arg in path_args]), + "page_path": "/".join(path_args), + "pio_user_agent": app.get_user_agent(), + "pio_python_version": python_platform.python_version(), + } if is_ci(): - measure_ci() + params["ci_actor"] = resolve_ci_actor() or "Unknown" + log_event("page_view", params) -def on_exception(e): +def resolve_ci_actor(): + known_cis = ( + "GITHUB_ACTIONS", + "TRAVIS", + "APPVEYOR", + "GITLAB_CI", + "CIRCLECI", + "SHIPPABLE", + "DRONE", + ) + for name in known_cis: + if os.getenv(name, "false").lower() == "true": + return name + return None + + +def log_exception(e): skip_conditions = [ isinstance(e, cls) for cls in ( @@ -302,68 +218,58 @@ def on_exception(e): type(e).__name__, " ".join(reversed(format_exc().split("\n"))) if is_fatal else str(e), ) - send_exception(description, is_fatal) - - -def measure_ci(): - event = {"category": "CI", "action": "NoName", "label": None} - known_cis = ( - "GITHUB_ACTIONS", - "TRAVIS", - "APPVEYOR", - "GITLAB_CI", - "CIRCLECI", - "SHIPPABLE", - "DRONE", - ) - for name in known_cis: - if os.getenv(name, "false").lower() == "true": - event["action"] = name - break - send_event(**event) + params = { + "description": description[:100].strip(), + "is_fatal": int(is_fatal), + "pio_user_agent": app.get_user_agent(), + } + log_event("pio_exception", params) -def dump_run_environment(options): +def dump_project_env_params(config, env, platform): non_sensitive_data = [ "platform", - "platform_packages", "framework", "board", "upload_protocol", "check_tool", "debug_tool", - "monitor_filters", "test_framework", ] - safe_options = {k: v for k, v in options.items() if k in non_sensitive_data} - if is_platformio_project(os.getcwd()): - phash = hashlib.sha1(hashlib_encode_data(app.get_cid())) - safe_options["pid"] = phash.hexdigest() - return json.dumps(safe_options, sort_keys=True, ensure_ascii=False) - - -def send_run_environment(options, targets): - send_event( - "Env", - " ".join([t.title() for t in targets or ["run"]]), - dump_run_environment(options), + section = f"env:{env}" + params = { + f"pio_{option}": config.get(section, option) + for option in non_sensitive_data + if config.has_option(section, option) + } + params["pio_pid"] = hashlib.sha1(hashlib_encode_data(config.path)).hexdigest() + params["pio_platform_name"] = platform.name + params["pio_platform_version"] = platform.version + params["pio_framework"] = params.get("pio_framework", "__bare_metal__") + # join multi-value options + for key, value in params.items(): + if isinstance(value, list): + params[key] = ", ".join(value) + return params + + +def log_platform_run(platform, project_config, project_env, targets=None): + params = dump_project_env_params(project_config, project_env, platform) + if targets: + params["targets"] = ", ".join(targets) + log_event("pio_platform_run", params) + + +def log_debug_started(debug_config: DebugConfigBase): + log_event( + "pio_debug_started", + dump_project_env_params( + debug_config.project_config, debug_config.env_name, debug_config.platform + ), ) -def send_event(category, action, label=None, value=None, screen_name=None): - mp = MeasurementProtocol() - mp["event_category"] = category[:150] - mp["event_action"] = action[:500] - if label: - mp["event_label"] = label[:500] - if value: - mp["event_value"] = int(value) - if screen_name: - mp["screen_name"] = screen_name[:2048] - mp.send("event") - - -def send_exception(description, is_fatal=False): +def log_debug_exception(description, debug_config: DebugConfigBase): # cleanup sensitive information, such as paths description = description.replace("Traceback (most recent call last):", "") description = description.replace("\\", "/") @@ -374,11 +280,16 @@ def send_exception(description, is_fatal=False): re.I | re.M, ) description = re.sub(r"\s+", " ", description, flags=re.M) - - mp = MeasurementProtocol() - mp["exd"] = description[:8192].strip() - mp["exf"] = 1 if is_fatal else 0 - mp.send("exception") + params = { + "description": description[:100].strip(), + "pio_user_agent": app.get_user_agent(), + } + params.update( + dump_project_env_params( + debug_config.project_config, debug_config.env_name, debug_config.platform + ) + ) + log_event("pio_debug_exception", params) @atexit.register @@ -387,54 +298,62 @@ def _finalize(): elapsed = 0 try: while elapsed < timeout: - if not MPDataPusher().in_wait(): + if not TelemetryLogger().in_wait(): break sleep(0.2) elapsed += 200 - backup_reports(MPDataPusher().get_items()) + postpone_logs(TelemetryLogger().get_unprocessed()) except KeyboardInterrupt: pass -def backup_reports(items): - if not items: - return - - KEEP_MAX_REPORTS = 100 - tm = app.get_state_item("telemetry", {}) - if "backup" not in tm: - tm["backup"] = [] - - for params in items: - # skip static options - for key in list(params.keys()): - if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): - del params[key] - - # store time in UNIX format - if "qt" not in params: - params["qt"] = time() - elif not isinstance(params["qt"], float): - params["qt"] = time() - (params["qt"] / 1000) +def load_postponed_events(): + state_path = app.resolve_state_path( + "cache_dir", "telemetry.json", ensure_dir_exists=False + ) + if not os.path.isfile(state_path): + return [] + with app.State(state_path) as state: + return state.get("events", []) - tm["backup"].append(params) - tm["backup"] = tm["backup"][KEEP_MAX_REPORTS * -1 :] - app.set_state_item("telemetry", tm) +def save_postponed_events(items): + state_path = app.resolve_state_path("cache_dir", "telemetry.json") + if not items: + try: + if os.path.isfile(state_path): + os.remove(state_path) + except: # pylint: disable=bare-except + pass + return None + with app.State(state_path, lock=True) as state: + state["events"] = items + state.modified = True + return True -def resend_backuped_reports(): - tm = app.get_state_item("telemetry", {}) - if "backup" not in tm or not tm["backup"]: - return False +def postpone_logs(payloads): + if not payloads: + return None + postponed_events = load_postponed_events() or [] + timestamp_micros = int(time() * 1000000) + for payload in payloads: + for event in payload.get("events", []): + event["timestamp_micros"] = timestamp_micros + postponed_events.append(event) + save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :]) + return True - for report in tm["backup"]: - mp = MeasurementProtocol() - for key, value in report.items(): - mp[key] = value - mp.send(report["t"]) - # clean - tm["backup"] = [] - app.set_state_item("telemetry", tm) +def resend_postponed_logs(): + events = load_postponed_events() + if not events or not ensure_internet_on(): + return None + save_postponed_events(events[SEND_MAX_EVENTS:]) # clean + mp = MeasurementProtocol() + payload = mp.to_payload() + payload["events"] = events[0:SEND_MAX_EVENTS] + TelemetryLogger().log(payload) + if len(events) > SEND_MAX_EVENTS: + resend_postponed_logs() return True diff --git a/platformio/util.py b/platformio/util.py index c981384ad8..549bccf166 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 import functools import math import platform @@ -206,3 +207,12 @@ def humanize_duration_time(duration): def strip_ansi_codes(text): # pylint: disable=protected-access return click._compat.strip_ansi(text) + + +def decrypt_message(key, message): + result = "" + message = bytearray(base64.b64decode(message)) + for i, c in enumerate(message): + key_c = key[i % len(key)] + result += chr((256 + c - ord(key_c)) % 256) + return result From 395a4053aaa0827ee6669bcc276605dea26b2d13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jun 2023 18:25:02 +0300 Subject: [PATCH 11/69] Bump version to 6.1.8a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 8294a86c4f..cee8552064 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8a1") +VERSION = (6, 1, "8a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 55e7b36dc4569af66f3dfdd3d98b9a2e3288a299 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jun 2023 18:51:58 +0300 Subject: [PATCH 12/69] Minor fixes --- platformio/telemetry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 56aa173ada..d74f099d21 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -57,11 +57,13 @@ def set_user_property(self, name, value): self._user_properties[name] = {"value": value} def add_event(self, name, params): + params["engagement_time_msec"] = params.get("engagement_time_msec", 1) self._events.append({"name": name, "params": params}) def to_payload(self): return { "client_id": app.get_cid(), + "non_personalized_ads": True, "user_properties": self._user_properties, "events": self._events, } From 7f7bc76b200d1eb70f4ebe12e199d0c7e6ca8685 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 5 Jun 2023 20:10:37 +0300 Subject: [PATCH 13/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index bc8ee6e923..f7e6122dbe 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bc8ee6e923c7f7f0cef09dc035763e2fa343977e +Subproject commit f7e6122dbe235ba58216c5ec13bbc3db170ad7ac From 4ae24a619ff071503846b1e95d790bb1d6599268 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 6 Jun 2023 14:14:38 +0300 Subject: [PATCH 14/69] Implement anonymous session mechanism to respect user privacy --- platformio/app.py | 1 + platformio/maintenance.py | 4 +-- platformio/platform/_run.py | 10 ++++++- platformio/project/helpers.py | 3 ++ platformio/telemetry.py | 55 +++++++++++++++++++++++++++++------ 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index c01e13c39b..d94cdfefc7 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -69,6 +69,7 @@ def projects_dir_validate(projects_dir): "command_ctx": None, "caller_id": None, "custom_project_conf": None, + "pause_telemetry": False, } diff --git a/platformio/maintenance.py b/platformio/maintenance.py index c7a3d3b753..f0f7a67d0b 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -32,9 +32,7 @@ def on_platformio_start(ctx, caller): app.set_session_var("command_ctx", ctx) set_caller(caller) - telemetry.log_command(ctx) - telemetry.resend_postponed_logs() - + telemetry.on_platformio_start(ctx) if PlatformioCLI.in_silence(): return after_upgrade(ctx) diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 6a29d955eb..f16a46a38e 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -17,6 +17,7 @@ import os import re import sys +import time from urllib.parse import quote import click @@ -63,8 +64,15 @@ def run( # pylint: disable=too-many-arguments if not os.path.isfile(variables["build_script"]): raise BuildScriptNotFound(variables["build_script"]) - telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) + started_at = time.time() result = self._run_scons(variables, targets, jobs) + telemetry.log_platform_run( + self, + self.config, + variables["pioenv"], + targets, + elapsed_time=time.time() - started_at, + ) assert "returncode" in result return result diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index aec12dd98a..546cd67063 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -164,6 +164,7 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False): def _load_build_metadata(project_dir, env_names, debug=False): # pylint: disable=import-outside-toplevel + from platformio import app from platformio.run.cli import cli as cmd_run args = ["--project-dir", project_dir, "--target", "__idedata"] @@ -171,7 +172,9 @@ def _load_build_metadata(project_dir, env_names, debug=False): args.extend(["--target", "__debug"]) for name in env_names: args.extend(["-e", name]) + app.set_session_var("pause_telemetry", True) result = CliRunner().invoke(cmd_run, args) + app.set_session_var("pause_telemetry", False) if result.exit_code != 0 and not isinstance( result.exception, exception.ReturnErrorCode ): diff --git a/platformio/telemetry.py b/platformio/telemetry.py index d74f099d21..06e400cec6 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -19,8 +19,8 @@ import queue import re import threading +import time from collections import deque -from time import sleep, time from traceback import format_exc import requests @@ -34,10 +34,13 @@ KEEP_MAX_REPORTS = 100 SEND_MAX_EVENTS = 25 +SESSION_TIMEOUT_DURATION = 30 * 60 # secs class MeasurementProtocol: def __init__(self): + self.client_id = app.get_cid() + self.session_id = start_session() self._user_properties = {} self._events = [] @@ -57,12 +60,13 @@ def set_user_property(self, name, value): self._user_properties[name] = {"value": value} def add_event(self, name, params): + params["session_id"] = params.get("session_id", self.session_id) params["engagement_time_msec"] = params.get("engagement_time_msec", 1) self._events.append({"name": name, "params": params}) def to_payload(self): return { - "client_id": app.get_cid(), + "client_id": self.client_id, "non_personalized_ads": True, "user_properties": self._user_properties, "events": self._events, @@ -81,7 +85,9 @@ def __init__(self): self._workers = [] def log(self, payload): - if not app.get_setting("enable_telemetry"): + if not app.get_setting("enable_telemetry") or app.get_session_var( + "pause_telemetry" + ): return None # if network is off-line @@ -126,13 +132,11 @@ def _worker(self): try: item = self._queue.get() _item = item.copy() - if "qt" not in _item: - _item["qt"] = time() self._failedque.append(_item) if self._send(item): self._failedque.remove(_item) self._queue.task_done() - except: # pylint: disable=W0702 + except: # pylint: disable=bare-except pass def _send(self, payload): @@ -164,6 +168,35 @@ def _send(self, payload): return False +@util.memoized("1m") +def start_session(): + with app.State( + app.resolve_state_path("cache_dir", "session.json"), lock=True + ) as state: + state.modified = True + start_at = state.get("start_at") + last_seen_at = state.get("last_seen_at") + + if ( + not start_at + or not last_seen_at + or last_seen_at < (time.time() - SESSION_TIMEOUT_DURATION) + ): + start_at = last_seen_at = int(time.time()) + state["start_at"] = state["last_seen_at"] = start_at + else: + state["last_seen_at"] = int(time.time()) + + session_hash = hashlib.sha1(hashlib_encode_data(app.get_cid())) + session_hash.update(hashlib_encode_data(start_at)) + return session_hash.hexdigest() + + +def on_platformio_start(cmd_ctx): + log_command(cmd_ctx) + resend_postponed_logs() + + def log_event(name, params): mp = MeasurementProtocol() mp.add_event(name, params) @@ -255,10 +288,14 @@ def dump_project_env_params(config, env, platform): return params -def log_platform_run(platform, project_config, project_env, targets=None): +def log_platform_run( + platform, project_config, project_env, targets=None, elapsed_time=None +): params = dump_project_env_params(project_config, project_env, platform) if targets: params["targets"] = ", ".join(targets) + if elapsed_time: + params["engagement_time_msec"] = int(elapsed_time * 1000) log_event("pio_platform_run", params) @@ -302,7 +339,7 @@ def _finalize(): while elapsed < timeout: if not TelemetryLogger().in_wait(): break - sleep(0.2) + time.sleep(0.2) elapsed += 200 postpone_logs(TelemetryLogger().get_unprocessed()) except KeyboardInterrupt: @@ -338,7 +375,7 @@ def postpone_logs(payloads): if not payloads: return None postponed_events = load_postponed_events() or [] - timestamp_micros = int(time() * 1000000) + timestamp_micros = int(time.time() * 1000000) for payload in payloads: for event in payload.get("events", []): event["timestamp_micros"] = timestamp_micros From 6bc915f7db8dcccf24acdb6d5a0573c8c548f784 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 6 Jun 2023 14:15:02 +0300 Subject: [PATCH 15/69] Bump version to 6.1.8a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index cee8552064..a2e7b9f888 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8a2") +VERSION = (6, 1, "8a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 141d6fc4a6efd43e3ab066cf38356c55c2009c00 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 6 Jun 2023 14:23:19 +0300 Subject: [PATCH 16/69] Use unix timestamp as session id --- platformio/telemetry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 06e400cec6..5064c3dde2 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -186,10 +186,7 @@ def start_session(): state["start_at"] = state["last_seen_at"] = start_at else: state["last_seen_at"] = int(time.time()) - - session_hash = hashlib.sha1(hashlib_encode_data(app.get_cid())) - session_hash.update(hashlib_encode_data(start_at)) - return session_hash.hexdigest() + return start_at def on_platformio_start(cmd_ctx): From 9df692529bc4d1615743789f1263e2a7b434dd48 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 6 Jun 2023 14:23:30 +0300 Subject: [PATCH 17/69] Bump version to 6.1.8a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index a2e7b9f888..6ad687229d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8a3") +VERSION = (6, 1, "8a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1bcec6654d8aa45ecd556602952b18c255959329 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 7 Jun 2023 12:10:00 +0300 Subject: [PATCH 18/69] Update deps --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c463302e61..ea6000d781 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ home_requirements = [ "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.27.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.28.*"), "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] From e3c33596dbb2db1dd5e7eb42f64ebadc55527a4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 7 Jun 2023 12:13:55 +0300 Subject: [PATCH 19/69] Use `python -m pip` instead of `pip` --- docs | 2 +- platformio/__main__.py | 2 +- platformio/maintenance.py | 2 +- platformio/telemetry.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs b/docs index f7e6122dbe..210813c303 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f7e6122dbe235ba58216c5ec13bbc3db170ad7ac +Subproject commit 210813c303bbb3dd06965a5a3e897fa9e346ffcf diff --git a/platformio/__main__.py b/platformio/__main__.py index c0ad38aca8..dd07d57e9c 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -117,7 +117,7 @@ def main(argv=None): An unexpected error occurred. Further steps: * Verify that you have the latest version of PlatformIO using - `pip install -U platformio` command + `python -m pip install -U platformio` command * Try to find answer in FAQ Troubleshooting section https://docs.platformio.org/page/faq/index.html diff --git a/platformio/maintenance.py b/platformio/maintenance.py index f0f7a67d0b..3ce31609bc 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -229,7 +229,7 @@ def check_platformio_upgrade(): else: click.secho("platformio upgrade", fg="cyan", nl=False) click.secho("` or `", fg="yellow", nl=False) - click.secho("pip install -U platformio", fg="cyan", nl=False) + click.secho("python -m pip install -U platformio", fg="cyan", nl=False) click.secho("` command.", fg="yellow") click.secho("Changes: ", fg="yellow", nl=False) click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 5064c3dde2..9b57351f80 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -67,6 +67,7 @@ def add_event(self, name, params): def to_payload(self): return { "client_id": self.client_id, + "user_id": f"cid:{self.client_id}", "non_personalized_ads": True, "user_properties": self._user_properties, "events": self._events, From f819cbb4b8679d349e2a180c0f3f94cf4141b46b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Jun 2023 14:51:52 +0300 Subject: [PATCH 20/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 210813c303..4fab8cfb59 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 210813c303bbb3dd06965a5a3e897fa9e346ffcf +Subproject commit 4fab8cfb5926210b865eb0afb1ce01460d35f515 From 2b36c7086a7820a35693d97fb0d5650a70660f11 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Jun 2023 14:52:18 +0300 Subject: [PATCH 21/69] Publish wheels to the PyPi registry --- .github/workflows/deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index ff353c31b8..3f0a4529f4 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -35,7 +35,7 @@ jobs: tox -e testcore - name: Build Python source tarball - run: python setup.py sdist + run: python setup.py sdist bdist_wheel - name: Publish package to PyPI if: ${{ github.ref == 'refs/heads/master' }} From 355f5afab93eb01dec75b901c52e9d93352ac534 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Jun 2023 15:03:42 +0300 Subject: [PATCH 22/69] * Optimized project integration templates to address the issue of long paths on Windows // Resolve #4652 --- HISTORY.rst | 1 + platformio/project/integration/generator.py | 2 +- .../integration/tpls}/atom/.clang_complete.tpl | 0 .../integration/tpls}/atom/.gcc-flags.json.tpl | 0 .../integration/tpls}/atom/.gitignore.tpl | 0 .../integration/tpls}/clion/.gitignore.tpl | 0 .../integration/tpls}/clion/CMakeLists.txt.tpl | 0 .../integration/tpls}/clion/CMakeListsPrivate.txt.tpl | 0 .../integration/tpls}/codeblocks/platformio.cbp.tpl | 0 .../integration/tpls}/eclipse/.cproject.tpl | 0 .../integration/tpls}/eclipse/.project.tpl | 0 .../eclipse/.settings/PlatformIO Debugger.launch.tpl | 0 .../tpls}/eclipse/.settings/language.settings.xml.tpl | 0 .../eclipse/.settings/org.eclipse.cdt.core.prefs.tpl | 0 .../integration/tpls}/emacs/.ccls.tpl | 0 .../integration/tpls}/emacs/.gitignore.tpl | 0 .../tpls}/netbeans/nbproject/configurations.xml.tpl | 0 .../netbeans/nbproject/private/configurations.xml.tpl | 0 .../netbeans/nbproject/private/launcher.properties.tpl | 0 .../tpls}/netbeans/nbproject/private/private.xml.tpl | 0 .../tpls}/netbeans/nbproject/project.xml.tpl | 0 .../integration/tpls}/qtcreator/.gitignore.tpl | 0 .../integration/tpls}/qtcreator/Makefile.tpl | 0 .../integration/tpls}/qtcreator/platformio.cflags.tpl | 0 .../integration/tpls}/qtcreator/platformio.config.tpl | 0 .../integration/tpls}/qtcreator/platformio.creator.tpl | 0 .../tpls}/qtcreator/platformio.cxxflags.tpl | 0 .../integration/tpls}/qtcreator/platformio.files.tpl | 0 .../tpls}/qtcreator/platformio.includes.tpl | 0 .../integration/tpls}/sublimetext/.ccls.tpl | 0 .../tpls}/sublimetext/platformio.sublime-project.tpl | 0 .../integration/tpls}/vim/.ccls.tpl | 0 .../integration/tpls}/vim/.gitignore.tpl | 0 .../tpls}/visualstudio/platformio.vcxproj.filters.tpl | 0 .../tpls}/visualstudio/platformio.vcxproj.tpl | 0 .../integration/tpls}/vscode/.gitignore.tpl | 0 .../tpls}/vscode/.vscode/c_cpp_properties.json.tpl | 0 .../tpls}/vscode/.vscode/extensions.json.tpl | 0 .../integration/tpls}/vscode/.vscode/launch.json.tpl | 0 setup.py | 10 +++++----- 40 files changed, 7 insertions(+), 6 deletions(-) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/atom/.clang_complete.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/atom/.gcc-flags.json.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/atom/.gitignore.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/clion/.gitignore.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/clion/CMakeLists.txt.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/clion/CMakeListsPrivate.txt.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/codeblocks/platformio.cbp.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/eclipse/.cproject.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/eclipse/.project.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/eclipse/.settings/PlatformIO Debugger.launch.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/eclipse/.settings/language.settings.xml.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/emacs/.ccls.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/emacs/.gitignore.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/netbeans/nbproject/configurations.xml.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/netbeans/nbproject/private/configurations.xml.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/netbeans/nbproject/private/launcher.properties.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/netbeans/nbproject/private/private.xml.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/netbeans/nbproject/project.xml.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/.gitignore.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/Makefile.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/platformio.cflags.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/platformio.config.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/platformio.creator.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/platformio.cxxflags.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/platformio.files.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/qtcreator/platformio.includes.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/sublimetext/.ccls.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/sublimetext/platformio.sublime-project.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/vim/.ccls.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/vim/.gitignore.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/visualstudio/platformio.vcxproj.filters.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/visualstudio/platformio.vcxproj.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/vscode/.gitignore.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/vscode/.vscode/c_cpp_properties.json.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/vscode/.vscode/extensions.json.tpl (100%) rename platformio/{assets/templates/ide-projects => project/integration/tpls}/vscode/.vscode/launch.json.tpl (100%) diff --git a/HISTORY.rst b/HISTORY.rst index a29f23d1dc..f277cbdac0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 6 ~~~~~~~~~~~~~~~~~~ * Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information +* Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 654e5ac353..d55f9aa7dc 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -52,7 +52,7 @@ def get_best_envname(self, boards=None): @staticmethod def get_ide_tpls_dir(): - return os.path.join(fs.get_assets_dir(), "templates", "ide-projects") + return os.path.join(os.path.dirname(__file__), "tpls") @classmethod def get_supported_ides(cls): diff --git a/platformio/assets/templates/ide-projects/atom/.clang_complete.tpl b/platformio/project/integration/tpls/atom/.clang_complete.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/atom/.clang_complete.tpl rename to platformio/project/integration/tpls/atom/.clang_complete.tpl diff --git a/platformio/assets/templates/ide-projects/atom/.gcc-flags.json.tpl b/platformio/project/integration/tpls/atom/.gcc-flags.json.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/atom/.gcc-flags.json.tpl rename to platformio/project/integration/tpls/atom/.gcc-flags.json.tpl diff --git a/platformio/assets/templates/ide-projects/atom/.gitignore.tpl b/platformio/project/integration/tpls/atom/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/atom/.gitignore.tpl rename to platformio/project/integration/tpls/atom/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/clion/.gitignore.tpl b/platformio/project/integration/tpls/clion/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/clion/.gitignore.tpl rename to platformio/project/integration/tpls/clion/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/clion/CMakeLists.txt.tpl b/platformio/project/integration/tpls/clion/CMakeLists.txt.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/clion/CMakeLists.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeLists.txt.tpl diff --git a/platformio/assets/templates/ide-projects/clion/CMakeListsPrivate.txt.tpl b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/clion/CMakeListsPrivate.txt.tpl rename to platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl diff --git a/platformio/assets/templates/ide-projects/codeblocks/platformio.cbp.tpl b/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/codeblocks/platformio.cbp.tpl rename to platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.cproject.tpl b/platformio/project/integration/tpls/eclipse/.cproject.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.cproject.tpl rename to platformio/project/integration/tpls/eclipse/.cproject.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.project.tpl b/platformio/project/integration/tpls/eclipse/.project.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.project.tpl rename to platformio/project/integration/tpls/eclipse/.project.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.settings/PlatformIO Debugger.launch.tpl rename to platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.settings/language.settings.xml.tpl b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.settings/language.settings.xml.tpl rename to platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl diff --git a/platformio/assets/templates/ide-projects/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl rename to platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl diff --git a/platformio/assets/templates/ide-projects/emacs/.ccls.tpl b/platformio/project/integration/tpls/emacs/.ccls.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/emacs/.ccls.tpl rename to platformio/project/integration/tpls/emacs/.ccls.tpl diff --git a/platformio/assets/templates/ide-projects/emacs/.gitignore.tpl b/platformio/project/integration/tpls/emacs/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/emacs/.gitignore.tpl rename to platformio/project/integration/tpls/emacs/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/private/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/private/configurations.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/private/launcher.properties.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/private/launcher.properties.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/private/private.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/private/private.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl diff --git a/platformio/assets/templates/ide-projects/netbeans/nbproject/project.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/netbeans/nbproject/project.xml.tpl rename to platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/.gitignore.tpl b/platformio/project/integration/tpls/qtcreator/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/.gitignore.tpl rename to platformio/project/integration/tpls/qtcreator/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/Makefile.tpl b/platformio/project/integration/tpls/qtcreator/Makefile.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/Makefile.tpl rename to platformio/project/integration/tpls/qtcreator/Makefile.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.cflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.cflags.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.config.tpl b/platformio/project/integration/tpls/qtcreator/platformio.config.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.config.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.config.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.creator.tpl b/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.creator.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.creator.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.cxxflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.cxxflags.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.files.tpl b/platformio/project/integration/tpls/qtcreator/platformio.files.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.files.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.files.tpl diff --git a/platformio/assets/templates/ide-projects/qtcreator/platformio.includes.tpl b/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/qtcreator/platformio.includes.tpl rename to platformio/project/integration/tpls/qtcreator/platformio.includes.tpl diff --git a/platformio/assets/templates/ide-projects/sublimetext/.ccls.tpl b/platformio/project/integration/tpls/sublimetext/.ccls.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/sublimetext/.ccls.tpl rename to platformio/project/integration/tpls/sublimetext/.ccls.tpl diff --git a/platformio/assets/templates/ide-projects/sublimetext/platformio.sublime-project.tpl b/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/sublimetext/platformio.sublime-project.tpl rename to platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl diff --git a/platformio/assets/templates/ide-projects/vim/.ccls.tpl b/platformio/project/integration/tpls/vim/.ccls.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vim/.ccls.tpl rename to platformio/project/integration/tpls/vim/.ccls.tpl diff --git a/platformio/assets/templates/ide-projects/vim/.gitignore.tpl b/platformio/project/integration/tpls/vim/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vim/.gitignore.tpl rename to platformio/project/integration/tpls/vim/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.filters.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.filters.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl diff --git a/platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/visualstudio/platformio.vcxproj.tpl rename to platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.gitignore.tpl b/platformio/project/integration/tpls/vscode/.gitignore.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.gitignore.tpl rename to platformio/project/integration/tpls/vscode/.gitignore.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.vscode/c_cpp_properties.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/extensions.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.vscode/extensions.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl similarity index 100% rename from platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl rename to platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl diff --git a/setup.py b/setup.py index ea6000d781..e8fad3e17e 100644 --- a/setup.py +++ b/setup.py @@ -65,11 +65,11 @@ package_data={ "platformio": [ "assets/system/99-platformio-udev.rules", - "assets/templates/ide-projects/*/*.tpl", - "assets/templates/ide-projects/*/.*.tpl", # include hidden files - "assets/templates/ide-projects/*/.*/*.tpl", # include hidden folders - "assets/templates/ide-projects/*/*/*.tpl", # NetBeans - "assets/templates/ide-projects/*/*/*/*.tpl", # NetBeans + "project/integration/tpls/*/*.tpl", + "project/integration/tpls/*/.*.tpl", # include hidden files + "project/integration/tpls/*/.*/*.tpl", # include hidden folders + "project/integration/tpls/*/*/*.tpl", # NetBeans + "project/integration/tpls/*/*/*/*.tpl", # NetBeans ] }, entry_points={ From c9235a52761657ed5a8182a5018c067096f5500f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 9 Jun 2023 15:18:07 +0300 Subject: [PATCH 23/69] Use UTC-based timestamp --- platformio/account/client.py | 4 ++-- platformio/app.py | 5 ++--- platformio/builder/main.py | 3 ++- platformio/telemetry.py | 1 - platformio/util.py | 10 +++++++--- tests/commands/pkg/test_install.py | 6 +++--- tests/commands/pkg/test_uninstall.py | 4 ++-- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/platformio/account/client.py b/platformio/account/client.py index a898de481d..2b9cb57e72 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -15,7 +15,7 @@ import os import time -from platformio import __accounts_api__, app +from platformio import __accounts_api__, app, util from platformio.exception import PlatformioException from platformio.http import HTTPClient, HTTPClientError @@ -68,7 +68,7 @@ def fetch_authentication_token(self): return os.environ.get("PLATFORMIO_AUTH_TOKEN") auth = app.get_state_item("account", {}).get("auth", {}) if auth.get("access_token") and auth.get("access_token_expire"): - if auth.get("access_token_expire") > time.time(): + if auth.get("access_token_expire") > util.get_timestamp(): return auth.get("access_token") if auth.get("refresh_token"): try: diff --git a/platformio/app.py b/platformio/app.py index d94cdfefc7..534214d19a 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -18,10 +18,9 @@ import os import platform import socket -import time import uuid -from platformio import __version__, exception, fs, proc +from platformio import __version__, exception, fs, proc, util from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.config import ProjectConfig @@ -254,7 +253,7 @@ def get_cid(): cid = str(cid) if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) - set_state_item("created_at", int(time.time())) + set_state_item("created_at", int(util.get_timestamp())) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8670278345..3aa460cdff 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -72,7 +72,8 @@ variables=clivars, # Propagating External Environment ENV=os.environ, - UNIX_TIME=int(time()), + TIMESTAMP=int(time()), + UNIX_TIME="$TIMESTAMP", # deprecated BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 9b57351f80..5064c3dde2 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -67,7 +67,6 @@ def add_event(self, name, params): def to_payload(self): return { "client_id": self.client_id, - "user_id": f"cid:{self.client_id}", "non_personalized_ads": True, "user_properties": self._user_properties, "events": self._events, diff --git a/platformio/util.py b/platformio/util.py index 549bccf166..3666bf6055 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -13,13 +13,13 @@ # limitations under the License. import base64 +import datetime import functools import math import platform import re import shutil import time -from datetime import datetime import click @@ -168,10 +168,14 @@ def items_in_list(needle, haystack): return set(needle) & set(haystack) +def get_timestamp(utc=True): + return datetime.datetime.now(datetime.timezone.utc if utc else None).timestamp() + + def parse_datetime(datestr): if "T" in datestr and "Z" in datestr: - return datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") - return datetime.strptime(datestr) + return datetime.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") + return datetime.datetime.strptime(datestr) def merge_dicts(d1, d2, path=None): diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py index 61ccde0da7..ea995c6810 100644 --- a/tests/commands/pkg/test_install.py +++ b/tests/commands/pkg/test_install.py @@ -469,7 +469,7 @@ def test_custom_project_tools( project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) - spec = "platformio/tool-openocd" + spec = "platformio/tool-openocd @ ^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], @@ -503,7 +503,7 @@ def test_custom_project_tools( # check saved deps assert config.get("env:devkit", "platform_packages") == [ - "platformio/tool-openocd@^2.1100.211028", + "platformio/tool-openocd@^2", ] # install tool without saving to config @@ -518,7 +518,7 @@ def test_custom_project_tools( PackageSpec("tool-openocd@2.1100.211028"), ] assert config.get("env:devkit", "platform_packages") == [ - "platformio/tool-openocd@^2.1100.211028", + "platformio/tool-openocd@^2", ] # unknown tool diff --git a/tests/commands/pkg/test_uninstall.py b/tests/commands/pkg/test_uninstall.py index 1dae981c7a..d776b142e8 100644 --- a/tests/commands/pkg/test_uninstall.py +++ b/tests/commands/pkg/test_uninstall.py @@ -313,7 +313,7 @@ def test_custom_project_tools( project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) - spec = "platformio/tool-openocd" + spec = "platformio/tool-openocd@^2" result = clirunner.invoke( package_install_cmd, ["-d", str(project_dir), "-e", "devkit", "-t", spec], @@ -329,7 +329,7 @@ def test_custom_project_tools( assert not os.path.exists(config.get("platformio", "platforms_dir")) # check saved deps assert config.get("env:devkit", "platform_packages") == [ - "platformio/tool-openocd@^2.1100.211028", + "platformio/tool-openocd@^2", ] # uninstall result = clirunner.invoke( From 41cc7359795c2c557d934e957d56e49e29760725 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Jun 2023 20:06:00 +0300 Subject: [PATCH 24/69] VCS base exception is UserSideException --- platformio/package/vcsclient.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index 058a4983f7..c10235c73b 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -18,14 +18,10 @@ from urllib.parse import urlparse from platformio import proc -from platformio.package.exception import ( - PackageException, - PlatformioException, - UserSideException, -) +from platformio.package.exception import PlatformioException, UserSideException -class VCSBaseException(PackageException): +class VCSBaseException(UserSideException): pass @@ -75,7 +71,7 @@ def check_client(self): else: assert self.run_cmd(["--version"]) except (AssertionError, OSError, PlatformioException) as exc: - raise UserSideException( + raise VCSBaseException( "VCS: `%s` client is not installed in your system" % self.command ) from exc return True From 363fee4ba06f1db8fc57ecdd87b1a226e75b3cf5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Jun 2023 20:06:46 +0300 Subject: [PATCH 25/69] Use direct urllib3 module --- platformio/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/http.py b/platformio/http.py index 5f3021bf63..e5c57835f2 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -18,7 +18,7 @@ from urllib.parse import urljoin import requests.adapters -from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error +from urllib3.util.retry import Retry from platformio import __check_internet_hosts__, app, util from platformio.cache import ContentCache, cleanup_content_cache From 91487f179ea78ccc0d9fce07ac847af06545ba3f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Jun 2023 20:07:26 +0300 Subject: [PATCH 26/69] Catch debug init error --- platformio/debug/exception.py | 4 ++++ platformio/debug/process/gdb.py | 9 ++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/platformio/debug/exception.py b/platformio/debug/exception.py index 7f4d0f4cf1..5147815d3c 100644 --- a/platformio/debug/exception.py +++ b/platformio/debug/exception.py @@ -30,3 +30,7 @@ class DebugSupportError(DebugError, UserSideException): class DebugInvalidOptionsError(DebugError, UserSideException): pass + + +class DebugInitError(DebugError, UserSideException): + pass diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 2a1f2f7255..29a8ba3a23 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -13,13 +13,13 @@ # limitations under the License. import os -import re import signal import time from platformio import telemetry from platformio.compat import aio_get_running_loop, is_bytes from platformio.debug import helpers +from platformio.debug.exception import DebugInitError from platformio.debug.process.client import DebugClientProcess @@ -175,12 +175,7 @@ def _handle_error(self, data): and b"Error in sourced" in self._errors_buffer ): return - - last_errors = self._errors_buffer.decode() - last_errors = " ".join(reversed(last_errors.split("\n"))) - last_errors = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_errors, flags=re.M) telemetry.log_debug_exception( - "DebugInitError: %s" % last_errors, self.debug_config + DebugInitError(self._errors_buffer.decode()), self.debug_config ) - self.transport.close() From 378528abfca0e8685a8eb6ad769ceb01f51c9745 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Jun 2023 20:08:45 +0300 Subject: [PATCH 27/69] Switch to Python's time module --- platformio/account/client.py | 4 ++-- platformio/app.py | 5 +++-- platformio/platform/_run.py | 11 ++--------- platformio/system/commands/info.py | 1 - platformio/util.py | 14 -------------- 5 files changed, 7 insertions(+), 28 deletions(-) diff --git a/platformio/account/client.py b/platformio/account/client.py index 2b9cb57e72..a898de481d 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -15,7 +15,7 @@ import os import time -from platformio import __accounts_api__, app, util +from platformio import __accounts_api__, app from platformio.exception import PlatformioException from platformio.http import HTTPClient, HTTPClientError @@ -68,7 +68,7 @@ def fetch_authentication_token(self): return os.environ.get("PLATFORMIO_AUTH_TOKEN") auth = app.get_state_item("account", {}).get("auth", {}) if auth.get("access_token") and auth.get("access_token_expire"): - if auth.get("access_token_expire") > util.get_timestamp(): + if auth.get("access_token_expire") > time.time(): return auth.get("access_token") if auth.get("refresh_token"): try: diff --git a/platformio/app.py b/platformio/app.py index 534214d19a..d94cdfefc7 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -18,9 +18,10 @@ import os import platform import socket +import time import uuid -from platformio import __version__, exception, fs, proc, util +from platformio import __version__, exception, fs, proc from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.config import ProjectConfig @@ -253,7 +254,7 @@ def get_cid(): cid = str(cid) if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) - set_state_item("created_at", int(util.get_timestamp())) + set_state_item("created_at", int(time.time())) return cid diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index f16a46a38e..daca77c311 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -17,7 +17,6 @@ import os import re import sys -import time from urllib.parse import quote import click @@ -64,15 +63,9 @@ def run( # pylint: disable=too-many-arguments if not os.path.isfile(variables["build_script"]): raise BuildScriptNotFound(variables["build_script"]) - started_at = time.time() + telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) result = self._run_scons(variables, targets, jobs) - telemetry.log_platform_run( - self, - self.config, - variables["pioenv"], - targets, - elapsed_time=time.time() - started_at, - ) + assert "returncode" in result return result diff --git a/platformio/system/commands/info.py b/platformio/system/commands/info.py index 52a066362d..4db1f42895 100644 --- a/platformio/system/commands/info.py +++ b/platformio/system/commands/info.py @@ -77,7 +77,6 @@ def system_info_cmd(json_output): ).get_installed() ), } - click.echo( json.dumps(data) if json_output diff --git a/platformio/util.py b/platformio/util.py index 3666bf6055..004ac5c9f1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 import datetime import functools import math @@ -168,10 +167,6 @@ def items_in_list(needle, haystack): return set(needle) & set(haystack) -def get_timestamp(utc=True): - return datetime.datetime.now(datetime.timezone.utc if utc else None).timestamp() - - def parse_datetime(datestr): if "T" in datestr and "Z" in datestr: return datetime.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") @@ -211,12 +206,3 @@ def humanize_duration_time(duration): def strip_ansi_codes(text): # pylint: disable=protected-access return click._compat.strip_ansi(text) - - -def decrypt_message(key, message): - result = "" - message = bytearray(base64.b64decode(message)) - for i, c in enumerate(message): - key_c = key[i % len(key)] - result += chr((256 + c - ord(key_c)) % 256) - return result From d017a8197e62c7cb0cec24202b8b1cba8e4dce82 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Jun 2023 20:09:23 +0300 Subject: [PATCH 28/69] Refactor event logging --- platformio/telemetry.py | 356 +++++++++++++++++++--------------------- 1 file changed, 171 insertions(+), 185 deletions(-) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 5064c3dde2..0c8adc5233 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -15,59 +15,53 @@ import atexit import hashlib import os -import platform as python_platform import queue import re import threading import time +import traceback from collections import deque -from traceback import format_exc import requests -from platformio import __title__, __version__, app, exception, util +from platformio import __title__, __version__, app, exception, fs, util from platformio.cli import PlatformioCLI from platformio.compat import hashlib_encode_data from platformio.debug.config.base import DebugConfigBase from platformio.http import HTTPSession, ensure_internet_on -from platformio.proc import is_ci, is_container +from platformio.proc import is_ci KEEP_MAX_REPORTS = 100 SEND_MAX_EVENTS = 25 -SESSION_TIMEOUT_DURATION = 30 * 60 # secs class MeasurementProtocol: - def __init__(self): + def __init__(self, events=None): self.client_id = app.get_cid() - self.session_id = start_session() + self._events = events or [] self._user_properties = {} - self._events = [] - - caller_id = app.get_session_var("caller_id") - if caller_id: - self.set_user_property("pio_caller_id", caller_id) - self.set_user_property("pio_core_version", __version__) - self.set_user_property( - "pio_human_actor", int(bool(caller_id or not (is_ci() or is_container()))) - ) - self.set_user_property("pio_systype", util.get_systype()) + + self.set_user_property("systype", util.get_systype()) created_at = app.get_state_item("created_at", None) if created_at: - self.set_user_property("pio_created_at", int(created_at)) + self.set_user_property("created_at", int(created_at)) + + @staticmethod + def event_to_dict(name, params, timestamp=None): + event = {"name": name, "params": params} + if timestamp is not None: + event["timestamp"] = timestamp + return event def set_user_property(self, name, value): - self._user_properties[name] = {"value": value} + self._user_properties[name] = value def add_event(self, name, params): - params["session_id"] = params.get("session_id", self.session_id) - params["engagement_time_msec"] = params.get("engagement_time_msec", 1) - self._events.append({"name": name, "params": params}) + self._events.append(self.event_to_dict(name, params)) def to_payload(self): return { "client_id": self.client_id, - "non_personalized_ads": True, "user_properties": self._user_properties, "events": self._events, } @@ -75,142 +69,122 @@ def to_payload(self): @util.singleton class TelemetryLogger: - MAX_WORKERS = 5 - def __init__(self): - self._queue = queue.LifoQueue() - self._failedque = deque() + self._events = deque() + + self._sender_thread = None + self._sender_queue = queue.Queue() + self._sender_terminated = False + self._http_session = HTTPSession() self._http_offline = False - self._workers = [] - def log(self, payload): + def close(self): + self._http_session.close() + + def log_event(self, name, params, timestamp=None, instant_sending=False): if not app.get_setting("enable_telemetry") or app.get_session_var( "pause_telemetry" ): return None - - # if network is off-line - if self._http_offline: - self._failedque.append(payload) + timestamp = timestamp or int(time.time()) + self._events.append( + MeasurementProtocol.event_to_dict(name, params, timestamp=timestamp) + ) + if self._http_offline: # if network is off-line return False - - self._queue.put(payload) - self._tune_workers() + if instant_sending: + self.send() return True - def in_wait(self): - return self._queue.unfinished_tasks - - def get_unprocessed(self): - items = list(self._failedque) - try: - while True: - items.append(self._queue.get_nowait()) - except queue.Empty: - pass - return items - - def _tune_workers(self): - for i, w in enumerate(self._workers): - if not w.is_alive(): - del self._workers[i] - - need_nums = min(self._queue.qsize(), self.MAX_WORKERS) - active_nums = len(self._workers) - if need_nums <= active_nums: + def send(self): + if not self._events or self._sender_terminated: return + if not self._sender_thread: + self._sender_thread = threading.Thread( + target=self._sender_worker, daemon=True + ) + self._sender_thread.start() + while self._events: + events = [] + try: + while len(events) < SEND_MAX_EVENTS: + events.append(self._events.popleft()) + except IndexError: + pass + self._sender_queue.put(events) - for i in range(need_nums - active_nums): - t = threading.Thread(target=self._worker) - t.daemon = True - t.start() - self._workers.append(t) - - def _worker(self): + def _sender_worker(self): while True: + if self._sender_terminated: + return try: - item = self._queue.get() - _item = item.copy() - self._failedque.append(_item) - if self._send(item): - self._failedque.remove(_item) - self._queue.task_done() - except: # pylint: disable=bare-except + events = self._sender_queue.get() + if not self._commit_events(events): + self._events.extend(events) + self._sender_queue.task_done() + except (queue.Empty, ValueError): pass - def _send(self, payload): + def _commit_events(self, events): if self._http_offline: return False + mp = MeasurementProtocol(events) + payload = mp.to_payload() + # print("_commit_payload", payload) try: r = self._http_session.post( - "https://www.google-analytics.com/mp/collect", - params={ - "measurement_id": util.decrypt_message( - __title__, "t5m7rKu6tbqwx8Cw" - ), - "api_secret": util.decrypt_message( - __title__, "48SRy5rmut28ptm7zLjS5sa7tdmhrQ==" - ), - }, + "https://telemetry.platformio.org/collect", json=payload, - timeout=1, + timeout=(2, 5), # connect, read ) r.raise_for_status() return True except requests.exceptions.HTTPError as exc: # skip Bad Request - if 400 >= exc.response.status_code < 500: + if exc.response.status_code >= 400 and exc.response.status_code < 500: return True except: # pylint: disable=bare-except pass self._http_offline = True return False + def terminate_sender(self): + self._sender_terminated = True -@util.memoized("1m") -def start_session(): - with app.State( - app.resolve_state_path("cache_dir", "session.json"), lock=True - ) as state: - state.modified = True - start_at = state.get("start_at") - last_seen_at = state.get("last_seen_at") + def is_sending(self): + return self._sender_queue.unfinished_tasks + + def get_unsent_events(self): + result = list(self._events) + try: + while True: + result.extend(self._sender_queue.get_nowait()) + except queue.Empty: + pass + return result - if ( - not start_at - or not last_seen_at - or last_seen_at < (time.time() - SESSION_TIMEOUT_DURATION) - ): - start_at = last_seen_at = int(time.time()) - state["start_at"] = state["last_seen_at"] = start_at - else: - state["last_seen_at"] = int(time.time()) - return start_at + +def log_event(name, params, instant_sending=False): + TelemetryLogger().log_event(name, params, instant_sending=instant_sending) def on_platformio_start(cmd_ctx): + process_postponed_logs() log_command(cmd_ctx) - resend_postponed_logs() -def log_event(name, params): - mp = MeasurementProtocol() - mp.add_event(name, params) - TelemetryLogger().log(mp.to_payload()) +def on_platformio_end(): + TelemetryLogger().send() def log_command(ctx): - path_args = PlatformioCLI.reveal_cmd_path_args(ctx) params = { - "page_title": " ".join([arg.title() for arg in path_args]), - "page_path": "/".join(path_args), - "pio_user_agent": app.get_user_agent(), - "pio_python_version": python_platform.python_version(), + "path_args": PlatformioCLI.reveal_cmd_path_args(ctx), } if is_ci(): params["ci_actor"] = resolve_ci_actor() or "Unknown" - log_event("page_view", params) + log_event("cmd_run", params) def resolve_ci_actor(): @@ -229,35 +203,6 @@ def resolve_ci_actor(): return None -def log_exception(e): - skip_conditions = [ - isinstance(e, cls) - for cls in ( - IOError, - exception.ReturnErrorCode, - exception.UserSideException, - ) - ] - if any(skip_conditions): - return - is_fatal = any( - [ - not isinstance(e, exception.PlatformioException), - "Error" in e.__class__.__name__, - ] - ) - description = "%s: %s" % ( - type(e).__name__, - " ".join(reversed(format_exc().split("\n"))) if is_fatal else str(e), - ) - params = { - "description": description[:100].strip(), - "is_fatal": int(is_fatal), - "pio_user_agent": app.get_user_agent(), - } - log_event("pio_exception", params) - - def dump_project_env_params(config, env, platform): non_sensitive_data = [ "platform", @@ -270,77 +215,113 @@ def dump_project_env_params(config, env, platform): ] section = f"env:{env}" params = { - f"pio_{option}": config.get(section, option) + option: config.get(section, option) for option in non_sensitive_data if config.has_option(section, option) } - params["pio_pid"] = hashlib.sha1(hashlib_encode_data(config.path)).hexdigest() - params["pio_platform_name"] = platform.name - params["pio_platform_version"] = platform.version - params["pio_framework"] = params.get("pio_framework", "__bare_metal__") - # join multi-value options - for key, value in params.items(): - if isinstance(value, list): - params[key] = ", ".join(value) + params["pid"] = hashlib.sha1(hashlib_encode_data(config.path)).hexdigest() + params["platform_name"] = platform.name + params["platform_version"] = platform.version return params -def log_platform_run( - platform, project_config, project_env, targets=None, elapsed_time=None -): +def log_platform_run(platform, project_config, project_env, targets=None): params = dump_project_env_params(project_config, project_env, platform) if targets: - params["targets"] = ", ".join(targets) - if elapsed_time: - params["engagement_time_msec"] = int(elapsed_time * 1000) - log_event("pio_platform_run", params) + params["targets"] = targets + log_event("platform_run", params, instant_sending=True) + + +def log_exception(exc): + skip_conditions = [ + isinstance(exc, cls) + for cls in ( + IOError, + exception.ReturnErrorCode, + exception.UserSideException, + ) + ] + skip_conditions.append(not isinstance(exc, Exception)) + if any(skip_conditions): + return + is_fatal = any( + [ + not isinstance(exc, exception.PlatformioException), + "Error" in exc.__class__.__name__, + ] + ) + + def _strip_module_path(match): + module_path = match.group(1).replace(fs.get_source_dir() + os.sep, "") + sp_folder_name = "site-packages" + sp_pos = module_path.find(sp_folder_name) + if sp_pos != -1: + module_path = module_path[sp_pos + len(sp_folder_name) + 1 :] + module_path = fs.to_unix_path(module_path) + return f'File "{module_path}",' + + trace = re.sub( + r'File "([^"]+)",', + _strip_module_path, + traceback.format_exc(), + flags=re.MULTILINE, + ) + + params = { + "name": exc.__class__.__name__, + "description": str(exc), + "traceback": trace, + "is_fatal": is_fatal, + } + log_event("exception", params) def log_debug_started(debug_config: DebugConfigBase): log_event( - "pio_debug_started", + "debug_started", dump_project_env_params( debug_config.project_config, debug_config.env_name, debug_config.platform ), ) -def log_debug_exception(description, debug_config: DebugConfigBase): +def log_debug_exception(exc, debug_config: DebugConfigBase): # cleanup sensitive information, such as paths - description = description.replace("Traceback (most recent call last):", "") - description = description.replace("\\", "/") + description = fs.to_unix_path(str(exc)) description = re.sub( r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)', lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]), description, re.I | re.M, ) - description = re.sub(r"\s+", " ", description, flags=re.M) params = { - "description": description[:100].strip(), - "pio_user_agent": app.get_user_agent(), + "name": exc.__class__.__name__, + "description": description.strip(), } params.update( dump_project_env_params( debug_config.project_config, debug_config.env_name, debug_config.platform ) ) - log_event("pio_debug_exception", params) + log_event("debug_exception", params) @atexit.register def _finalize(): timeout = 1000 # msec elapsed = 0 + telemetry = TelemetryLogger() + telemetry.terminate_sender() try: while elapsed < timeout: - if not TelemetryLogger().in_wait(): + if not telemetry.is_sending(): break time.sleep(0.2) elapsed += 200 - postpone_logs(TelemetryLogger().get_unprocessed()) except KeyboardInterrupt: pass + postpone_events(telemetry.get_unsent_events()) + telemetry.close() def load_postponed_events(): @@ -353,9 +334,9 @@ def load_postponed_events(): return state.get("events", []) -def save_postponed_events(items): +def save_postponed_events(events): state_path = app.resolve_state_path("cache_dir", "telemetry.json") - if not items: + if not events: try: if os.path.isfile(state_path): os.remove(state_path) @@ -363,33 +344,38 @@ def save_postponed_events(items): pass return None with app.State(state_path, lock=True) as state: - state["events"] = items + state["events"] = events state.modified = True return True -def postpone_logs(payloads): - if not payloads: +def postpone_events(events): + if not events: return None postponed_events = load_postponed_events() or [] - timestamp_micros = int(time.time() * 1000000) - for payload in payloads: - for event in payload.get("events", []): - event["timestamp_micros"] = timestamp_micros - postponed_events.append(event) + timestamp = int(time.time()) + for event in events: + if "timestamp" not in event: + event["timestamp"] = timestamp + postponed_events.append(event) save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :]) return True -def resend_postponed_logs(): +def process_postponed_logs(): + if not ensure_internet_on(): + return None events = load_postponed_events() - if not events or not ensure_internet_on(): + if not events: return None - save_postponed_events(events[SEND_MAX_EVENTS:]) # clean - mp = MeasurementProtocol() - payload = mp.to_payload() - payload["events"] = events[0:SEND_MAX_EVENTS] - TelemetryLogger().log(payload) - if len(events) > SEND_MAX_EVENTS: - resend_postponed_logs() + save_postponed_events([]) # clean + telemetry = TelemetryLogger() + for event in events: + if set(["name", "params", "timestamp"]) <= set(event.keys()): + telemetry.log_event( + event["name"], + event["params"], + timestamp=event["timestamp"], + instant_sending=False, + ) return True From c0d2abc9a7ca62064bd509fa8f6c7497098aba6e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Jun 2023 20:10:39 +0300 Subject: [PATCH 29/69] Call "on_platformio_end" when Click finished --- platformio/__main__.py | 13 +++++-------- platformio/maintenance.py | 3 ++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index dd07d57e9c..acc50ad87d 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,7 +14,7 @@ import os import sys -from traceback import format_exc +import traceback import click @@ -56,12 +56,6 @@ def cli(ctx, force, caller, no_ansi): # pylint: disable=unused-argument maintenance.on_platformio_start(ctx, caller) -@cli.result_callback() -@click.pass_context -def process_result(ctx, result, *_, **__): - maintenance.on_platformio_end(ctx, result) - - def configure(): if IS_CYGWIN: raise exception.CygwinEnvDetected() @@ -96,6 +90,7 @@ def main(argv=None): if argv: assert isinstance(argv, list) sys.argv = argv + try: ensure_python3(raise_exception=True) configure() @@ -110,7 +105,7 @@ def main(argv=None): if isinstance(exc, exception.PlatformioException): error_str += str(exc) else: - error_str += format_exc() + error_str += traceback.format_exc() error_str += """ ============================================================ @@ -129,6 +124,8 @@ def main(argv=None): """ click.secho(error_str, fg="red", err=True) exit_code = int(str(exc)) if str(exc).isdigit() else 1 + + maintenance.on_platformio_end() sys.argv = prev_sys_argv return exit_code diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 3ce31609bc..936a0d8626 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -38,7 +38,8 @@ def on_platformio_start(ctx, caller): after_upgrade(ctx) -def on_platformio_end(ctx, result): # pylint: disable=unused-argument +def on_platformio_end(): + telemetry.on_platformio_end() if PlatformioCLI.in_silence(): return From e48dfbaadca2b50b2a0e32128a8fa132fd8a1bd6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Jun 2023 11:38:43 +0300 Subject: [PATCH 30/69] Update SPDX Data to v3.21 --- platformio/package/manifest/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 2075ac0381..f1ec94451b 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -276,9 +276,9 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.20" + version = "3.21" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" - "v%s/json/licenses.json" % version + f"v{version}/json/licenses.json" ) return json.loads(fetch_remote_content(spdx_data_url)) From 292dc3fd7135ab503633aa2f9b92af3d7cd05e83 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Jun 2023 11:40:40 +0300 Subject: [PATCH 31/69] Update user-related exceptions to UserSideException --- platformio/account/client.py | 6 +++--- platformio/package/exception.py | 8 ++++---- platformio/package/vcsclient.py | 4 ++-- platformio/platform/exception.py | 4 ++-- platformio/project/helpers.py | 2 +- platformio/telemetry.py | 2 ++ 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/platformio/account/client.py b/platformio/account/client.py index a898de481d..064321448c 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -16,7 +16,7 @@ import time from platformio import __accounts_api__, app -from platformio.exception import PlatformioException +from platformio.exception import PlatformioException, UserSideException from platformio.http import HTTPClient, HTTPClientError @@ -24,11 +24,11 @@ class AccountError(PlatformioException): MESSAGE = "{0}" -class AccountNotAuthorized(AccountError): +class AccountNotAuthorized(AccountError, UserSideException): MESSAGE = "You are not authorized! Please log in to PlatformIO Account." -class AccountAlreadyAuthorized(AccountError): +class AccountAlreadyAuthorized(AccountError, UserSideException): MESSAGE = "You are already authorized with {0} account." diff --git a/platformio/package/exception.py b/platformio/package/exception.py index d38c0e602b..ba896faef8 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -13,10 +13,10 @@ # limitations under the License. from platformio import util -from platformio.exception import PlatformioException, UserSideException +from platformio.exception import UserSideException -class PackageException(PlatformioException): +class PackageException(UserSideException): pass @@ -51,14 +51,14 @@ class MissingPackageManifestError(ManifestException): MESSAGE = "Could not find one of '{0}' manifest files in the package" -class UnknownPackageError(UserSideException): +class UnknownPackageError(PackageException): MESSAGE = ( "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() ) -class NotGlobalLibDir(UserSideException): +class NotGlobalLibDir(PackageException): MESSAGE = ( "The `{0}` is not a PlatformIO project.\n\n" "To manage libraries in global storage `{1}`,\n" diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index c10235c73b..67348391c5 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -18,7 +18,7 @@ from urllib.parse import urlparse from platformio import proc -from platformio.package.exception import PlatformioException, UserSideException +from platformio.exception import UserSideException class VCSBaseException(UserSideException): @@ -70,7 +70,7 @@ def check_client(self): self.get_cmd_output(["--version"]) else: assert self.run_cmd(["--version"]) - except (AssertionError, OSError, PlatformioException) as exc: + except (AssertionError, OSError) as exc: raise VCSBaseException( "VCS: `%s` client is not installed in your system" % self.command ) from exc diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index f044d3b7a7..18ace0dee0 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import PlatformioException +from platformio.exception import UserSideException -class PlatformException(PlatformioException): +class PlatformException(UserSideException): pass diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 546cd67063..cb004d23ca 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -180,7 +180,7 @@ def _load_build_metadata(project_dir, env_names, debug=False): ): raise result.exception if '"includes":' not in result.output: - raise exception.PlatformioException(result.output) + raise exception.UserSideException(result.output) return _get_cached_build_metadata(project_dir, env_names) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 0c8adc5233..d288872ae9 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -17,6 +17,7 @@ import os import queue import re +import sys import threading import time import traceback @@ -271,6 +272,7 @@ def _strip_module_path(match): "name": exc.__class__.__name__, "description": str(exc), "traceback": trace, + "cmd_args": sys.argv[1:], "is_fatal": is_fatal, } log_event("exception", params) From 3a230dfb51e3700e405561bcafae5534ac5da509 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Jun 2023 14:24:47 +0300 Subject: [PATCH 32/69] Handle PlatformIO's "on_exit" event --- platformio/__main__.py | 10 ++++++++-- platformio/maintenance.py | 11 +++++++---- platformio/telemetry.py | 4 ++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index acc50ad87d..909a026076 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -53,7 +53,13 @@ def cli(ctx, force, caller, no_ansi): # pylint: disable=unused-argument except: # pylint: disable=bare-except pass - maintenance.on_platformio_start(ctx, caller) + maintenance.on_cmd_start(ctx, caller) + + +@cli.result_callback() +@click.pass_context +def process_result(*_, **__): + maintenance.on_cmd_end() def configure(): @@ -125,7 +131,7 @@ def main(argv=None): click.secho(error_str, fg="red", err=True) exit_code = int(str(exc)) if str(exc).isdigit() else 1 - maintenance.on_platformio_end() + maintenance.on_platformio_exit() sys.argv = prev_sys_argv return exit_code diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 936a0d8626..174f1f9650 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -29,17 +29,16 @@ from platformio.system.prune import calculate_unnecessary_system_data -def on_platformio_start(ctx, caller): +def on_cmd_start(ctx, caller): app.set_session_var("command_ctx", ctx) set_caller(caller) - telemetry.on_platformio_start(ctx) + telemetry.on_cmd_start(ctx) if PlatformioCLI.in_silence(): return after_upgrade(ctx) -def on_platformio_end(): - telemetry.on_platformio_end() +def on_cmd_end(): if PlatformioCLI.in_silence(): return @@ -62,6 +61,10 @@ def on_platformio_exception(exc): telemetry.log_exception(exc) +def on_platformio_exit(): + telemetry.on_exit() + + def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") if not caller: diff --git a/platformio/telemetry.py b/platformio/telemetry.py index d288872ae9..820b1507a6 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -170,12 +170,12 @@ def log_event(name, params, instant_sending=False): TelemetryLogger().log_event(name, params, instant_sending=instant_sending) -def on_platformio_start(cmd_ctx): +def on_cmd_start(cmd_ctx): process_postponed_logs() log_command(cmd_ctx) -def on_platformio_end(): +def on_exit(): TelemetryLogger().send() From 425332040e516bb7ed45d4dcff6105572cf028d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Jun 2023 14:25:21 +0300 Subject: [PATCH 33/69] Bump version to 6.1.8a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6ad687229d..5133bf86d2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8a4") +VERSION = (6, 1, "8a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From a5052433f2c38f94b43ba95c0483204b4c6ad440 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Jun 2023 16:49:14 +0300 Subject: [PATCH 34/69] Refactored Unit Testing engine to resolve compiler warnings with "-Wpedantic" option // Resolve #4671 --- HISTORY.rst | 1 + platformio/test/runners/unity.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f277cbdac0..47aab2c11e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 6 * Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) +* Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index 25fea0ab62..935a1328c9 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -55,8 +55,8 @@ class UnityTestRunner(TestRunnerBase): void unityOutputStart(unsigned long); void unityOutputChar(unsigned int); -void unityOutputFlush(); -void unityOutputComplete(); +void unityOutputFlush(void); +void unityOutputComplete(void); #define UNITY_OUTPUT_START() unityOutputStart((unsigned long) $baudrate) #define UNITY_OUTPUT_CHAR(c) unityOutputChar(c) @@ -246,18 +246,20 @@ def generate_unity_extras(self, dst_dir): unity_h = dst_dir / "unity_config.h" if not unity_h.is_file(): unity_h.write_text( - string.Template(self.UNITY_CONFIG_H).substitute( - baudrate=self.get_test_speed() - ), + string.Template(self.UNITY_CONFIG_H) + .substitute(baudrate=self.get_test_speed()) + .strip() + + "\n", encoding="utf8", ) framework_config = self.get_unity_framework_config() unity_c = dst_dir / ("unity_config.%s" % framework_config.get("language", "c")) if not unity_c.is_file(): unity_c.write_text( - string.Template(self.UNITY_CONFIG_C).substitute( - framework_config_code=framework_config["code"] - ), + string.Template(self.UNITY_CONFIG_C) + .substitute(framework_config_code=framework_config["code"]) + .strip() + + "\n", encoding="utf8", ) From 5ffa42a5a2f9466ce7eac53e9e116a74f8ee34b6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Jun 2023 19:01:05 +0300 Subject: [PATCH 35/69] Eliminated erroneous warning regarding the use of obsolete PlatformIO Core // Resolve #4664 --- HISTORY.rst | 1 + platformio/maintenance.py | 88 +++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 47aab2c11e..9da4a37043 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,7 @@ PlatformIO Core 6 * Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) +* Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 174f1f9650..45b4e5c95c 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -80,9 +80,8 @@ def set_caller(caller=None): class Upgrader: def __init__(self, from_version, to_version): - self.from_version = pepver_to_semver(from_version) - self.to_version = pepver_to_semver(to_version) - + self.from_version = from_version + self.to_version = to_version self._upgraders = [ (semantic_version.Version("6.1.8-a.1"), self._appstate_migration), ] @@ -121,17 +120,22 @@ def _appstate_migration(_): def after_upgrade(ctx): terminal_width = shutil.get_terminal_size().columns - last_version = app.get_state_item("last_version", "0.0.0") - if last_version == __version__: - return + last_version_str = app.get_state_item("last_version", "0.0.0") + if last_version_str == __version__: + return None - if last_version == "0.0.0": + if last_version_str == "0.0.0": app.set_state_item("last_version", __version__) - elif pepver_to_semver(last_version) > pepver_to_semver(__version__): + return print_welcome_banner() + + last_version = pepver_to_semver(last_version_str) + current_version = pepver_to_semver(__version__) + + if last_version > current_version and not last_version.prerelease: click.secho("*" * terminal_width, fg="yellow") click.secho( "Obsolete PIO Core v%s is used (previous was %s)" - % (__version__, last_version), + % (__version__, last_version_str), fg="yellow", ) click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") @@ -141,46 +145,50 @@ def after_upgrade(ctx): fg="cyan", ) click.secho("*" * terminal_width, fg="yellow") - return - else: - click.secho("Please wait while upgrading PlatformIO...", fg="yellow") - - # Update PlatformIO's Core packages - cleanup_content_cache("http") - update_core_packages() - - u = Upgrader(last_version, __version__) - if u.run(ctx): - app.set_state_item("last_version", __version__) - click.secho( - "PlatformIO has been successfully upgraded to %s!\n" % __version__, - fg="green", - ) - telemetry.log_event( - "pio_upgrade_core", - { - "label": "%s > %s" % (last_version, __version__), - "from_version": last_version, - "to_version": __version__, - }, - ) + return None + + click.secho("Please wait while upgrading PlatformIO...", fg="yellow") + + # Update PlatformIO's Core packages + cleanup_content_cache("http") + update_core_packages() - # PlatformIO banner + u = Upgrader(last_version, current_version) + if u.run(ctx): + app.set_state_item("last_version", __version__) + click.secho( + "PlatformIO has been successfully upgraded to %s!\n" % __version__, + fg="green", + ) + telemetry.log_event( + "pio_upgrade_core", + { + "label": "%s > %s" % (last_version_str, __version__), + "from_version": last_version_str, + "to_version": __version__, + }, + ) + + return print_welcome_banner() + + +def print_welcome_banner(): + terminal_width = shutil.get_terminal_size().columns click.echo("*" * terminal_width) click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) click.echo( - "- %s us on Twitter to stay up-to-date " - "on the latest project news > %s" + "- %s it on GitHub > %s" % ( - click.style("follow", fg="cyan"), - click.style("https://twitter.com/PlatformIO_Org", fg="cyan"), + click.style("star", fg="cyan"), + click.style("https://github.com/platformio/platformio-core", fg="cyan"), ) ) click.echo( - "- %s it on GitHub > %s" + "- %s us on LinkedIn to stay up-to-date " + "on the latest project news > %s" % ( - click.style("star", fg="cyan"), - click.style("https://github.com/platformio/platformio", fg="cyan"), + click.style("follow", fg="cyan"), + click.style("https://www.linkedin.com/company/platformio/", fg="cyan"), ) ) if not os.getenv("PLATFORMIO_IDE"): From e9cf551101c6eff118554627e2ce89134d36f7a6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Jun 2023 14:25:30 +0300 Subject: [PATCH 36/69] Added a new --lint option to the pio project config command // Resolve #4644 --- HISTORY.rst | 1 + platformio/project/commands/config.py | 53 +++++++++++++++++++++++++-- platformio/project/config.py | 36 +++++++++++++++++- tests/project/test_config.py | 31 ++++++++++++++++ 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9da4a37043..c5b2c46e2c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO Core 6 6.1.8 (2023-??-??) ~~~~~~~~~~~~~~~~~~ +* Added a new ``--lint`` option to the `pio project config `__ command, enabling users to efficiently perform linting on the |PIOCONF| * Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) diff --git a/platformio/project/commands/config.py b/platformio/project/commands/config.py index 9f218e8024..5e4ed21fd6 100644 --- a/platformio/project/commands/config.py +++ b/platformio/project/commands/config.py @@ -30,16 +30,23 @@ default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True), ) +@click.option("--lint", is_flag=True) @click.option("--json-output", is_flag=True) -def project_config_cmd(project_dir, json_output): +def project_config_cmd(project_dir, lint, json_output): if not is_platformio_project(project_dir): raise NotPlatformIOProjectError(project_dir) with fs.cd(project_dir): - config = ProjectConfig.get_instance() + if lint: + return lint_configuration(json_output) + return print_configuration(json_output) + + +def print_configuration(json_output=False): + config = ProjectConfig.get_instance() if json_output: return click.echo(config.to_json()) click.echo( - "Computed project configuration for %s" % click.style(project_dir, fg="cyan") + "Computed project configuration for %s" % click.style(os.getcwd(), fg="cyan") ) for section, options in config.as_tuple(): click.secho(section, fg="cyan") @@ -55,3 +62,43 @@ def project_config_cmd(project_dir, json_output): ) click.echo() return None + + +def lint_configuration(json_output=False): + result = ProjectConfig.lint() + errors = result["errors"] + warnings = result["warnings"] + if json_output: + return click.echo(result) + if not errors and not warnings: + return click.secho( + 'The "platformio.ini" configuration file is free from linting errors.', + fg="green", + ) + if errors: + click.echo( + tabulate( + [ + ( + click.style(error["type"], fg="red"), + error["message"], + error.get("source", "") + (f":{error.get('lineno')}") + if "lineno" in error + else "", + ) + for error in errors + ], + tablefmt="plain", + ) + ) + if warnings: + click.echo( + tabulate( + [ + (click.style("Warning", fg="yellow"), warning) + for warning in warnings + ], + tablefmt="plain", + ) + ) + return None diff --git a/platformio/project/config.py b/platformio/project/config.py index 08ab422cb5..4658acfa9d 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -433,7 +433,41 @@ def get_optional_dir(self, name): return self.get("platformio", f"{name}_dir") -class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): +class ProjectConfigLintMixin: + @classmethod + def lint(cls, path=None): + errors = [] + warnings = [] + try: + config = cls.get_instance(path) + config.validate(silent=True) + warnings = config.warnings + config.as_tuple() + except Exception as exc: # pylint: disable=broad-exception-caught + if exc.__cause__ is not None: + exc = exc.__cause__ + + item = {"type": exc.__class__.__name__, "message": str(exc)} + for attr in ("lineno", "source"): + if hasattr(exc, attr): + item[attr] = getattr(exc, attr) + + if item["type"] == "ParsingError" and hasattr(exc, "errors"): + for lineno, line in getattr(exc, "errors"): + errors.append( + { + "type": item["type"], + "message": f"Parsing error: {line}", + "lineno": lineno, + "source": item["source"], + } + ) + else: + errors.append(item) + return {"errors": errors, "warnings": warnings} + + +class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin, ProjectConfigLintMixin): _instances = {} @staticmethod diff --git a/tests/project/test_config.py b/tests/project/test_config.py index f559502ed3..6b74285538 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -684,3 +684,34 @@ def test_invalid_env_names(tmp_path: Path): config = ProjectConfig(str(project_conf)) with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): config.validate() + + +def test_linting_errors(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app1] +lib_use = 1 +broken_line + """ + ) + result = ProjectConfig.lint(str(project_conf)) + assert not result["warnings"] + assert result["errors"] and len(result["errors"]) == 1 + error = result["errors"][0] + assert error["type"] == "ParsingError" + assert error["lineno"] == 4 + + +def test_linting_warnings(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app1] +lib_use = 1 + """ + ) + result = ProjectConfig.lint(str(project_conf)) + assert not result["errors"] + assert result["warnings"] and len(result["warnings"]) == 1 + assert "deprecated" in result["warnings"][0] From 3881a8c677477cf96f6a7ec8b84a043ff0d53a54 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Jun 2023 14:47:20 +0300 Subject: [PATCH 37/69] Removed PlatformIO IDE for Atom from the documentation as Atom has been deprecated --- HISTORY.rst | 1 + docs | 2 +- platformio/project/config.py | 11 +---------- tox.ini | 2 +- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c5b2c46e2c..adf42844d2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 6 * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) * Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) +* Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated `__ 6.1.7 (2023-05-08) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 4fab8cfb59..7b02416590 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4fab8cfb5926210b865eb0afb1ce01460d35f515 +Subproject commit 7b02416590fed8484ee9c6fbefcc2f9901ddb325 diff --git a/platformio/project/config.py b/platformio/project/config.py index 4658acfa9d..414dada3d3 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -424,15 +424,6 @@ def validate(self, envs=None, silent=False): return True -class ProjectConfigDirsMixin: - def get_optional_dir(self, name): - """ - Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs - PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0 - """ - return self.get("platformio", f"{name}_dir") - - class ProjectConfigLintMixin: @classmethod def lint(cls, path=None): @@ -467,7 +458,7 @@ def lint(cls, path=None): return {"errors": errors, "warnings": warnings} -class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin, ProjectConfigLintMixin): +class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin): _instances = {} @staticmethod diff --git a/tox.ini b/tox.ini index fe24f9c6a6..f8a15f3bc5 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ commands = [testenv:docs] deps = sphinx - sphinx-rtd-theme==1.2.0 + sphinx-rtd-theme==1.2.2 sphinx-notfound-page sphinx-copybutton restructuredtext-lint From 4d89593b05fdd7c18161d3870cf08a0fa2df075a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Jun 2023 17:53:37 +0300 Subject: [PATCH 38/69] Removed PlatformIO IDE for Atom from the documentation as Atom has been deprecated --- .../project/integration/tpls/atom/.clang_complete.tpl | 6 ------ .../project/integration/tpls/atom/.gcc-flags.json.tpl | 9 --------- platformio/project/integration/tpls/atom/.gitignore.tpl | 3 --- 3 files changed, 18 deletions(-) delete mode 100644 platformio/project/integration/tpls/atom/.clang_complete.tpl delete mode 100644 platformio/project/integration/tpls/atom/.gcc-flags.json.tpl delete mode 100644 platformio/project/integration/tpls/atom/.gitignore.tpl diff --git a/platformio/project/integration/tpls/atom/.clang_complete.tpl b/platformio/project/integration/tpls/atom/.clang_complete.tpl deleted file mode 100644 index 6d8e70ed23..0000000000 --- a/platformio/project/integration/tpls/atom/.clang_complete.tpl +++ /dev/null @@ -1,6 +0,0 @@ -% for include in filter_includes(includes): --I{{include}} -% end -% for define in defines: --D{{!define}} -% end diff --git a/platformio/project/integration/tpls/atom/.gcc-flags.json.tpl b/platformio/project/integration/tpls/atom/.gcc-flags.json.tpl deleted file mode 100644 index 85b4e9dabc..0000000000 --- a/platformio/project/integration/tpls/atom/.gcc-flags.json.tpl +++ /dev/null @@ -1,9 +0,0 @@ -% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines]) -{ - "execPath": "{{ cxx_path }}", - "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccErrorLimit": 15, - "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}", - "gccSuppressWarnings": false -} diff --git a/platformio/project/integration/tpls/atom/.gitignore.tpl b/platformio/project/integration/tpls/atom/.gitignore.tpl deleted file mode 100644 index bbdd36c798..0000000000 --- a/platformio/project/integration/tpls/atom/.gitignore.tpl +++ /dev/null @@ -1,3 +0,0 @@ -.pio -.clang_complete -.gcc-flags.json From f219f35ac8bac251f0bc28fba862e0b6a82856be Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Jun 2023 20:55:00 +0300 Subject: [PATCH 39/69] Updated the "pio project metadata" command to return C/C++ flags as parsed Unix shell arguments --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piointegration.py | 9 +++- platformio/check/tools/base.py | 4 +- platformio/compat.py | 8 +++ platformio/project/commands/metadata.py | 1 + .../tpls/clion/CMakeListsPrivate.txt.tpl | 14 ++--- .../.settings/language.settings.xml.tpl | 4 +- .../project/integration/tpls/emacs/.ccls.tpl | 6 ++- .../tpls/qtcreator/platformio.cflags.tpl | 4 +- .../tpls/qtcreator/platformio.cxxflags.tpl | 4 +- .../integration/tpls/sublimetext/.ccls.tpl | 6 ++- .../project/integration/tpls/vim/.ccls.tpl | 6 ++- .../vscode/.vscode/c_cpp_properties.json.tpl | 53 ++++++++----------- tests/commands/test_init.py | 2 +- 15 files changed, 69 insertions(+), 55 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index adf42844d2..7aa40fdee7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 6 * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) * Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) +* Updated the `pio project metadata `__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata * Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated `__ 6.1.7 (2023-05-08) diff --git a/docs b/docs index 7b02416590..45e6fad5dc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7b02416590fed8484ee9c6fbefcc2f9901ddb325 +Subproject commit 45e6fad5dcd601e88aec0411c0ed76e9849ef7a6 diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py index 3e27dd365c..25ca1eb070 100644 --- a/platformio/builder/tools/piointegration.py +++ b/platformio/builder/tools/piointegration.py @@ -16,6 +16,7 @@ import glob import os +import click import SCons.Defaults # pylint: disable=import-error import SCons.Subst # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error @@ -154,8 +155,12 @@ def DumpIntegrationData(*args): ], "defines": dump_defines(projenv), "includes": projenv.DumpIntegrationIncludes(), - "cc_flags": _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"), - "cxx_flags": _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), + "cc_flags": click.parser.split_arg_string( + _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS") + ), + "cxx_flags": click.parser.split_arg_string( + _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS") + ), "cc_path": where_is_program( globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}") ), diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index 77f0e55c16..ccf331f453 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -60,8 +60,8 @@ def _load_cpp_data(self): data = load_build_metadata(self.project_dir, self.envname) if not data: return - self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", "")) - self.cxx_flags = click.parser.split_arg_string(data.get("cxx_flags", "")) + self.cc_flags = data.get("cc_flags", []) + self.cxx_flags = data.get("cxx_flags", []) self.cpp_includes = self._dump_includes(data.get("includes", {})) self.cpp_defines = data.get("defines", []) self.cc_path = data.get("cc_path") diff --git a/platformio/compat.py b/platformio/compat.py index 17b4cc0aff..694783c2d8 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -17,6 +17,7 @@ import importlib.util import inspect import locale +import shlex import sys from platformio.exception import UserSideException @@ -36,6 +37,13 @@ MISSING = object() string_types = (str,) +try: + from shlex import join as shlex_join +except ImportError: + + def shlex_join(split_command): + return " ".join(shlex.quote(arg) for arg in split_command) + def is_bytes(x): return isinstance(x, (bytes, memoryview, bytearray)) diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py index fbf71b8bbd..ee9f5b194c 100644 --- a/platformio/project/commands/metadata.py +++ b/platformio/project/commands/metadata.py @@ -37,6 +37,7 @@ @click.option("--json-output", is_flag=True) @click.option("--json-output-path", type=click.Path()) def project_metadata_cmd(project_dir, environments, json_output, json_output_path): + project_dir = os.path.abspath(project_dir) with fs.cd(project_dir): config = ProjectConfig.get_instance() config.validate(environments) diff --git a/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl index 6297706f9c..c65848f6b9 100644 --- a/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/project/integration/tpls/clion/CMakeListsPrivate.txt.tpl @@ -8,6 +8,7 @@ % import os % import re % +% from platformio.compat import shlex_join % from platformio.project.helpers import load_build_metadata % % def _normalize_path(path): @@ -64,17 +65,16 @@ set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Periph SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") -SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}") -SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}") +SET(CMAKE_CXX_FLAGS {{ _normalize_path(to_unix_path(shlex_join(cxx_flags))) }}) +SET(CMAKE_C_FLAGS {{ _normalize_path(to_unix_path(shlex_join(cc_flags))) }}) -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) +% cc_stds = [arg for arg in cc_flags if arg.startswith("-std=")] +% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")] % if cc_stds: -SET(CMAKE_C_STANDARD {{ cc_stds[-1] }}) +SET(CMAKE_C_STANDARD {{ cc_stds[-1][-2:] }}) % end % if cxx_stds: -set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }}) +set(CMAKE_CXX_STANDARD {{ cxx_stds[-1][-2:] }}) % end if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}") diff --git a/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl index d8826fca80..f2123be679 100644 --- a/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl @@ -1,6 +1,4 @@ -% import re -% STD_RE = re.compile(r"(\-std=[a-z\+]+\w+)") -% cxx_stds = STD_RE.findall(cxx_flags) +% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")] % cxx_std = cxx_stds[-1] if cxx_stds else "" % % if cxx_path.startswith(user_home_dir): diff --git a/platformio/project/integration/tpls/emacs/.ccls.tpl b/platformio/project/integration/tpls/emacs/.ccls.tpl index a747bb61b2..94d376802a 100644 --- a/platformio/project/integration/tpls/emacs/.ccls.tpl +++ b/platformio/project/integration/tpls/emacs/.ccls.tpl @@ -1,7 +1,9 @@ +% from platformio.compat import shlex_join +% clang -{{"%c"}} {{ !cc_flags }} -{{"%cpp"}} {{ !cxx_flags }} +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} diff --git a/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl index f09a94f906..024c813ddc 100644 --- a/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl +++ b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl @@ -1 +1,3 @@ -{{cc_flags.replace('-mlongcalls', '-mlong-calls')}} +% from platformio.compat import shlex_join +% +{{shlex_join(cc_flags).replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl index c5b3051128..551f02e1ab 100644 --- a/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl +++ b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl @@ -1 +1,3 @@ -{{cxx_flags.replace('-mlongcalls', '-mlong-calls')}} +% from platformio.compat import shlex_join +% +{{shlex_join(cxx_flags).replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/project/integration/tpls/sublimetext/.ccls.tpl b/platformio/project/integration/tpls/sublimetext/.ccls.tpl index a747bb61b2..94d376802a 100644 --- a/platformio/project/integration/tpls/sublimetext/.ccls.tpl +++ b/platformio/project/integration/tpls/sublimetext/.ccls.tpl @@ -1,7 +1,9 @@ +% from platformio.compat import shlex_join +% clang -{{"%c"}} {{ !cc_flags }} -{{"%cpp"}} {{ !cxx_flags }} +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} diff --git a/platformio/project/integration/tpls/vim/.ccls.tpl b/platformio/project/integration/tpls/vim/.ccls.tpl index a747bb61b2..94d376802a 100644 --- a/platformio/project/integration/tpls/vim/.ccls.tpl +++ b/platformio/project/integration/tpls/vim/.ccls.tpl @@ -1,7 +1,9 @@ +% from platformio.compat import shlex_join +% clang -{{"%c"}} {{ !cc_flags }} -{{"%cpp"}} {{ !cxx_flags }} +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} % for include in filter_includes(includes): -I{{ !include }} diff --git a/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl index ac525e56cd..9abadeef09 100644 --- a/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -1,27 +1,20 @@ % import os % import platform -% import re -% -% import click % % systype = platform.system().lower() % % cpp_standards_remap = { -% "0x": "11", -% "1y": "14", -% "1z": "17", -% "2a": "20", -% "2b": "23" +% "c++0x": "c++11", +% "c++1y": "c++14", +% "c++1z": "c++17", +% "c++2a": "c++20", +% "c++2b": "c++23" % } % % def _escape(text): % return to_unix_path(text).replace('"', '\\"') % end % -% def split_args(args_string): -% return click.parser.split_arg_string(to_unix_path(args_string)) -% end -% % def filter_args(args, allowed, ignore=None): % if not allowed: % return [] @@ -31,18 +24,18 @@ % result = [] % i = 0 % length = len(args) -% while(i < length): -% if any(args[i].startswith(f) for f in allowed) and not any( -% args[i].startswith(f) for f in ignore): -% result.append(args[i]) -% if i + 1 < length and not args[i + 1].startswith("-"): -% i += 1 -% result.append(args[i]) -% end +% while(i < length): +% if any(args[i].startswith(f) for f in allowed) and not any( +% args[i].startswith(f) for f in ignore): +% result.append(args[i]) +% if i + 1 < length and not args[i + 1].startswith("-"): +% i += 1 +% result.append(args[i]) % end -% i += 1 -% end -% return result +% end +% i += 1 +% end +% return result % end % % def _find_abs_path(inc, inc_paths): @@ -76,12 +69,10 @@ % % cleaned_includes = filter_includes(includes, ["toolchain"]) % -% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") -% cc_stds = STD_RE.findall(cc_flags) -% cxx_stds = STD_RE.findall(cxx_flags) -% cc_m_flags = split_args(cc_flags) +% cc_stds = [arg[5:] for arg in cc_flags if arg.startswith("-std=")] +% cxx_stds = [arg[5:] for arg in cxx_flags if arg.startswith("-std=")] % forced_includes = _find_forced_includes( -% filter_args(cc_m_flags, ["-include", "-imacros"]), cleaned_includes) +% filter_args(cc_flags, ["-include", "-imacros"]), cleaned_includes) % // // !!! WARNING !!! AUTO-GENERATED FILE! @@ -114,10 +105,10 @@ "" ], % if cc_stds: - "cStandard": "c{{ cc_stds[-1] }}", + "cStandard": "{{ cc_stds[-1] }}", % end % if cxx_stds: - "cppStandard": "c++{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", + "cppStandard": "{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", % end % if forced_includes: "forcedInclude": [ @@ -130,7 +121,7 @@ "compilerPath": "{{ cc_path }}", "compilerArgs": [ % for flag in [ -% f for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) +% f for f in filter_args(cc_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) % ]: "{{ flag }}", % end diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index b3d9f0133a..80ef0d9194 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -62,7 +62,7 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): - result = clirunner.invoke(project_init_cmd, ["--ide", "atom"]) + result = clirunner.invoke(project_init_cmd, ["--ide", "vscode"]) assert result.exit_code != 0 assert isinstance(result.exception, ProjectEnvsNotAvailableError) From 8b604c1a03ce6731e453bad762e40aee57df7334 Mon Sep 17 00:00:00 2001 From: Dawid Nowak Date: Tue, 20 Jun 2023 20:03:57 +0200 Subject: [PATCH 40/69] fix typo in a method name (#4672) --- platformio/project/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 414dada3d3..2cb982fb5b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -86,7 +86,7 @@ def __init__(self, path=None, parse_extra=True, expand_interpolations=True): if path and os.path.isfile(path): self.read(path, parse_extra) - self._maintain_renaimed_options() + self._maintain_renamed_options() def __getattr__(self, name): return getattr(self._parser, name) @@ -110,7 +110,7 @@ def read(self, path, parse_extra=True): for item in glob.glob(pattern, recursive=True): self.read(item) - def _maintain_renaimed_options(self): + def _maintain_renamed_options(self): renamed_options = {} for option in ProjectOptions.values(): if option.oldnames: From 326ebcf593236cd21ef7803ce19edc86d4bed790 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Jun 2023 12:45:28 +0300 Subject: [PATCH 41/69] Support -std=gnu++2a and similar standards // Issue #3653 --- .../tpls/vscode/.vscode/c_cpp_properties.json.tpl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 9abadeef09..0be8a3b480 100644 --- a/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -8,7 +8,12 @@ % "c++1y": "c++14", % "c++1z": "c++17", % "c++2a": "c++20", -% "c++2b": "c++23" +% "c++2b": "c++23", +% "gnu++0x": "gnu++11", +% "gnu++1y": "gnu++14", +% "gnu++1z": "gnu++17", +% "gnu++2a": "gnu++20", +% "gnu++2b": "gnu++23" % } % % def _escape(text): From e25b170b34cadcd0eef1b477648c677144e8f2e5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Jun 2023 13:47:45 +0300 Subject: [PATCH 42/69] Show error name when raising UserSideException --- platformio/__main__.py | 2 +- platformio/package/meta.py | 17 ++++++++++------- platformio/package/version.py | 8 +++++++- platformio/project/exception.py | 18 +++++++++--------- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 909a026076..58cabe8b66 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -107,7 +107,7 @@ def main(argv=None): except Exception as exc: # pylint: disable=broad-except if not isinstance(exc, exception.ReturnErrorCode): maintenance.on_platformio_exception(exc) - error_str = "Error: " + error_str = f"{exc.__class__.__name__}: " if isinstance(exc, exception.PlatformioException): error_str += str(exc) else: diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 4db5310607..36578a3726 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -24,7 +24,7 @@ from platformio import fs from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType -from platformio.package.version import cast_version_to_semver +from platformio.package.version import SemanticVersionError, cast_version_to_semver from platformio.util import items_in_list @@ -175,7 +175,7 @@ def __init__( # pylint: disable=redefined-builtin,too-many-arguments if requirements: try: self.requirements = requirements - except ValueError as exc: + except SemanticVersionError as exc: if not self.name or self.uri or self.raw: raise exc self.raw = "%s=%s" % (self.name, requirements) @@ -224,11 +224,14 @@ def requirements(self, value): if not value: self._requirements = None return - self._requirements = ( - value - if isinstance(value, semantic_version.SimpleSpec) - else semantic_version.SimpleSpec(str(value)) - ) + try: + self._requirements = ( + value + if isinstance(value, semantic_version.SimpleSpec) + else semantic_version.SimpleSpec(str(value)) + ) + except ValueError as exc: + raise SemanticVersionError(exc) from exc def humanize(self): result = "" diff --git a/platformio/package/version.py b/platformio/package/version.py index 770be9e4b2..909d83e134 100644 --- a/platformio/package/version.py +++ b/platformio/package/version.py @@ -16,6 +16,12 @@ import semantic_version +from platformio.exception import UserSideException + + +class SemanticVersionError(UserSideException): + pass + def cast_version_to_semver(value, force=True, raise_exception=False): assert value @@ -29,7 +35,7 @@ def cast_version_to_semver(value, force=True, raise_exception=False): except ValueError: pass if raise_exception: - raise ValueError("Invalid SemVer version %s" % value) + raise SemanticVersionError("Invalid SemVer version %s" % value) # parse commit hash if re.match(r"^[\da-f]+$", value, flags=re.I): return semantic_version.Version("0.0.0+sha." + value) diff --git a/platformio/project/exception.py b/platformio/project/exception.py index 95681bc0df..d6e53fa6b6 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import PlatformioException, UserSideException +from platformio.exception import UserSideException -class ProjectError(PlatformioException): +class ProjectError(UserSideException): pass -class NotPlatformIOProjectError(ProjectError, UserSideException): +class NotPlatformIOProjectError(ProjectError): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " @@ -27,28 +27,28 @@ class NotPlatformIOProjectError(ProjectError, UserSideException): ) -class InvalidProjectConfError(ProjectError, UserSideException): +class InvalidProjectConfError(ProjectError): MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" -class UndefinedEnvPlatformError(ProjectError, UserSideException): +class UndefinedEnvPlatformError(ProjectError): MESSAGE = "Please specify platform for '{0}' environment" -class ProjectEnvsNotAvailableError(ProjectError, UserSideException): +class ProjectEnvsNotAvailableError(ProjectError): MESSAGE = "Please setup environments in `platformio.ini` file" -class UnknownEnvNamesError(ProjectError, UserSideException): +class UnknownEnvNamesError(ProjectError): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" -class InvalidEnvNameError(ProjectError, UserSideException): +class InvalidEnvNameError(ProjectError): MESSAGE = ( "Invalid environment name '{0}'. The name can contain " "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" ) -class ProjectOptionValueError(ProjectError, UserSideException): +class ProjectOptionValueError(ProjectError): MESSAGE = "{0} for option `{1}` in section [{2}]" From 31218060db80c2be8b6b22beebd48ec3b54a2532 Mon Sep 17 00:00:00 2001 From: Dawid Nowak Date: Wed, 21 Jun 2023 12:49:22 +0200 Subject: [PATCH 43/69] ProjectOptionValueError: displays the config option description (#4674) ProjectOptionValueError: display the config option description It gives a bit more context of the problem for the user. Example message with the description included: `Error: Invalid value: 'invalid_debug_mode' is not one of 'always', 'modified', 'manual'. for option `debug_load_mode` (Allows one to control when PlatformIO should load debugging firmware to the end target) in section [env:nodemcu]` Co-authored-by: Ivan Kravets --- platformio/project/config.py | 11 +++++++++-- platformio/project/exception.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 2cb982fb5b..4e1a667396 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -324,6 +324,7 @@ def _re_interpolation_handler(self, parent_section, match): f"`${{this.__env__}}` is called from the `{parent_section}` " "section that is not valid PlatformIO environment, see", option, + " ", section, ) return parent_section[4:] @@ -332,7 +333,10 @@ def _re_interpolation_handler(self, parent_section, match): value = self.get(section, option) except RecursionError as exc: raise exception.ProjectOptionValueError( - "Infinite recursion has been detected", option, section + "Infinite recursion has been detected", + option, + " ", + section, ) from exc if isinstance(value, list): return "\n".join(value) @@ -359,7 +363,10 @@ def get(self, section, option, default=MISSING): if not self.expand_interpolations: return value raise exception.ProjectOptionValueError( - exc.format_message(), option, section + exc.format_message(), + option, + " (%s) " % option_meta.description, + section, ) @staticmethod diff --git a/platformio/project/exception.py b/platformio/project/exception.py index d6e53fa6b6..91fc10bce5 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -51,4 +51,4 @@ class InvalidEnvNameError(ProjectError): class ProjectOptionValueError(ProjectError): - MESSAGE = "{0} for option `{1}` in section [{2}]" + MESSAGE = "{0} for option `{1}`{2}in section [{3}]" From 82de26d4012dd519d16ec4e88c81812646ebb0e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Jun 2023 13:56:50 +0300 Subject: [PATCH 44/69] Bump version to 6.1.8b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 5133bf86d2..1782750510 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8a5") +VERSION = (6, 1, "8b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e022b67161dc14c25b9c28d18c4452e5720b8578 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Jun 2023 16:31:05 +0300 Subject: [PATCH 45/69] Import PlatformioException (espressif32 dev-platform monitor depends on it) --- platformio/project/exception.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/platformio/project/exception.py b/platformio/project/exception.py index 91fc10bce5..3821c865b6 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import UserSideException +from platformio.exception import PlatformioException, UserSideException -class ProjectError(UserSideException): +class ProjectError(PlatformioException): pass -class NotPlatformIOProjectError(ProjectError): +class NotPlatformIOProjectError(ProjectError, UserSideException): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " @@ -27,28 +27,28 @@ class NotPlatformIOProjectError(ProjectError): ) -class InvalidProjectConfError(ProjectError): +class InvalidProjectConfError(ProjectError, UserSideException): MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" -class UndefinedEnvPlatformError(ProjectError): +class UndefinedEnvPlatformError(ProjectError, UserSideException): MESSAGE = "Please specify platform for '{0}' environment" -class ProjectEnvsNotAvailableError(ProjectError): +class ProjectEnvsNotAvailableError(ProjectError, UserSideException): MESSAGE = "Please setup environments in `platformio.ini` file" -class UnknownEnvNamesError(ProjectError): +class UnknownEnvNamesError(ProjectError, UserSideException): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" -class InvalidEnvNameError(ProjectError): +class InvalidEnvNameError(ProjectError, UserSideException): MESSAGE = ( "Invalid environment name '{0}'. The name can contain " "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" ) -class ProjectOptionValueError(ProjectError): +class ProjectOptionValueError(ProjectError, UserSideException): MESSAGE = "{0} for option `{1}`{2}in section [{3}]" From 1d97982230a70f707e019696d3ac069b2bf17a16 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Jun 2023 16:31:43 +0300 Subject: [PATCH 46/69] Bump version to 6.1.8b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1782750510..3657c3b100 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8b1") +VERSION = (6, 1, "8b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From a754a28cd8be1c94372243ae4c5083cef97c2b8e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 22 Jun 2023 13:55:44 +0300 Subject: [PATCH 47/69] Unlock "urllib3" package version if SSL module is supported // Issue #4614 --- setup.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e8fad3e17e..82241982d8 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,6 @@ "marshmallow==%s" % ("3.14.1" if PY36 else "3.*"), "pyelftools>=0.27,<1", "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" - "requests==2.*", - "urllib3<2", # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ "requests==%s" % ("2.27.1" if PY36 else "2.*"), "semantic_version==2.10.*", "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"), @@ -50,6 +48,20 @@ "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] +# issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ +try: + import ssl + + if ssl.OPENSSL_VERSION.startswith("OpenSSL ") and ssl.OPENSSL_VERSION_INFO < ( + 1, + 1, + 1, + ): + minimal_requirements.append("urllib3<2") +except ImportError: + pass + + setup( name=__title__, version=__version__, @@ -69,7 +81,7 @@ "project/integration/tpls/*/.*.tpl", # include hidden files "project/integration/tpls/*/.*/*.tpl", # include hidden folders "project/integration/tpls/*/*/*.tpl", # NetBeans - "project/integration/tpls/*/*/*/*.tpl", # NetBeans + "project/integration/tpls/*/*/*/*.tpl", # NetBeans ] }, entry_points={ From e78efff33b24c4c2da242a9118aad01739f51f07 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 22 Jun 2023 20:41:44 +0300 Subject: [PATCH 48/69] Allow the use of the underscore symbol in the "keywords" field --- HISTORY.rst | 1 + docs | 2 +- platformio/package/manifest/schema.py | 2 +- tests/package/test_manifest.py | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7aa40fdee7..8c1802935e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 6 * Added a new ``--lint`` option to the `pio project config `__ command, enabling users to efficiently perform linting on the |PIOCONF| * Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information +* Expanded the functionality of the |LIBRARYJSON| manifest by allowing the use of the underscore symbol in the `keywords `__ field * Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) * Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) diff --git a/docs b/docs index 45e6fad5dc..eaf1ae10f9 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 45e6fad5dcd601e88aec0411c0ed76e9849ef7a6 +Subproject commit eaf1ae10f9b2d3d37a5c55056dd7d17ba344d060 diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index f1ec94451b..95e081080b 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -183,7 +183,7 @@ class ManifestSchema(BaseSchema): validate=[ validate.Length(min=1, max=50), validate.Regexp( - r"^[a-z\d\-\+\. ]+$", error="Only [a-z0-9-+. ] chars are allowed" + r"^[a-z\d\-_\+\. ]+$", error="Only [a-z0-9+_-. ] chars are allowed" ), ] ) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index a3279e051d..bcc7e00a6f 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -28,7 +28,7 @@ def test_library_json_parser(): contents = """ { "name": "TestPackage", - "keywords": "kw1, KW2, kw3, KW2", + "keywords": "kw1, KW2, kw3, KW2, kw 4, kw_5, kw-6", "headers": "include1.h, Include2.hpp", "platforms": ["atmelavr", "espressif"], "repository": { @@ -62,7 +62,7 @@ def test_library_json_parser(): "url": "https://github.com/username/repo.git", }, "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, - "keywords": ["kw1", "kw2", "kw3"], + "keywords": ["kw1", "kw2", "kw3", "kw 4", "kw_5", "kw-6"], "headers": ["include1.h", "Include2.hpp"], "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, From 53f1d828906838363139b4eba7993649e80ba80f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 23 Jun 2023 12:15:42 +0300 Subject: [PATCH 49/69] Handle HTTP Client Error as UserSideException --- platformio/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/http.py b/platformio/http.py index e5c57835f2..da6abb4086 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -27,7 +27,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) -class HTTPClientError(PlatformioException): +class HTTPClientError(UserSideException): def __init__(self, message, response=None): super().__init__() self.message = message From f720cd841c3ff699407d12930376d44536586dd5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 24 Jun 2023 19:39:34 +0300 Subject: [PATCH 50/69] Use typex name instead of type_ --- platformio/package/commands/list.py | 20 ++++++++++---------- platformio/package/commands/publish.py | 14 +++++++------- platformio/registry/client.py | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index c2426c9c6c..eff7110001 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -135,20 +135,20 @@ def list_global_packages(options): ("libraries", LibraryPackageManager(options.get("storage_dir"))), ] only_packages = any( - options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data + options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data ) - for type_, pm in data: + for typex, pm in data: skip_conds = [ only_packages - and not options.get(type_) - and not options.get(f"only_{type_}"), + and not options.get(typex) + and not options.get(f"only_{typex}"), not pm.get_installed(), ] if any(skip_conds): continue - click.secho(type_.capitalize(), bold=True) + click.secho(typex.capitalize(), bold=True) print_dependency_tree( - pm, filter_specs=options.get(type_), verbose=options.get("verbose") + pm, filter_specs=options.get(typex), verbose=options.get("verbose") ) click.echo() @@ -156,12 +156,12 @@ def list_global_packages(options): def list_project_packages(options): environments = options["environments"] only_packages = any( - options.get(type_) or options.get(f"only_{type_}") - for type_ in ("platforms", "tools", "libraries") + options.get(typex) or options.get(f"only_{typex}") + for typex in ("platforms", "tools", "libraries") ) only_platform_packages = any( - options.get(type_) or options.get(f"only_{type_}") - for type_ in ("platforms", "tools") + options.get(typex) or options.get(f"only_{typex}") + for typex in ("platforms", "tools") ) only_library_packages = options.get("libraries") or options.get("only_libraries") diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index 228c1411ba..c8b4ea7774 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -56,7 +56,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument ) @click.option( "--type", - "type_", + "typex", type=click.Choice(list(PackageType.items().values())), help="Custom package type", ) @@ -83,7 +83,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument hidden=True, ) def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals - package, owner, type_, released_at, private, notify, no_interactive, non_interactive + package, owner, typex, released_at, private, notify, no_interactive, non_interactive ): click.secho("Preparing a package...", fg="cyan") no_interactive = no_interactive or non_interactive @@ -103,14 +103,14 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals p = PackagePacker(package) archive_path = p.pack() - type_ = type_ or PackageType.from_archive(archive_path) + typex = typex or PackageType.from_archive(archive_path) manifest = ManifestSchema().load_manifest( ManifestParserFactory.new_from_archive(archive_path).as_dict() ) name = manifest.get("name") version = manifest.get("version") data = [ - ("Type:", type_), + ("Type:", typex), ("Owner:", owner), ("Name:", name), ("Version:", version), @@ -124,13 +124,13 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals check_archive_file_names(archive_path) # look for duplicates - check_package_duplicates(owner, type_, name, version, manifest.get("system")) + check_package_duplicates(owner, typex, name, version, manifest.get("system")) if not no_interactive: click.confirm( "Are you sure you want to publish the %s %s to the registry?\n" % ( - type_, + typex, click.style( "%s/%s@%s" % (owner, name, version), fg="cyan", @@ -146,7 +146,7 @@ def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals ) click.echo("Publishing...") response = RegistryClient().publish_package( - owner, type_, archive_path, released_at, private, notify + owner, typex, archive_path, released_at, private, notify ) if not do_not_pack: os.remove(archive_path) diff --git a/platformio/registry/client.py b/platformio/registry/client.py index 1a89e345a2..6173f7dfe6 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -142,12 +142,12 @@ def list_packages(self, query=None, qualifiers=None, page=None, sort=None): x_with_authorization=self.allowed_private_packages(), ) - def get_package(self, type_, owner, name, version=None, extra_path=None): + def get_package(self, typex, owner, name, version=None, extra_path=None): try: return self.fetch_json_data( "get", "/v3/packages/{owner}/{type}/{name}{extra_path}".format( - type=type_, + type=typex, owner=owner.lower(), name=name.lower(), extra_path=extra_path or "", From 4dc44868eafeffe9abe4008870af29b612076b9f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 24 Jun 2023 19:40:54 +0300 Subject: [PATCH 51/69] Use native asyncio.to_thread if available --- platformio/compat.py | 21 ++++++++++++++------- platformio/home/rpc/handlers/os.py | 4 ++-- platformio/home/rpc/handlers/piocore.py | 6 +++--- platformio/home/rpc/handlers/registry.py | 4 ++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index 694783c2d8..008dc7ae01 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -30,6 +30,20 @@ from asyncio import get_event_loop as aio_get_running_loop +if sys.version_info >= (3, 8): + from shlex import join as shlex_join +else: + + def shlex_join(split_command): + return " ".join(shlex.quote(arg) for arg in split_command) + + +if sys.version_info >= (3, 9): + from asyncio import to_thread as aio_to_thread +else: + from starlette.concurrency import run_in_threadpool as aio_to_thread + + PY2 = sys.version_info[0] == 2 # DO NOT REMOVE IT. ESP8266/ESP32 depend on it IS_CYGWIN = sys.platform.startswith("cygwin") IS_WINDOWS = WINDOWS = sys.platform.startswith("win") @@ -37,13 +51,6 @@ MISSING = object() string_types = (str,) -try: - from shlex import join as shlex_join -except ImportError: - - def shlex_join(split_command): - return " ".join(shlex.quote(arg) for arg in split_command) - def is_bytes(x): return isinstance(x, (bytes, memoryview, bytearray)) diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 0d6b9f48ad..dd6511934b 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -19,10 +19,10 @@ from functools import cmp_to_key import click -from starlette.concurrency import run_in_threadpool from platformio import fs from platformio.cache import ContentCache +from platformio.compat import aio_to_thread from platformio.device.list.util import list_logical_devices from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.http import HTTPSession, ensure_internet_on @@ -33,7 +33,7 @@ async def request( # pylint: disable=signature-differs,invalid-overridden-metho self, *args, **kwargs ): func = super().request - return await run_in_threadpool(func, *args, **kwargs) + return await aio_to_thread(func, *args, **kwargs) class OSRPC(BaseRPCHandler): diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index 080ba96c7f..b09338e51b 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -22,13 +22,13 @@ import click from ajsonrpc.core import JSONRPC20DispatchException -from starlette.concurrency import run_in_threadpool from platformio import __main__, __version__, app, fs, proc, util from platformio.compat import ( IS_WINDOWS, aio_create_task, aio_get_running_loop, + aio_to_thread, get_locale_encoding, is_bytes, ) @@ -177,7 +177,7 @@ async def call(args, options=None): @staticmethod async def _call_subprocess(args, options): - result = await run_in_threadpool( + result = await aio_to_thread( proc.exec_command, [get_core_fullpath()] + args, cwd=options.get("cwd") or os.getcwd(), @@ -197,7 +197,7 @@ def _thread_safe_call(args, cwd): exit_code, ) - return await run_in_threadpool( + return await aio_to_thread( _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd() ) diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index 1370f32898..84a374b611 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -13,8 +13,8 @@ # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException -from starlette.concurrency import run_in_threadpool +from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.registry.client import RegistryClient @@ -24,7 +24,7 @@ class RegistryRPC(BaseRPCHandler): async def call_client(method, *args, **kwargs): try: client = RegistryClient() - return await run_in_threadpool(getattr(client, method), *args, **kwargs) + return await aio_to_thread(getattr(client, method), *args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( code=5000, message="Registry Call Error", data=str(exc) From 1b55da0af28d4795d091e0f8c79e1ac25ae21f6d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 27 Jun 2023 15:10:26 +0300 Subject: [PATCH 52/69] Force to installed dev-platform is available --- platformio/home/rpc/handlers/platform.py | 83 ++++++++++++++++++------ 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py index 2cc959df03..fe368a5adc 100644 --- a/platformio/home/rpc/handlers/platform.py +++ b/platformio/home/rpc/handlers/platform.py @@ -12,21 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. +from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory class PlatformRPC(BaseRPCHandler): + async def fetch_platforms(self, search_query=None, page=0, force_installed=False): + if force_installed: + return { + "items": await aio_to_thread( + self._load_installed_platforms, search_query + ) + } + + search_result = await self.factory.manager.dispatcher["registry.call_client"]( + method="list_packages", + query=search_query, + qualifiers={ + "types": ["platform"], + }, + page=page, + ) + return { + "page": search_result["page"], + "limit": search_result["limit"], + "total": search_result["total"], + "items": [ + { + "id": item["id"], + "name": item["name"], + "description": item["description"], + "tier": item["tier"], + "ownername": item["owner"]["username"], + "version": item["version"]["name"], + } + for item in search_result["items"] + ], + } + @staticmethod - def list_installed(options=None): - result = [] - options = options or {} + def _load_installed_platforms(search_query=None): + search_query = (search_query or "").strip() def _matchSearchQuery(p): - searchQuery = options.get("searchQuery") - if not searchQuery: - return True content_blocks = [p.name, p.title, p.description] if p.frameworks: content_blocks.append(" ".join(p.frameworks.keys())) @@ -34,27 +65,41 @@ def _matchSearchQuery(p): board_data = board.get_brief_data() for key in ("id", "mcu", "vendor"): content_blocks.append(board_data.get(key)) - return searchQuery.strip() in " ".join(content_blocks) + return search_query in " ".join(content_blocks) + items = [] pm = PlatformPackageManager() for pkg in pm.get_installed(): p = PlatformFactory.new(pkg) - if not _matchSearchQuery(p): + if search_query and not _matchSearchQuery(p): continue - result.append( - dict( - __pkg_path=pkg.path, - __pkg_meta=pkg.metadata.as_dict(), - name=p.name, - title=p.title, - description=p.description, - ) + items.append( + { + "__pkg_path": pkg.path, + "name": p.name, + "title": p.title, + "description": p.description, + "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None, + "version": str(pkg.metadata.version), + } + ) + return items + + async def fetch_boards(self, platform_spec): + spec = PackageSpec(platform_spec) + if spec.owner: + return await self.factory.manager.dispatcher["registry.call_client"]( + method="get_package", + typex="platform", + owner=spec.owner, + name=spec.name, + extra_path="/boards", ) - return result + return await aio_to_thread(self._load_installed_boards, spec) @staticmethod - def get_boards(spec): - p = PlatformFactory.new(spec) + def _load_installed_boards(platform_spec): + p = PlatformFactory.new(platform_spec) return sorted( [b.get_brief_data() for b in p.get_boards().values()], key=lambda item: item["name"], From 3fd063d8ed48a92196b926cad95ff793d0437a82 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 27 Jun 2023 16:03:37 +0300 Subject: [PATCH 53/69] Minor improvements --- platformio/device/finder.py | 2 +- platformio/exception.py | 2 +- platformio/http.py | 5 ++++- platformio/package/lockfile.py | 6 +++--- platformio/package/manager/_registry.py | 3 ++- platformio/platform/board.py | 4 ++-- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/platformio/device/finder.py b/platformio/device/finder.py index fc6ad3c48b..621adaece6 100644 --- a/platformio/device/finder.py +++ b/platformio/device/finder.py @@ -163,7 +163,7 @@ def _reveal_device_port(self, device): for item in list_serial_ports(as_objects=True): if item.vid == device.vid and item.pid == device.pid: candidates.append(item) - if len(candidates) == 1: + if len(candidates) <= 1: return device.device for item in candidates: if ("GDB" if self.prefer_gdb_port else "UART") in item.description: diff --git a/platformio/exception.py b/platformio/exception.py index bf2cec8c90..80ffb496bc 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -81,7 +81,7 @@ class InvalidSettingValue(UserSideException): MESSAGE = "Invalid value '{0}' for the setting '{1}'" -class InvalidJSONFile(PlatformioException): +class InvalidJSONFile(ValueError, UserSideException): MESSAGE = "Could not load broken JSON: {0}" diff --git a/platformio/http.py b/platformio/http.py index da6abb4086..ee67b631e9 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -157,7 +157,10 @@ def fetch_json_data(self, method, path, **kwargs): with ContentCache("http") as cc: result = cc.get(cache_key) if result is not None: - return json.loads(result) + try: + return json.loads(result) + except json.JSONDecodeError: + pass response = self.send_request(method, path, **kwargs) data = self._parse_json_response(response) cc.set(cache_key, response.text, cache_valid) diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 3c6b2047d5..bc2a4347ca 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -15,7 +15,7 @@ import os from time import sleep, time -from platformio.exception import PlatformioException +from platformio.exception import UserSideException LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour LOCKFILE_DELAY = 0.2 @@ -36,11 +36,11 @@ LOCKFILE_CURRENT_INTERFACE = None -class LockFileExists(PlatformioException): +class LockFileExists(UserSideException): pass -class LockFileTimeoutError(PlatformioException): +class LockFileTimeoutError(UserSideException): pass diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 494e2d2412..4c339be8a7 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -25,12 +25,13 @@ class PackageManagerRegistryMixin: def install_from_registry(self, spec, search_qualifiers=None): + package = version = None if spec.owner and spec.name and not search_qualifiers: package = self.fetch_registry_package(spec) if not package: raise UnknownPackageError(spec.humanize()) version = self.pick_best_registry_version(package["versions"], spec) - else: + elif spec.id or spec.name: packages = self.search_registry_packages(spec, search_qualifiers) if not packages: raise UnknownPackageError(spec.humanize()) diff --git a/platformio/platform/board.py b/platformio/platform/board.py index 4cd102d1b2..78d48943e6 100644 --- a/platformio/platform/board.py +++ b/platformio/platform/board.py @@ -17,7 +17,7 @@ from platformio import fs, util from platformio.compat import MISSING from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError -from platformio.exception import UserSideException +from platformio.exception import InvalidJSONFile, UserSideException from platformio.platform.exception import InvalidBoardManifest @@ -28,7 +28,7 @@ def __init__(self, manifest_path): self.manifest_path = manifest_path try: self._manifest = fs.load_json(manifest_path) - except ValueError as exc: + except InvalidJSONFile as exc: raise InvalidBoardManifest(manifest_path) from exc if not set(["name", "url", "vendor"]) <= set(self._manifest): raise UserSideException( From 9f0efdeb5c854d8fb85c78e65f44ace719af324a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 27 Jun 2023 16:03:59 +0300 Subject: [PATCH 54/69] Bump version to 6.1.8b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 3657c3b100..ab39768b6c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8b2") +VERSION = (6, 1, "8b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 98edf7609f59e35fcc0c214e205c280395afbcde Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jun 2023 21:26:00 +0300 Subject: [PATCH 55/69] Mock ajsonrpc.utils.is_invalid_params broken implementation --- platformio/home/rpc/server.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index 2437e40ec0..ab7c13dad2 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -14,6 +14,7 @@ from urllib.parse import parse_qs +import ajsonrpc.utils import click from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request from ajsonrpc.dispatcher import Dispatcher @@ -24,6 +25,10 @@ from platformio.http import InternetConnectionError from platformio.proc import force_exit +# Remove this line when PR is merged +# https://github.com/pavlov99/ajsonrpc/pull/22 +ajsonrpc.utils.is_invalid_params = lambda: False + class JSONRPCServerFactoryBase: connection_nums = 0 From 939b9b91129dad4ea73cd39726035df3e106c1b8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jun 2023 21:26:33 +0300 Subject: [PATCH 56/69] Support "file://" scheme for the requested URL --- platformio/home/rpc/handlers/os.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index dd6511934b..57bbdcef99 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -73,9 +73,9 @@ async def fetch_content(url, data=None, headers=None, cache_valid=None): async def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): return await self.fetch_content(uri, data, headers, cache_valid) - if os.path.isfile(uri): - with io.open(uri, encoding="utf-8") as fp: - return fp.read() + local_path = uri[7:] if uri.startswith("file://") else uri + with io.open(local_path, encoding="utf-8") as fp: + return fp.read() return None @staticmethod From 813861ddaec28c84d50bd3e0cdb850bb3f689422 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jun 2023 21:26:59 +0300 Subject: [PATCH 57/69] Implement new project.init_v2 endpoint --- platformio/home/rpc/handlers/project.py | 58 +++++++++++++++++++------ 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 904d00e4d3..65d05063a0 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -15,6 +15,7 @@ import os import shutil import time +from pathlib import Path import semantic_version from ajsonrpc.core import JSONRPC20DispatchException @@ -267,15 +268,39 @@ async def import_pio(project_dir): ) return new_project_dir - async def create_empty(self, configuration, options=None): + async def init_v2(self, configuration, options=None): project_dir = os.path.join(configuration["location"], configuration["name"]) if not os.path.isdir(project_dir): os.makedirs(project_dir) + envclone = os.environ.copy() + envclone["PLATFORMIO_FORCE_ANSI"] = "true" + options = options or {} + options["spawn"] = {"env": envclone, "cwd": project_dir} + + args = ["project", "init"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + + if configuration.get("example"): + await self.factory.notify_clients( + method=options.get("stdoutNotificationMethod"), + params=["Copying example files...\n"], + actor="frontend", + ) + await self._pre_init_example(configuration, project_dir) + else: + args.extend(self._pre_init_empty(configuration)) + + return await self.factory.manager.dispatcher["core.exec"](args, options=options) + + @staticmethod + def _pre_init_empty(configuration): project_options = [] platform = configuration["platform"] - board = configuration.get("board", {}).get("id") - env_name = board or platform["name"] + board_id = configuration.get("board", {}).get("id") + env_name = board_id or platform["name"] if configuration.get("description"): project_options.append(("description", configuration.get("description"))) try: @@ -288,20 +313,25 @@ async def create_empty(self, configuration, options=None): project_options.append( ("platform", "{name} @ {version}".format(**platform)) ) - if board: - project_options.append(("board", board)) + if board_id: + project_options.append(("board", board_id)) if configuration.get("framework"): project_options.append(("framework", configuration["framework"]["name"])) - args = ["project", "init", "-e", env_name, "--sample-code"] - ide = app.get_session_var("caller_id") - if ide in ProjectGenerator.get_supported_ides(): - args.extend(["--ide", ide]) + args = ["-e", env_name, "--sample-code"] for name, value in project_options: args.extend(["-O", f"{name}={value}"]) + return args - envclone = os.environ.copy() - envclone["PLATFORMIO_FORCE_ANSI"] = "true" - options = options or {} - options["spawn"] = {"env": envclone, "cwd": project_dir} - return await self.factory.manager.dispatcher["core.exec"](args, options=options) + async def _pre_init_example(self, configuration, project_dir): + for item in configuration["example"]["files"]: + p = Path(project_dir).joinpath(item["path"]) + if not p.parent.is_dir(): + p.parent.mkdir(parents=True) + p.write_text( + await self.factory.manager.dispatcher["os.request_content"]( + item["url"] + ), + encoding="utf-8", + ) + return [] From 450f48ba81e39c7a79e874556320e8e222ddba74 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jun 2023 21:28:00 +0300 Subject: [PATCH 58/69] Implement platform.fetch_examples endpoint --- platformio/home/rpc/handlers/platform.py | 42 +++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py index fe368a5adc..34daf5405a 100644 --- a/platformio/home/rpc/handlers/platform.py +++ b/platformio/home/rpc/handlers/platform.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path + from platformio.compat import aio_to_thread from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory @@ -43,11 +46,11 @@ async def fetch_platforms(self, search_query=None, page=0, force_installed=False "items": [ { "id": item["id"], + "ownername": item["owner"]["username"], "name": item["name"], + "version": item["version"]["name"], "description": item["description"], "tier": item["tier"], - "ownername": item["owner"]["username"], - "version": item["version"]["name"], } for item in search_result["items"] ], @@ -76,11 +79,11 @@ def _matchSearchQuery(p): items.append( { "__pkg_path": pkg.path, + "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None, "name": p.name, + "version": str(pkg.metadata.version), "title": p.title, "description": p.description, - "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None, - "version": str(pkg.metadata.version), } ) return items @@ -104,3 +107,34 @@ def _load_installed_boards(platform_spec): [b.get_brief_data() for b in p.get_boards().values()], key=lambda item: item["name"], ) + + async def fetch_examples(self, platform_spec): + spec = PackageSpec(platform_spec) + if spec.owner: + return await self.factory.manager.dispatcher["registry.call_client"]( + method="get_package", + typex="platform", + owner=spec.owner, + name=spec.name, + extra_path="/examples", + ) + return await aio_to_thread(self._load_installed_examples, spec) + + @staticmethod + def _load_installed_examples(platform_spec): + platform = PlatformFactory.new(platform_spec) + platform_dir = platform.get_dir() + parser = ManifestParserFactory.new_from_dir(platform_dir) + result = parser.as_dict().get("examples") or [] + for example in result: + example["files"] = [ + { + "path": item, + "url": ( + "file://%s" + + os.path.join(platform_dir, "examples", example["name"], item) + ), + } + for item in example["files"] + ] + return result From a28a3d31c9c70024c2bf47b6ba404569808d4bf0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jun 2023 21:28:46 +0300 Subject: [PATCH 59/69] Keep http session per active PIO Home --- platformio/home/rpc/handlers/os.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 57bbdcef99..b8dcfc9bfc 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -37,8 +37,10 @@ async def request( # pylint: disable=signature-differs,invalid-overridden-metho class OSRPC(BaseRPCHandler): - @staticmethod - async def fetch_content(url, data=None, headers=None, cache_valid=None): + _http_session = None + + @classmethod + async def fetch_content(cls, url, data=None, headers=None, cache_valid=None): if not headers: headers = { "User-Agent": ( @@ -57,11 +59,13 @@ async def fetch_content(url, data=None, headers=None, cache_valid=None): # check internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) - session = HTTPAsyncSession() + if not cls._http_session: + cls._http_session = HTTPAsyncSession() + if data: - r = await session.post(url, data=data, headers=headers) + r = await cls._http_session.post(url, data=data, headers=headers) else: - r = await session.get(url, headers=headers) + r = await cls._http_session.get(url, headers=headers) r.raise_for_status() result = r.text From dd033bf675de4599b56c9b32af0a3bd94c4e2bb8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jul 2023 13:52:09 +0300 Subject: [PATCH 60/69] Sync docs --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index eaf1ae10f9..50bbe8ac61 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit eaf1ae10f9b2d3d37a5c55056dd7d17ba344d060 +Subproject commit 50bbe8ac61de55ba31d153d2000fc04ee5dd09a4 diff --git a/examples b/examples index 3e23b5ac43..4b572ec9fe 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 3e23b5ac43ab7ec277ce7d68618458f3980f8089 +Subproject commit 4b572ec9fef6df5aacb1e113177f5ec28dda1cde From 0ff46bdd88aa82f628bae12c5c3f4ed874a11f65 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jul 2023 13:53:12 +0300 Subject: [PATCH 61/69] Improve support for Python 3.6 --- platformio/commands/upgrade.py | 13 ++++++------- platformio/system/completion.py | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index da8a2b3e55..9e4d61d2eb 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -53,16 +53,15 @@ def cli(dev, verbose): subprocess.run( [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], check=True, - capture_output=not verbose, + stdout=subprocess.PIPE if not verbose else None, ) - r = subprocess.run( + output = subprocess.run( [python_exe, "-m", "platformio", "--version"], check=True, - capture_output=True, - text=True, - ) - assert "version" in r.stdout - actual_version = r.stdout.split("version", 1)[1].strip() + stdout=subprocess.PIPE, + ).stdout.decode() + assert "version" in output + actual_version = output.split("version", 1)[1].strip() click.secho( "PlatformIO has been successfully upgraded to %s" % actual_version, fg="green", diff --git a/platformio/system/completion.py b/platformio/system/completion.py index 57e2f1d325..100d1a1d2b 100644 --- a/platformio/system/completion.py +++ b/platformio/system/completion.py @@ -29,8 +29,10 @@ class ShellType(Enum): def get_bash_version(): - result = subprocess.run(["bash", "--version"], capture_output=True, check=True) - match = re.search(r"version\s+(\d+)\.(\d+)", result.stdout.decode(), re.IGNORECASE) + output = subprocess.run( + ["bash", "--version"], check=True, stdout=subprocess.PIPE + ).stdout.decode() + match = re.search(r"version\s+(\d+)\.(\d+)", output, re.IGNORECASE) if match: return (int(match.group(1)), int(match.group(2))) return (0, 0) From 01ab1fa4c08d0f5f854b84d6098a111760ebbe1b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jul 2023 18:37:57 +0300 Subject: [PATCH 62/69] Resolved a critical issue related to the usage of the ``-include`` flag // Resolve #4682 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piobuild.py | 5 ++++- tests/commands/test_run.py | 13 +++++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8c1802935e..b36ed2183a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,7 @@ PlatformIO Core 6 * Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) * Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) * Updated the `pio project metadata `__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata +* Resolved a critical issue related to the usage of the ``-include`` flag within the `build_flags `__ option, specifically when employing dynamic variables (`issue #4682 `_) * Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated `__ 6.1.7 (2023-05-08) diff --git a/docs b/docs index 50bbe8ac61..daa389f68b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 50bbe8ac61de55ba31d153d2000fc04ee5dd09a4 +Subproject commit daa389f68b9e3505f8e6aceda5d9362ba0cb5e54 diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index ba194eb076..1646dc95f5 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -200,13 +200,16 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): + p = env.subst(p) if os.path.isdir(p): result[k][i] = os.path.abspath(p) # fix relative path for "-include" for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": - result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path()))) + p = env.subst(f[1].get_path()) + if os.path.exists(p): + result["CCFLAGS"][i] = (f[0], os.path.abspath(p)) return result diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index cb9e3d859e..de13a5a8fc 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -23,6 +23,10 @@ def test_generic_build(clirunner, validate_cliresult, tmpdir): ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), ("-Iextra_inc", "-Iextra_inc"), + ( + "-include $PROJECT_DIR/lib/component/component-forced-include.h", + "component-forced-include.h", + ), ] tmpdir.join("platformio.ini").write( @@ -95,6 +99,10 @@ def post_prog_action(source, target, env): #error "POST_SCRIPT_MACRO" #endif +#ifndef I_AM_FORCED_COMPONENT_INCLUDE +#error "I_AM_FORCED_COMPONENT_INCLUDE" +#endif + #ifdef COMMENTED_MACRO #error "COMMENTED_MACRO" #endif @@ -124,6 +132,11 @@ def post_prog_action(source, target, env): void dummy(void ) {}; """ ) + component_dir.join("component-forced-include.h").write( + """ +#define I_AM_FORCED_COMPONENT_INCLUDE + """ + ) result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) validate_cliresult(result) From 1c419ef71a29df32d37247a805b81de7e40b8f60 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jul 2023 18:38:48 +0300 Subject: [PATCH 63/69] Bump version to 6.1.8rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index ab39768b6c..913426e735 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8b3") +VERSION = (6, 1, "8rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 0f9a5f8eeeee24c134ff75fc30a89305a7a2c386 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 4 Jul 2023 15:35:57 +0300 Subject: [PATCH 64/69] Show only the package name in "cyan" color --- platformio/package/commands/list.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index eff7110001..9d5fee4046 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -59,7 +59,8 @@ def humanize_package(pkg, spec=None, verbose=False): if spec and not isinstance(spec, PackageSpec): spec = PackageSpec(spec) data = [ - click.style("{name} @ {version}".format(**pkg.metadata.as_dict()), fg="cyan") + click.style(pkg.metadata.name, fg="cyan"), + click.style(f"@ {str(pkg.metadata.version)}", bold=True), ] extra_data = ["required: %s" % (spec.humanize() if spec else "Any")] if verbose: From 01423a7659fb637587d19d0786a9b5e0b6112ca3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 4 Jul 2023 15:36:19 +0300 Subject: [PATCH 65/69] Revert back ProjectConfigDirsMixin (PIO Core 3.x depends on it) --- platformio/project/config.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 4e1a667396..8020fdd305 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -465,7 +465,17 @@ def lint(cls, path=None): return {"errors": errors, "warnings": warnings} -class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin): +class ProjectConfigDirsMixin: + def get_optional_dir(self, name): + """ + Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs + PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0 + PIO Home 3.0 Project Inspection depends on it + """ + return self.get("platformio", f"{name}_dir") + + +class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin, ProjectConfigDirsMixin): _instances = {} @staticmethod From 6eff31b5d3df5891a35f743933011904a8216a38 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 4 Jul 2023 15:36:46 +0300 Subject: [PATCH 66/69] Bump version to 6.1.8rc2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 913426e735..76b5f0629b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8rc1") +VERSION = (6, 1, "8rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 355f57e8884fda6ac68a04376cea106b46f30611 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jul 2023 14:08:09 +0300 Subject: [PATCH 67/69] Update deps --- setup.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 82241982d8..f37da675a6 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys from setuptools import find_packages, setup from platformio import ( @@ -25,27 +24,32 @@ __version__, ) -PY36 = sys.version_info < (3, 7) - +env_marker_below_37 = "python_version < '3.7'" +env_marker_gte_37 = "python_version >= '3.7'" minimal_requirements = [ "bottle==0.12.*", - "click%s" % ("==8.0.4" if PY36 else ">=8.0.4,<9"), + "click==8.0.4; " + env_marker_below_37, + "click==8.1.*; " + env_marker_gte_37, "colorama", - "marshmallow==%s" % ("3.14.1" if PY36 else "3.*"), - "pyelftools>=0.27,<1", + "marshmallow==3.14.1; " + env_marker_below_37, + "marshmallow==3.19.*; " + env_marker_gte_37, + "pyelftools==0.29", "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" - "requests==%s" % ("2.27.1" if PY36 else "2.*"), + "requests==2.*", "semantic_version==2.10.*", - "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"), + "tabulate==0.*", ] home_requirements = [ - "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), - "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.28.*"), - "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), - "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), + "aiofiles>=0.8.0", + "ajsonrpc==1.2.*", + "starlette==0.19.1; " + env_marker_below_37, + "starlette==0.28.*; " + env_marker_gte_37, + "uvicorn==0.16.0; " + env_marker_below_37, + "uvicorn==0.22.*; " + env_marker_gte_37, + "wsproto==1.0.0; " + env_marker_below_37, + "wsproto==1.2.*; " + env_marker_gte_37, ] # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ From a3e66d6325d2871cc2aa9efd9d8af74ed2882b56 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jul 2023 14:08:26 +0300 Subject: [PATCH 68/69] Enable "esphome" project for CI --- .github/workflows/projects.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/projects.yml b/.github/workflows/projects.yml index 080b8162a1..1d37c5a38a 100644 --- a/.github/workflows/projects.yml +++ b/.github/workflows/projects.yml @@ -13,11 +13,11 @@ jobs: folder: "Marlin" config_dir: "Marlin" env_name: "mega2560" - # - esphome: - # repository: "esphome/esphome" - # folder: "esphome" - # config_dir: "esphome" - # env_name: "esp32-arduino" + - esphome: + repository: "esphome/esphome" + folder: "esphome" + config_dir: "esphome" + env_name: "esp32-arduino" - smartknob: repository: "scottbez1/smartknob" folder: "smartknob" From 3b3fbecbf35551e4eae8ec2fb407c043cdf36a56 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jul 2023 15:11:34 +0300 Subject: [PATCH 69/69] c --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b36ed2183a..2aa6774484 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,7 +15,7 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** -6.1.8 (2023-??-??) +6.1.8 (2023-07-05) ~~~~~~~~~~~~~~~~~~ * Added a new ``--lint`` option to the `pio project config `__ command, enabling users to efficiently perform linting on the |PIOCONF| diff --git a/docs b/docs index daa389f68b..3f462c9ae6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit daa389f68b9e3505f8e6aceda5d9362ba0cb5e54 +Subproject commit 3f462c9ae63623710d48c85e16241bbb0a2b6553 diff --git a/platformio/__init__.py b/platformio/__init__.py index 76b5f0629b..75b51791c2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (6, 1, "8rc2") +VERSION = (6, 1, 8) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"