diff --git a/.isort.cfg b/.isort.cfg index c147908fc8..eba1b0f322 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc,tabulate diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b38098d67..d9764348be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,18 +4,18 @@ Contributing To get started, sign the Contributor License Agreement. 1. Fork the repository on GitHub. -2. Make a branch off of ``develop`` -3. Run ``pip install tox`` -4. Go to the root of project where is located ``tox.ini`` and run ``tox -e develop`` +2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` +3. Run `pip install tox` +4. Go to the root of project where is located `tox.ini` and run `tox -e py27` 5. Activate current development environment: - * Windows: ``.tox\develop\Scripts\activate`` - * Bash/ZSH: ``source .tox/develop/bin/activate`` - * Fish: ``source .tox/bin/activate.fish`` + * Windows: `.tox\py27\Scripts\activate` + * Bash/ZSH: `source .tox/py27/bin/activate` + * Fish: `source .tox/py27/bin/activate.fish` 6. Make changes to code, documentation, etc. -7. Lint source code ``tox -e lint`` -8. Run the tests ``tox -e py27`` -9. Build documentation ``tox -e docs`` (creates a directory _build under docs where you can find the html) +7. Lint source code `make lint` +8. Run the tests `make test` +9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html) 10. Commit changes to your forked repository 11. Submit a Pull Request on GitHub. diff --git a/HISTORY.rst b/HISTORY.rst index efa87c51fe..c941ba7252 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,21 @@ Release Notes PlatformIO 4.0 -------------- +4.0.1 (2019-08-22) +~~~~~~~~~~~~~~~~~~ + +* Print `debug tool `__ name for the active debugging session +* Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 `_) +* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding +* Improved printing of tabulated results +* Automatically normalize file system paths to UNIX-style for Project Generator (`issue #2857 `_) +* Ability to set "databaseFilename" for VSCode and C/C++ extension (`issue #2825 `_) +* Renamed "enable_ssl" setting to `strict_ssl `__ +* Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger `__ and "piped" openOCD +* Fixed an issue when "debug", "home", "run", and "test" commands were not shown in "platformio --help" CLI +* Fixed an issue with PIO Home's "No JSON object could be decoded" (`issue #2823 `_) +* Fixed an issue when `library.json `__ had priority over project configuration for `LDF `__ (`issue #2867 `_) + 4.0.0 (2019-07-10) ~~~~~~~~~~~~~~~~~~ diff --git a/Makefile b/Makefile index c82b7c544c..efbb4d2817 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,6 @@ profile: # Usage $ > make PIOARGS="boards" profile python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof + +publish: + python setup.py sdist upload diff --git a/README.rst b/README.rst index ee1f68961f..1b539c671d 100644 --- a/README.rst +++ b/README.rst @@ -10,20 +10,16 @@ PlatformIO .. image:: https://img.shields.io/pypi/v/platformio.svg :target: https://pypi.python.org/pypi/platformio/ :alt: Latest Version -.. image:: https://img.shields.io/pypi/l/platformio.svg +.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://pypi.python.org/pypi/platformio/ :alt: License -.. image:: https://img.shields.io/PlatformIO/Community.png +.. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg :alt: Community Forums :target: https://community.platformio.org?utm_source=github&utm_medium=core -.. image:: https://img.shields.io/PIO/Plus.png?color=orange - :alt: PIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem - :target: https://platformio.org/pricing?utm_source=github&utm_medium=core **Quick Links:** `Web `_ | -`PIO Plus `_ | `PlatformIO IDE `_ | -`Project Examples `_ | +`Project Examples `__ | `Docs `_ | `Donate `_ | `Contact Us `_ @@ -53,7 +49,7 @@ Open Source * `PlatformIO IDE `_ * `PlatformIO Core (CLI) `_ * `Library Management `_ -* `Project Examples `_ +* `Project Examples `__ * `Desktop IDEs Integration `_ * `Continuous Integration `_ * `Advanced Scripting API `_ @@ -132,6 +128,15 @@ Contributing See `contributing guidelines `_. +Telemetry / Privacy Policy +-------------------------- + +Share minimal diagnostics and usage information to help us make PlatformIO better. +It is enabled by default. For more information see: + +* `Telemetry Setting `_ +* `SSL Setting `_ + License ------- diff --git a/docs b/docs index ae7deefa58..29f80d45f2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ae7deefa584f8c0fbb98c36b45649cc86bdf46b7 +Subproject commit 29f80d45f2d7fe14918b507d84ec8badc54fe087 diff --git a/examples b/examples index 70f28968f2..a71564ab46 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 70f28968f2fa3f76374d236156581ddc4e2e8670 +Subproject commit a71564ab46d27c387f17814056b659f826b7db24 diff --git a/platformio/__init__.py b/platformio/__init__.py index 9fb531a541..7f7b65d0e8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, 0) +VERSION = (4, 0, 1) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/app.py b/platformio/app.py index c2b95967fe..66a6f12679 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -22,7 +22,7 @@ import requests -from platformio import exception, lockfile, util +from platformio import exception, fs, lockfile from platformio.compat import (WINDOWS, dump_json_to_unicode, hashlib_encode_data) from platformio.proc import is_ci @@ -73,16 +73,14 @@ def projects_dir_validate(projects_dir): "description": "Enable caching for API requests and Library Manager", "value": True }, - "enable_ssl": { - "description": "Enable SSL for PlatformIO Services", + "strict_ssl": { + "description": "Strict SSL for PlatformIO Services", "value": False }, "enable_telemetry": { "description": - ("Telemetry service (Yes/No)"), - "value": - True + ("Telemetry service (Yes/No)"), + "value": True }, "force_verbose": { "description": "Force verbose output when processing environments", @@ -113,7 +111,7 @@ def __enter__(self): try: self._lock_state_file() if isfile(self.path): - self._storage = util.load_json(self.path) + self._storage = fs.load_json(self.path) assert isinstance(self._storage, dict) except (AssertionError, ValueError, UnicodeDecodeError, exception.InvalidJSONFile): @@ -157,6 +155,9 @@ def update(self, *args, **kwargs): self.modified = True return self._storage.update(*args, **kwargs) + def clear(self): + return self._storage.clear() + def __getitem__(self, key): return self._storage[key] @@ -287,7 +288,7 @@ def delete(self, keys=None): try: remove(path) if not listdir(dirname(path)): - util.rmtree_(dirname(path)) + fs.rmtree(dirname(path)) except OSError: pass @@ -301,7 +302,7 @@ def delete(self, keys=None): def clean(self): if not self.cache_dir or not isdir(self.cache_dir): return - util.rmtree_(self.cache_dir) + fs.rmtree(self.cache_dir) def clean_cache(): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8fa012d086..f29f2f853b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -27,7 +27,7 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import util +from platformio import fs from platformio.compat import PY2, dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path @@ -51,7 +51,7 @@ "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform", "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" ], - toolpath=[join(util.get_source_dir(), "builder", "tools")], + toolpath=[join(fs.get_source_dir(), "builder", "tools")], variables=clivars, # Propagating External Environment @@ -145,10 +145,10 @@ Default("checkprogsize") # Print configured protocols -env.AddPreAction( - ["upload", "program"], - env.VerboseAction(lambda source, target, env: env.PrintUploadInfo(), - "Configuring upload protocol...")) +env.AddPreAction(["upload", "program"], + env.VerboseAction( + lambda source, target, env: env.PrintUploadInfo(), + "Configuring upload protocol...")) AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 66f9db5739..1848cbf7fc 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -31,7 +31,7 @@ from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from platformio import exception, util +from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data, string_types) @@ -78,7 +78,7 @@ def get_used_frameworks(env, path): if "mbed_lib.json" in files: return ["mbed"] for fname in files: - if not env.IsFileWithExt( + if not fs.path_endswith_ext( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): continue content = get_file_contents(join(root, fname)) @@ -200,21 +200,6 @@ def build_unflags(self): def extra_script(self): return None - @property - def lib_archive(self): - return self.env.GetProjectOption("lib_archive", True) - - @property - def lib_ldf_mode(self): - return self.validate_ldf_mode( - self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT)) - - @property - def lib_compat_mode(self): - return self.validate_compat_mode( - self.env.GetProjectOption("lib_compat_mode", - self.COMPAT_MODE_DEFAULT)) - @property def depbuilders(self): return self._depbuilders @@ -227,6 +212,14 @@ def dependent(self): def is_built(self): return self._is_built + @property + def lib_archive(self): + return self.env.GetProjectOption("lib_archive", True) + + @property + def lib_ldf_mode(self): + return self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT) + @staticmethod def validate_ldf_mode(mode): if isinstance(mode, string_types): @@ -239,6 +232,11 @@ def validate_ldf_mode(mode): pass return LibBuilderBase.LDF_MODE_DEFAULT + @property + def lib_compat_mode(self): + return self.env.GetProjectOption("lib_compat_mode", + self.COMPAT_MODE_DEFAULT) + @staticmethod def validate_compat_mode(mode): if isinstance(mode, string_types): @@ -261,7 +259,7 @@ def load_manifest(self): return {} def process_extra_options(self): - with util.cd(self.path): + with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) @@ -351,7 +349,7 @@ def _get_found_includes(self, search_files=None): if not self.PARSE_SRC_BY_H_NAME: continue _h_path = item.get_abspath() - if not self.env.IsFileWithExt(_h_path, piotool.SRC_HEADER_EXT): + if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT): continue _f_part = _h_path[:_h_path.rindex(".")] for ext in piotool.SRC_C_EXT: @@ -533,7 +531,7 @@ class MbedLibBuilder(LibBuilderBase): def load_manifest(self): if not isfile(join(self.path, "module.json")): return {} - return util.load_json(join(self.path, "module.json")) + return fs.load_json(join(self.path, "module.json")) @property def include_dir(self): @@ -611,7 +609,7 @@ def _mbed_normalize_macro(macro): def _mbed_lib_conf_parse_macros(self, mbed_lib_path): macros = {} cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) - manifest = util.load_json(mbed_lib_path) + manifest = fs.load_json(mbed_lib_path) # default macros for macro in manifest.get("macros", []): @@ -682,7 +680,7 @@ class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): assert isfile(join(self.path, "library.json")) - manifest = util.load_json(join(self.path, "library.json")) + manifest = fs.load_json(join(self.path, "library.json")) assert "name" in manifest # replace "espressif" old name dev/platform with ESP8266 @@ -700,14 +698,14 @@ def _is_arduino_manifest(self): @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): - with util.cd(self.path): + with fs.cd(self.path): return realpath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): - with util.cd(self.path): + with fs.cd(self.path): return realpath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) @@ -741,23 +739,28 @@ def extra_script(self): @property def lib_archive(self): - if "libArchive" in self._manifest.get("build", {}): - return self._manifest.get("build").get("libArchive") - return LibBuilderBase.lib_archive.fget(self) + global_value = self.env.GetProjectOption("lib_archive") + if global_value is not None: + return global_value + return self._manifest.get("build", {}).get( + "libArchive", LibBuilderBase.lib_archive.fget(self)) @property def lib_ldf_mode(self): - if "libLDFMode" in self._manifest.get("build", {}): - return self.validate_ldf_mode( - self._manifest.get("build").get("libLDFMode")) - return LibBuilderBase.lib_ldf_mode.fget(self) + return self.validate_ldf_mode( + self.env.GetProjectOption( + "lib_ldf_mode", + self._manifest.get("build", {}).get( + "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self)))) @property def lib_compat_mode(self): - if "libCompatMode" in self._manifest.get("build", {}): - return self.validate_compat_mode( - self._manifest.get("build").get("libCompatMode")) - return LibBuilderBase.lib_compat_mode.fget(self) + return self.validate_ldf_mode( + self.env.GetProjectOption( + "lib_compat_mode", + self._manifest.get("build", {}).get( + "libCompatMode", + LibBuilderBase.lib_compat_mode.fget(self)))) def is_platforms_compatible(self, platforms): items = self._manifest.get("platforms") @@ -1000,7 +1003,7 @@ def ConfigureProjectLibBuilder(env): def _get_vcs_info(lb): path = LibraryManager.get_src_manifest_path(lb.path) - return util.load_json(path) if path else None + return fs.load_json(path) if path else None def _correct_found_libs(lib_builders): # build full dependency graph diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 1e01d59ec1..d7e17b0b9a 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -24,7 +24,7 @@ from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error -from platformio import util +from platformio import fs, util from platformio.compat import get_file_contents, glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -295,7 +295,7 @@ def PioClean(env, clean_dir): print("Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst))) print("Done cleaning") - util.rmtree_(clean_dir) + fs.rmtree(clean_dir) env.Exit(0) @@ -333,7 +333,7 @@ def GetExtraScripts(env, scope): items.append(item[len(scope) + 1:]) if not items: return items - with util.cd(env.subst("$PROJECT_DIR")): + with fs.cd(env.subst("$PROJECT_DIR")): return [realpath(item) for item in items] diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 09d712879e..8179982eae 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -20,7 +20,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error -from platformio import exception, util +from platformio import exception, fs, util from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectOptions @@ -129,7 +129,7 @@ def _get_plaform_data(): src_manifest_path = platform.pm.get_src_manifest_path( platform.get_dir()) if src_manifest_path: - src_manifest = util.load_json(src_manifest_path) + src_manifest = fs.load_json(src_manifest_path) if "version" in src_manifest: data.append("#" + src_manifest['version']) if int(ARGUMENTS.get("PIOVERBOSE", 0)): @@ -152,7 +152,7 @@ def _get_hardware_data(): ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") data.append("%s RAM, %s Flash" % - (util.format_filesize(ram), util.format_filesize(flash))) + (fs.format_filesize(ram), fs.format_filesize(flash))) return data def _get_debug_data(): diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index dd03894a09..a7ea2f4e5e 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -25,7 +25,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException -from platformio import exception, util +from platformio import exception, fs, util from platformio.compat import WINDOWS from platformio.proc import exec_command @@ -156,7 +156,7 @@ def _look_for_serial_port(): env.Replace(UPLOAD_PORT=_look_for_mbed_disk()) else: try: - util.ensure_udev_rules() + fs.ensure_udev_rules() except exception.InvalidUdevRules as e: sys.stderr.write("\n%s\n\n" % e) env.Replace(UPLOAD_PORT=_look_for_serial_port()) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c584d6546e..057d39aa4b 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -14,10 +14,8 @@ from __future__ import absolute_import -import re +import os import sys -from glob import glob -from os import sep, walk from os.path import basename, dirname, isdir, join, realpath from SCons import Builder, Util # pylint: disable=import-error @@ -27,14 +25,14 @@ from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio.compat import glob_escape, string_types +from platformio import fs +from platformio.compat import string_types from platformio.util import pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] -SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-" % sep] -SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>") +SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep] def scons_patched_match_splitext(path, suffixes=None): @@ -230,44 +228,11 @@ def ProcessUnFlags(env, flags): env[key].remove(current) -def IsFileWithExt(env, file_, ext): # pylint: disable=W0613 - if basename(file_).startswith("."): - return False - for e in ext: - if file_.endswith(".%s" % e): - return True - return False - - def MatchSourceFiles(env, src_dir, src_filter=None): - - def _append_build_item(items, item, src_dir): - if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT): - items.add(item.replace(src_dir + sep, "")) - - src_dir = env.subst(src_dir) src_filter = env.subst(src_filter) if src_filter else None src_filter = src_filter or SRC_FILTER_DEFAULT - if isinstance(src_filter, (list, tuple)): - src_filter = " ".join(src_filter) - - matches = set() - # correct fs directory separator - src_filter = src_filter.replace("/", sep).replace("\\", sep) - for (action, pattern) in SRC_FILTER_PATTERNS_RE.findall(src_filter): - items = set() - for item in glob(join(glob_escape(src_dir), pattern)): - if isdir(item): - for root, _, files in walk(item, followlinks=True): - for f in files: - _append_build_item(items, join(root, f), src_dir) - else: - _append_build_item(items, item, src_dir) - if action == "+": - matches |= items - else: - matches -= items - return sorted(list(matches)) + return fs.match_src_files(env.subst(src_dir), src_filter, + SRC_BUILD_EXT + SRC_HEADER_EXT) def CollectBuildFiles(env, @@ -279,7 +244,7 @@ def CollectBuildFiles(env, variants = [] src_dir = env.subst(src_dir) - if src_dir.endswith(sep): + if src_dir.endswith(os.sep): src_dir = src_dir[:-1] for item in env.MatchSourceFiles(src_dir, src_filter): @@ -291,7 +256,7 @@ def CollectBuildFiles(env, variants.append(_var_dir) env.VariantDir(_var_dir, _src_dir, duplicate) - if env.IsFileWithExt(item, SRC_BUILD_EXT): + if fs.path_endswith_ext(item, SRC_BUILD_EXT): sources.append(env.File(join(_var_dir, basename(item)))) return sources @@ -316,7 +281,7 @@ def BuildFrameworks(env, frameworks): env.Exit(1) for f in frameworks: - if f in ("arduino", "energia"): + if f == "arduino": # Arduino IDE appends .o the end of filename Builder.match_splitext = scons_patched_match_splitext if "nobuild" not in COMMAND_LINE_TARGETS: @@ -352,7 +317,6 @@ def generate(env): env.AddMethod(ParseFlagsExtended) env.AddMethod(ProcessFlags) env.AddMethod(ProcessUnFlags) - env.AddMethod(IsFileWithExt) env.AddMethod(MatchSourceFiles) env.AddMethod(CollectBuildFiles) env.AddMethod(BuildFrameworks) diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index 7dc69d589f..5d53349e31 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from os.path import dirname +from os.path import dirname, isfile, join import click @@ -38,11 +38,14 @@ def invoke(self, ctx): def list_commands(self, ctx): cmds = [] - for filename in os.listdir(dirname(__file__)): - if filename.startswith("__init__"): + cmds_dir = dirname(__file__) + for name in os.listdir(cmds_dir): + if name.startswith("__init__"): continue - if filename.endswith(".py"): - cmds.append(filename[:-3]) + if isfile(join(cmds_dir, name, "command.py")): + cmds.append(name) + elif name.endswith(".py"): + cmds.append(name[:-3]) cmds.sort() return cmds diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 6aff16813a..bf3f68efa3 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -15,8 +15,9 @@ import json import click +from tabulate import tabulate -from platformio import util +from platformio import fs from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformManager @@ -42,32 +43,18 @@ def cli(query, installed, json_output): # pylint: disable=R0912 click.echo("") click.echo("Platform: ", nl=False) click.secho(platform, bold=True) - click.echo("-" * terminal_width) + click.echo("=" * terminal_width) print_boards(boards) return True def print_boards(boards): - terminal_width, _ = click.get_terminal_size() - BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} " - " {flash:<7} {ram:<6} {name}") click.echo( - BOARDLIST_TPL.format(type=click.style("ID", fg="cyan"), - mcu="MCU", - frequency="Frequency", - flash="Flash", - ram="RAM", - name="Name")) - click.echo("-" * terminal_width) - - for board in boards: - click.echo( - BOARDLIST_TPL.format(type=click.style(board['id'], fg="cyan"), - mcu=board['mcu'], - frequency="%dMHz" % (board['fcpu'] / 1000000), - flash=util.format_filesize(board['rom']), - ram=util.format_filesize(board['ram']), - name=board['name'])) + tabulate([(click.style(b['id'], fg="cyan"), b['mcu'], "%dMHz" % + (b['fcpu'] / 1000000), fs.format_filesize( + b['rom']), fs.format_filesize(b['ram']), b['name']) + for b in boards], + headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"])) def _get_boards(installed=False): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 55ef07ad78..8ad68c2b8b 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -20,7 +20,7 @@ import click -from platformio import app, util +from platformio import app, fs from platformio.commands.init import cli as cmd_init from platformio.commands.init import validate_boards from platformio.commands.run import cli as cmd_run @@ -89,7 +89,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches app.set_session_var("force_option", True) if not keep_build_dir and isdir(build_dir): - util.rmtree_(build_dir) + fs.rmtree(build_dir) if not isdir(build_dir): makedirs(build_dir) @@ -119,7 +119,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose) finally: if not keep_build_dir: - util.rmtree_(build_dir) + fs.rmtree(build_dir) def _copy_contents(dst_dir, contents): @@ -161,7 +161,7 @@ def _exclude_contents(dst_dir, patterns): for path in contents: path = abspath(path) if isdir(path): - util.rmtree_(path) + fs.rmtree(path) elif isfile(path): remove(path) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index f6d403bb07..4a00cb9ef4 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -26,7 +26,7 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error -from platformio import app, exception, util +from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer @@ -66,9 +66,9 @@ def spawn(self, gdb_path, prog_path): self._kill_previous_session() patterns = { - "PROJECT_DIR": helpers.escape_path(self.project_dir), - "PROG_PATH": helpers.escape_path(prog_path), - "PROG_DIR": helpers.escape_path(dirname(prog_path)), + "PROJECT_DIR": self.project_dir, + "PROG_PATH": prog_path, + "PROG_DIR": dirname(prog_path), "PROG_NAME": basename(splitext(prog_path)[0]), "DEBUG_PORT": self.debug_options['port'], "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], @@ -157,6 +157,7 @@ def generate_pioinit(self, dst_dir, patterns): banner = [ "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", + "echo PlatformIO: debug_tool = %s\\n" % self.debug_options['tool'], "echo PlatformIO: Initializing remote target...\\n" ] footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] @@ -197,7 +198,7 @@ def onStdInData(self, data): def processEnded(self, reason): # pylint: disable=unused-argument self._unlock_session() if self._gdbsrc_dir and isdir(self._gdbsrc_dir): - util.rmtree_(self._gdbsrc_dir) + fs.rmtree(self._gdbsrc_dir) if self._debug_server: self._debug_server.terminate() @@ -252,8 +253,9 @@ def _handle_error(self, data): return configuration = {"debug": self.debug_options, "env": self.env_options} exd = re.sub(r'\\(?!")', "/", json.dumps(configuration)) - exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', lambda m: '"%s"' % join( - *m.group(1).split("/")[-2:]), exd, re.I | re.M) + exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', + lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]), exd, + re.I | re.M) mp = MeasurementProtocol() mp['exd'] = "DebugGDBPioInitError: %s" % exd mp['exf'] = 1 @@ -273,7 +275,7 @@ def _kill_previous_session(self): else: kill = ["kill", pid] try: - util.exec_command(kill) + proc.exec_command(kill) except: # pylint: disable=bare-except pass diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 14c4f665fd..8e78e72850 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -21,7 +21,7 @@ import click -from platformio import exception, util +from platformio import exception, fs, proc, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig @@ -61,7 +61,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if os.getenv(sysenv): project_dir = os.getenv(sysenv) - with util.cd(project_dir): + with fs.cd(project_dir): config = ProjectConfig.get_instance( project_conf or join(project_dir, "platformio.ini")) config.validate(envs=[environment] if environment else None) @@ -83,16 +83,14 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, "Could not load debug configuration") if "--version" in __unprocessed: - result = util.exec_command([configuration['gdb_path'], "--version"]) + result = proc.exec_command([configuration['gdb_path'], "--version"]) if result['returncode'] == 0: return click.echo(result['out']) raise exception.PlatformioException("\n".join( [result['out'], result['err']])) try: - util.ensure_udev_rules() - except NameError: - pass + fs.ensure_udev_rules() except exception.InvalidUdevRules as e: for line in str(e).split("\n") + [""]: click.echo( diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index daaa8d9388..91f0631818 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -44,10 +44,6 @@ def is_mi_mode(args): return "--interpreter" in " ".join(args) -def escape_path(path): - return path.replace("\\", "/") - - def get_default_debug_env(config): default_envs = config.default_envs() all_envs = config.envs() @@ -121,7 +117,7 @@ def _cleanup_cmds(items): cwd=server_package_dir if server_package else None, executable=tool_settings['server'].get("executable"), arguments=[ - a.replace("$PACKAGE_DIR", escape_path(server_package_dir)) + a.replace("$PACKAGE_DIR", server_package_dir) if server_package_dir else a for a in tool_settings['server'].get("arguments", []) ]) @@ -169,11 +165,11 @@ def configure_esp32_load_cmds(debug_options, configuration): mon_cmds = [ 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( - path=escape_path(item['path']), offset=item['offset']) + path=item['path'], offset=item['offset']) for item in configuration.get("flash_extra_images") ] mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' % - escape_path(configuration['prog_path'][:-4])) + configuration['prog_path'][:-4]) return mon_cmds diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 98c7cc1a01..f363ccb5e0 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -17,7 +17,6 @@ import click from twisted.internet import protocol # pylint: disable=import-error -from platformio.commands.debug import helpers from platformio.compat import string_types from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_core_dir @@ -30,8 +29,8 @@ class BaseProcess(protocol.ProcessProtocol, object): STDOUT_CHUNK_SIZE = 2048 COMMON_PATTERNS = { - "PLATFORMIO_HOME_DIR": helpers.escape_path(get_project_core_dir()), - "PLATFORMIO_CORE_DIR": helpers.escape_path(get_project_core_dir()), + "PLATFORMIO_HOME_DIR": get_project_core_dir(), + "PLATFORMIO_CORE_DIR": get_project_core_dir(), "PYTHONEXE": get_pythonexe_path() } diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 83bba3406b..98f1e0e14e 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -19,7 +19,6 @@ from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, util -from platformio.commands.debug import helpers from platformio.commands.debug.process import BaseProcess from platformio.proc import where_is_program @@ -67,15 +66,15 @@ def spawn(self, patterns): # pylint: disable=too-many-branches if openocd_pipe_allowed: args = [] if server['cwd']: - args.extend(["-s", helpers.escape_path(server['cwd'])]) + args.extend(["-s", server['cwd']]) args.extend([ "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled" ]) args.extend(server['arguments']) str_args = " ".join( [arg if arg.startswith("-") else '"%s"' % arg for arg in args]) - self._debug_port = '| "%s" %s' % ( - helpers.escape_path(server_executable), str_args) + self._debug_port = '| "%s" %s' % (server_executable, str_args) + self._debug_port = self._debug_port.replace("\\", "\\\\") else: env = os.environ.copy() # prepend server "lib" folder to LD path diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 1666dc1770..9e19edf1ab 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -25,6 +25,11 @@ class AppRPC(object): APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") + IGNORE_STORAGE_KEYS = [ + "cid", "coreVersion", "coreSystype", "coreCaller", "coreSettings", + "homeDir", "projectsDir" + ] + @staticmethod def load_state(): with app.State(AppRPC.APPSTATE_PATH, lock=True) as state: @@ -57,6 +62,7 @@ def load_state(): ] state['storage'] = storage + state.modified = False # skip saving extra fields return state.as_dict() @staticmethod @@ -66,6 +72,10 @@ def get_state(): @staticmethod def save_state(state): with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: - # s.clear() + s.clear() s.update(state) + storage = s.get("storage", {}) + for k in AppRPC.IGNORE_STORAGE_KEYS: + if k in storage: + del storage[k] return True diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index 5f8f31a47a..85bf98fae3 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -21,22 +21,24 @@ class IDERPC(object): def __init__(self): - self._queue = [] + self._queue = {} - def send_command(self, command, params): - if not self._queue: + def send_command(self, command, params, sid=0): + if not self._queue.get(sid): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4005, message="PIO Home IDE agent is not started") - while self._queue: - self._queue.pop().callback({ + while self._queue[sid]: + self._queue[sid].pop().callback({ "id": time.time(), "method": command, "params": params }) - def listen_commands(self): - self._queue.append(defer.Deferred()) - return self._queue[-1] + def listen_commands(self, sid=0): + if sid not in self._queue: + self._queue[sid] = [] + self._queue[sid].append(defer.Deferred()) + return self._queue[sid][-1] - def open_project(self, project_dir): - return self.send_command("open_project", project_dir) + def open_project(self, project_dir, sid=0): + return self.send_command("open_project", project_dir, sid) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index a88f495da4..4fa13b1a3c 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -21,10 +21,11 @@ import click import jsonrpc # pylint: disable=import-error +from twisted.internet import defer # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error from twisted.internet import utils # pylint: disable=import-error -from platformio import __main__, __version__, util +from platformio import __main__, __version__, fs from platformio.commands.home import helpers from platformio.compat import (PY2, get_filesystem_encoding, is_bytes, string_types) @@ -68,6 +69,10 @@ def get_value_and_reset(self): class PIOCoreRPC(object): + @staticmethod + def version(): + return __version__ + @staticmethod def setup_multithreading_std_streams(): if isinstance(sys.stdout, MultiThreadingStdStream): @@ -79,41 +84,67 @@ def setup_multithreading_std_streams(): @staticmethod def call(args, options=None): - PIOCoreRPC.setup_multithreading_std_streams() - cwd = (options or {}).get("cwd") or os.getcwd() + return defer.maybeDeferred(PIOCoreRPC._call_generator, args, options) + + @staticmethod + @defer.inlineCallbacks + def _call_generator(args, options=None): for i, arg in enumerate(args): if isinstance(arg, string_types): args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg else: args[i] = str(arg) - def _call_inline(): - with util.cd(cwd): + to_json = "--json-output" in args + + try: + if args and args[0] in ("account", "remote"): + result = yield PIOCoreRPC._call_subprocess(args, options) + defer.returnValue(PIOCoreRPC._process_result(result, to_json)) + else: + result = yield PIOCoreRPC._call_inline(args, options) + try: + defer.returnValue( + PIOCoreRPC._process_result(result, to_json)) + except ValueError: + # fall-back to subprocess method + result = yield PIOCoreRPC._call_subprocess(args, options) + defer.returnValue( + PIOCoreRPC._process_result(result, to_json)) + except Exception as e: # pylint: disable=bare-except + raise jsonrpc.exceptions.JSONRPCDispatchException( + code=4003, message="PIO Core Call Error", data=str(e)) + + @staticmethod + def _call_inline(args, options): + PIOCoreRPC.setup_multithreading_std_streams() + cwd = (options or {}).get("cwd") or os.getcwd() + + def _thread_task(): + with fs.cd(cwd): exit_code = __main__.main(["-c"] + args) return (PIOCoreRPC.thread_stdout.get_value_and_reset(), PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code) - if args and args[0] in ("account", "remote"): - d = utils.getProcessOutputAndValue( - helpers.get_core_fullpath(), - args, - path=cwd, - env={k: v - for k, v in os.environ.items() if "%" not in k}) - else: - d = threads.deferToThread(_call_inline) + return threads.deferToThread(_thread_task) - d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) - d.addErrback(PIOCoreRPC._call_errback) - return d + @staticmethod + def _call_subprocess(args, options): + cwd = (options or {}).get("cwd") or os.getcwd() + return utils.getProcessOutputAndValue( + helpers.get_core_fullpath(), + args, + path=cwd, + env={k: v + for k, v in os.environ.items() if "%" not in k}) @staticmethod - def _call_callback(result, json_output=False): + def _process_result(result, to_json=False): out, err, code = result text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - if not json_output: + if not to_json: return text try: return json.loads(out) @@ -129,14 +160,3 @@ def _call_callback(result, json_output=False): except ValueError: pass raise e - - @staticmethod - def _call_errback(failure): - raise jsonrpc.exceptions.JSONRPCDispatchException( - code=4003, - message="PIO Core Call Error", - data=failure.getErrorMessage()) - - @staticmethod - def version(): - return __version__ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 4ca5af635b..795bde77b7 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -22,7 +22,7 @@ import jsonrpc # pylint: disable=import-error -from platformio import exception, util +from platformio import exception, fs from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding @@ -77,7 +77,7 @@ def _path_to_name(path): data = {} boards = [] try: - with util.cd(project_dir): + with fs.cd(project_dir): data = _get_project_data(project_dir) except exception.PlatformIOProjectException: continue @@ -86,7 +86,7 @@ def _path_to_name(path): name = board_id try: name = pm.board_config(board_id)['name'] - except (exception.UnknownBoard, exception.UnknownPlatform): + except exception.PlatformioException: pass boards.append({"id": board_id, "name": name}) @@ -196,7 +196,7 @@ def _generate_project_main(_, project_dir, framework): ]) # yapf: disable if not main_content: return project_dir - with util.cd(project_dir): + with fs.cd(project_dir): src_dir = get_project_src_dir() main_path = join(src_dir, "main.cpp") if isfile(main_path): @@ -249,10 +249,10 @@ def import_arduino(self, board, use_arduino_libs, arduino_project_dir): @staticmethod def _finalize_arduino_import(_, project_dir, arduino_project_dir): - with util.cd(project_dir): + with fs.cd(project_dir): src_dir = get_project_src_dir() if isdir(src_dir): - util.rmtree_(src_dir) + fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir) return project_dir diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 09eddacf96..64b5465d46 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -19,7 +19,7 @@ import click -from platformio import exception, util +from platformio import exception, fs from platformio.commands.platform import \ platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator @@ -102,8 +102,7 @@ def cli( ide is not None) if ide: - pg = ProjectGenerator(project_dir, ide, - get_best_envname(project_dir, board)) + pg = ProjectGenerator(project_dir, ide, board) pg.generate() if is_new_project: @@ -131,32 +130,9 @@ def cli( fg="green") -def get_best_envname(project_dir, boards=None): - config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) - config.validate() - - envname = None - default_envs = config.default_envs() - if default_envs: - envname = default_envs[0] - if not boards: - return envname - - for env in config.envs(): - if not boards: - return env - if not envname: - envname = env - items = config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in boards: - return env - - return envname - - def init_base_project(project_dir): ProjectConfig(join(project_dir, "platformio.ini")).save() - with util.cd(project_dir): + with fs.cd(project_dir): dir_to_readme = [ (get_project_src_dir(), None), (get_project_include_dir(), init_include_readme), diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 1d75c961d8..1de0f96074 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -19,8 +19,9 @@ import click import semantic_version +from tabulate import tabulate -from platformio import exception, util +from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import (LibraryManager, get_builtin_libs, @@ -99,7 +100,7 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - with util.cd(storage_dir): + with fs.cd(storage_dir): libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) @@ -486,66 +487,48 @@ def lib_stats(json_output): if json_output: return click.echo(dump_json_to_unicode(result)) - printitem_tpl = "{name:<33} {url}" - printitemdate_tpl = "{name:<33} {date:23} {url}" - - def _print_title(title): - click.secho(title.upper(), bold=True) - click.echo("*" * len(title)) - - def _print_header(with_date=False): - click.echo((printitemdate_tpl if with_date else printitem_tpl).format( - name=click.style("Name", fg="cyan"), - date="Date", - url=click.style("Url", fg="blue"))) - - terminal_width, _ = click.get_terminal_size() - click.echo("-" * terminal_width) - - def _print_lib_item(item): - date = str( - time.strftime("%c", util.parse_date(item['date'])) if "date" in - item else "") - url = click.style("https://platformio.org/lib/show/%s/%s" % - (item['id'], quote(item['name'])), - fg="blue") - click.echo( - (printitemdate_tpl if "date" in item else printitem_tpl).format( - name=click.style(item['name'], fg="cyan"), date=date, url=url)) - - def _print_tag_item(name): - click.echo( - printitem_tpl.format( - name=click.style(name, fg="cyan"), - url=click.style("https://platformio.org/lib/search?query=" + - quote("keyword:%s" % name), - fg="blue"))) - for key in ("updated", "added"): - _print_title("Recently " + key) - _print_header(with_date=True) - for item in result.get(key, []): - _print_lib_item(item) + tabular_data = [(click.style(item['name'], fg="cyan"), + time.strftime("%c", util.parse_date(item['date'])), + "https://platformio.org/lib/show/%s/%s" % + (item['id'], quote(item['name']))) + for item in result.get(key, [])] + table = tabulate(tabular_data, + headers=[ + click.style("RECENTLY " + key.upper(), bold=True), + "Date", "URL" + ]) + click.echo(table) click.echo() - _print_title("Recent keywords") - _print_header(with_date=False) - for item in result.get("lastkeywords"): - _print_tag_item(item) - click.echo() - - _print_title("Popular keywords") - _print_header(with_date=False) - for item in result.get("topkeywords"): - _print_tag_item(item) - click.echo() + for key in ("lastkeywords", "topkeywords"): + tabular_data = [(click.style(name, fg="cyan"), + "https://platformio.org/lib/search?query=" + + quote("keyword:%s" % name)) + for name in result.get(key, [])] + table = tabulate( + tabular_data, + headers=[ + click.style( + ("RECENT" if key == "lastkeywords" else "POPULAR") + + " KEYWORDS", + bold=True), "URL" + ]) + click.echo(table) + click.echo() for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")): - _print_title("Featured: " + title) - _print_header(with_date=False) - for item in result.get(key, []): - _print_lib_item(item) + tabular_data = [(click.style(item['name'], fg="cyan"), + "https://platformio.org/lib/show/%s/%s" % + (item['id'], quote(item['name']))) + for item in result.get(key, [])] + table = tabulate(tabular_data, + headers=[ + click.style("FEATURED: " + title.upper(), + bold=True), "URL" + ]) + click.echo(table) click.echo() return True diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index 8dcdf9a298..b5649979d6 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -21,7 +21,7 @@ import click -from platformio import exception, util +from platformio import exception, fs from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.compat import get_file_contents from platformio.managers.core import pioplus_call @@ -202,4 +202,4 @@ def _tx_target(sock_dir): ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: - util.rmtree_(sock_dir) + fs.rmtree(sock_dir) diff --git a/platformio/commands/run/__init__.py b/platformio/commands/run/__init__.py index b9e6952168..05d4d370df 100644 --- a/platformio/commands/run/__init__.py +++ b/platformio/commands/run/__init__.py @@ -13,4 +13,3 @@ # limitations under the License. from platformio.commands.run.command import cli -from platformio.commands.run.helpers import print_header diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 84d1985791..7b5ca50c0d 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -18,13 +18,14 @@ from time import time import click +from tabulate import tabulate -from platformio import exception, util +from platformio import exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.run.helpers import (clean_build_dir, - handle_legacy_libdeps, - print_summary) + handle_legacy_libdeps) from platformio.commands.run.processor import EnvironmentProcessor +from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig from platformio.project.helpers import (find_project_dir_above, get_project_build_dir) @@ -73,11 +74,17 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, if isfile(project_dir): project_dir = find_project_dir_above(project_dir) - with util.cd(project_dir): + is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta + + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(environment) + # clean obsolete build dir if not disable_auto_clean: try: - clean_build_dir(get_project_build_dir()) + clean_build_dir(get_project_build_dir(), config) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " @@ -85,44 +92,114 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, get_project_build_dir(force=True), fg="yellow") - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) - config.validate(environment) - handle_legacy_libdeps(project_dir, config) - results = [] - start_time = time() default_envs = config.default_envs() - for envname in config.envs(): + results = [] + for env in config.envs(): skipenv = any([ - environment and envname not in environment, not environment - and default_envs and envname not in default_envs + environment and env not in environment, not environment + and default_envs and env not in default_envs ]) if skipenv: - results.append((envname, None)) + results.append({"env": env}) continue - if not silent and any(status is not None - for (_, status) in results): + # print empty line between multi environment project + if not silent and any( + r.get("succeeded") is not None for r in results): click.echo() - ep = EnvironmentProcessor(ctx, envname, config, target, - upload_port, silent, verbose, jobs) - result = (envname, ep.process()) - results.append(result) + results.append( + process_env(ctx, env, config, environment, target, upload_port, + silent, verbose, jobs, is_test_running)) - if result[1] and "monitor" in ep.get_build_targets() and \ - "nobuild" not in ep.get_build_targets(): - ctx.invoke(cmd_device_monitor, - environment=environment[0] if environment else None) + command_failed = any(r.get("succeeded") is False for r in results) - found_error = any(status is False for (_, status) in results) + if (not is_test_running and (command_failed or not silent) + and len(results) > 1): + print_processing_summary(results) - if (found_error or not silent) and len(results) > 1: - click.echo() - print_summary(results, start_time) - - if found_error: + if command_failed: raise exception.ReturnErrorCode(1) return True + + +def process_env(ctx, name, config, environments, targets, upload_port, silent, + verbose, jobs, is_test_running): + if not is_test_running and not silent: + print_processing_header(name, config, verbose) + + ep = EnvironmentProcessor(ctx, name, config, targets, upload_port, silent, + verbose, jobs) + result = {"env": name, "duration": time(), "succeeded": ep.process()} + result['duration'] = time() - result['duration'] + + # print footer on error or when is not unit testing + if not is_test_running and (not silent or not result['succeeded']): + print_processing_footer(result) + + if (result['succeeded'] and "monitor" in ep.get_build_targets() + and "nobuild" not in ep.get_build_targets()): + ctx.invoke(cmd_device_monitor, + environment=environments[0] if environments else None) + + return result + + +def print_processing_header(env, config, verbose=False): + env_dump = [] + for k, v in config.items(env=env): + if verbose or k in ("platform", "framework", "board"): + env_dump.append("%s: %s" % + (k, ", ".join(v) if isinstance(v, list) else v)) + click.echo("Processing %s (%s)" % + (click.style(env, fg="cyan", bold=True), "; ".join(env_dump))) + terminal_width, _ = click.get_terminal_size() + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" % + ((click.style("FAILED", fg="red", bold=True) if is_failed else + click.style("SUCCESS", fg="green", bold=True)), result['duration']), + is_error=is_failed) + + +def print_processing_summary(results): + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + status_str = "IGNORED" + else: + succeeded_nums += 1 + status_str = click.style("SUCCESS", fg="green") + + tabular_data.append( + (click.style(result['env'], fg="cyan"), status_str, + util.humanize_duration_time(result.get("duration")))) + + click.echo() + click.echo(tabulate(tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Status", "Duration") + ]), + err=failed_nums) + + util.print_labeled_bar( + "%s%d succeeded in %s" % + ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, + util.humanize_duration_time(duration)), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index 0c51012b52..3e6497f597 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -13,13 +13,12 @@ # limitations under the License. from os import makedirs -from os.path import getmtime, isdir, isfile, join -from time import time +from os.path import isdir, isfile, join import click -from platformio import util -from platformio.project.helpers import (calculate_project_hash, +from platformio import fs +from platformio.project.helpers import (compute_project_checksum, get_project_dir, get_project_libdeps_dir) @@ -43,67 +42,23 @@ def handle_legacy_libdeps(project_dir, config): fg="yellow") -def clean_build_dir(build_dir): +def clean_build_dir(build_dir, config): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") if isdir(legacy_build_dir) and legacy_build_dir != build_dir: - util.rmtree_(legacy_build_dir) - - structhash_file = join(build_dir, "structure.hash") - proj_hash = calculate_project_hash() - - # if project's config is modified - if (isdir(build_dir) and getmtime(join( - get_project_dir(), "platformio.ini")) > getmtime(build_dir)): - util.rmtree_(build_dir) - - # check project structure - if isdir(build_dir) and isfile(structhash_file): - with open(structhash_file) as f: - if f.read() == proj_hash: - return - util.rmtree_(build_dir) - - if not isdir(build_dir): - makedirs(build_dir) - - with open(structhash_file, "w") as f: - f.write(proj_hash) - - -def print_header(label, is_error=False, fg=None): - terminal_width, _ = click.get_terminal_size() - width = len(click.unstyle(label)) - half_line = "=" * int((terminal_width - width - 2) / 2) - click.secho("%s %s %s" % (half_line, label, half_line), - fg=fg, - err=is_error) - - -def print_summary(results, start_time): - print_header("[%s]" % click.style("SUMMARY")) - - succeeded_nums = 0 - failed_nums = 0 - envname_max_len = max( - [len(click.style(envname, fg="cyan")) for (envname, _) in results]) - for (envname, status) in results: - if status is False: - failed_nums += 1 - status_str = click.style("FAILED", fg="red") - elif status is None: - status_str = click.style("IGNORED", fg="yellow") - else: - succeeded_nums += 1 - status_str = click.style("SUCCESS", fg="green") - - format_str = "Environment {0:<%d}\t[{1}]" % envname_max_len - click.echo(format_str.format(click.style(envname, fg="cyan"), - status_str), - err=status is False) - - print_header("%s%d succeeded in %.2f seconds" % - ("%d failed, " % failed_nums if failed_nums else "", - succeeded_nums, time() - start_time), - is_error=failed_nums, - fg="red" if failed_nums else "green") + fs.rmtree(legacy_build_dir) + + checksum_file = join(build_dir, "project.checksum") + checksum = compute_project_checksum(config) + + if isdir(build_dir): + # check project structure + if isfile(checksum_file): + with open(checksum_file) as f: + if f.read() == checksum: + return + fs.rmtree(build_dir) + + makedirs(build_dir) + with open(checksum_file, "w") as f: + f.write(checksum) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 9ff1e05648..b061c00ab0 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,16 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from time import time - -import click - from platformio import exception, telemetry from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.commands.run.helpers import print_header -from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, - CTX_META_TEST_RUNNING_NAME) +from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-instance-attributes @@ -29,8 +23,6 @@ class EnvironmentProcessor(object): - DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board") - def __init__( # pylint: disable=too-many-arguments self, cmd_ctx, name, config, targets, upload_port, silent, verbose, jobs): @@ -44,37 +36,6 @@ def __init__( # pylint: disable=too-many-arguments self.jobs = jobs self.options = config.items(env=name, as_dict=True) - def process(self): - terminal_width, _ = click.get_terminal_size() - start_time = time() - env_dump = [] - - for k, v in self.options.items(): - if self.verbose or k in self.DEFAULT_PRINT_OPTIONS: - env_dump.append( - "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) - - if not self.silent: - click.echo("Processing %s (%s)" % (click.style( - self.name, fg="cyan", bold=True), "; ".join(env_dump))) - click.secho("-" * terminal_width, bold=True) - - result = self._run_platform() - is_error = result['returncode'] != 0 - - if self.silent and not is_error: - return True - - if is_error or CTX_META_TEST_IS_RUNNING not in self.cmd_ctx.meta: - print_header( - "[%s] Took %.2f seconds" % - ((click.style("ERROR", fg="red", bold=True) if - is_error else click.style("SUCCESS", fg="green", bold=True)), - time() - start_time), - is_error=is_error) - - return not is_error - def get_build_variables(self): variables = {"pioenv": self.name, "project_config": self.config.path} @@ -92,7 +53,7 @@ def get_build_targets(self): return [t for t in self.targets] return self.config.get("env:" + self.name, "targets", []) - def _run_platform(self): + def process(self): if "platform" not in self.options: raise exception.UndefinedEnvPlatform(self.name) @@ -113,5 +74,6 @@ def _run_platform(self): skip_default_package=True) p = PlatformFactory.newPlatform(self.options['platform']) - return p.run(build_vars, build_targets, self.silent, self.verbose, - self.jobs) + result = p.run(build_vars, build_targets, self.silent, self.verbose, + self.jobs) + return result['returncode'] == 0 diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index 5fa20993d0..c626e4caaf 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -13,11 +13,20 @@ # limitations under the License. import click +from tabulate import tabulate from platformio import app from platformio.compat import string_types +def format_value(raw): + if isinstance(raw, bool): + return "Yes" if raw else "No" + if isinstance(raw, string_types): + return raw + return str(raw) + + @click.group(short_help="Manage PlatformIO settings") def cli(): pass @@ -26,40 +35,27 @@ def cli(): @cli.command("get", short_help="Get existing setting/-s") @click.argument("name", required=False) def settings_get(name): - - list_tpl = u"{name:<40} {value:<35} {description}" - terminal_width, _ = click.get_terminal_size() - - click.echo( - list_tpl.format(name=click.style("Name", fg="cyan"), - value=(click.style("Value", fg="green") + - click.style(" [Default]", fg="yellow")), - description="Description")) - click.echo("-" * terminal_width) - - for _name, _data in sorted(app.DEFAULT_SETTINGS.items()): - if name and name != _name: + tabular_data = [] + for key, options in sorted(app.DEFAULT_SETTINGS.items()): + if name and name != key: continue - _value = app.get_setting(_name) + raw_value = app.get_setting(key) + formatted_value = format_value(raw_value) - _value_str = (str(_value) - if not isinstance(_value, string_types) else _value) - if isinstance(_value, bool): - _value_str = "Yes" if _value else "No" - _value_str = click.style(_value_str, fg="green") + if raw_value != options['value']: + default_formatted_value = format_value(options['value']) + formatted_value += "%s" % ( + "\n" if len(default_formatted_value) > 10 else " ") + formatted_value += "[%s]" % click.style(default_formatted_value, + fg="yellow") - if _value != _data['value']: - _defvalue_str = str(_data['value']) - if isinstance(_data['value'], bool): - _defvalue_str = "Yes" if _data['value'] else "No" - _value_str += click.style(" [%s]" % _defvalue_str, fg="yellow") - else: - _value_str += click.style(" ", fg="yellow") + tabular_data.append( + (click.style(key, + fg="cyan"), formatted_value, options['description'])) - click.echo( - list_tpl.format(name=click.style(_name, fg="cyan"), - value=_value_str, - description=_data['description'])) + click.echo( + tabulate(tabular_data, + headers=["Name", "Current value [Default]", "Description"])) @cli.command("set", short_help="Set new value for the setting") diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index d330a4107c..ce50b5d988 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -20,9 +20,9 @@ from time import time import click +from tabulate import tabulate -from platformio import exception, util -from platformio.commands.run.helpers import print_header +from platformio import exception, fs, util from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.native import NativeTestProcessor from platformio.project.config import ProjectConfig @@ -76,7 +76,7 @@ def cli( # pylint: disable=redefined-builtin ctx, environment, ignore, filter, upload_port, test_port, project_dir, project_conf, without_building, without_uploading, without_testing, no_reset, monitor_rts, monitor_dtr, verbose): - with util.cd(project_dir): + with fs.cd(project_dir): test_dir = get_project_test_dir() if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) @@ -87,12 +87,12 @@ def cli( # pylint: disable=redefined-builtin config.validate(envs=environment) click.echo("Verbose mode can be enabled via `-v, --verbose` option") - click.echo("Collected %d items" % len(test_names)) + click.secho("Collected %d items" % len(test_names), bold=True) results = [] - start_time = time() default_envs = config.default_envs() for testname in test_names: + for envname in config.envs(): section = "env:%s" % envname @@ -114,9 +114,12 @@ def cli( # pylint: disable=redefined-builtin for p in patterns['ignore']]), ] if any(skip_conditions): - results.append((None, testname, envname)) + results.append({"env": envname, "test": testname}) continue + click.echo() + print_processing_header(testname, envname) + cls = (NativeTestProcessor if config.get(section, "platform") == "native" else EmbeddedTestProcessor) @@ -133,43 +136,24 @@ def cli( # pylint: disable=redefined-builtin monitor_rts=monitor_rts, monitor_dtr=monitor_dtr, verbose=verbose)) - results.append((tp.process(), testname, envname)) + result = { + "env": envname, + "test": testname, + "duration": time(), + "succeeded": tp.process() + } + result['duration'] = time() - result['duration'] + results.append(result) + + print_processing_footer(result) if without_testing: return - passed_nums = 0 - failed_nums = 0 - testname_max_len = max([len(r[1]) for r in results]) - envname_max_len = max([len(click.style(r[2], fg="cyan")) for r in results]) - - print_header("[%s]" % click.style("TEST SUMMARY")) - click.echo() + print_testing_summary(results) - for result in results: - status, testname, envname = result - if status is False: - failed_nums += 1 - status_str = click.style("FAILED", fg="red") - elif status is None: - status_str = click.style("IGNORED", fg="yellow") - else: - passed_nums += 1 - status_str = click.style("PASSED", fg="green") - - format_str = "test/{:<%d} > {:<%d}\t[{}]" % (testname_max_len, - envname_max_len) - click.echo(format_str.format(testname, click.style(envname, fg="cyan"), - status_str), - err=status is False) - - print_header("%s%d passed in %.2f seconds" % - ("%d failed, " % failed_nums if failed_nums else "", - passed_nums, time() - start_time), - is_error=failed_nums, - fg="red" if failed_nums else "green") - - if failed_nums: + command_failed = any(r.get("succeeded") is False for r in results) + if command_failed: raise exception.ReturnErrorCode(1) @@ -181,3 +165,58 @@ def get_test_names(test_dir): if not names: names = ["*"] return names + + +def print_processing_header(test, env): + click.echo("Processing %s in %s environment" % (click.style( + test, fg="yellow", bold=True), click.style(env, fg="cyan", bold=True))) + terminal_width, _ = click.get_terminal_size() + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" % + ((click.style("FAILED", fg="red", bold=True) if is_failed else + click.style("PASSED", fg="green", bold=True)), result['duration']), + is_error=is_failed) + + +def print_testing_summary(results): + click.echo() + + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + status_str = "IGNORED" + else: + succeeded_nums += 1 + status_str = click.style("PASSED", fg="green") + + tabular_data.append( + (result['test'], click.style(result['env'], fg="cyan"), status_str, + util.humanize_duration_time(result.get("duration")))) + + click.echo(tabulate(tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Test", "Environment", "Status", + "Duration") + ]), + err=failed_nums) + + util.print_labeled_bar( + "%s%d succeeded in %s" % + ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, + util.humanize_duration_time(duration)), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 681bfb44b7..6c1c57c4f8 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -28,7 +28,7 @@ class EmbeddedTestProcessor(TestProcessorBase): def process(self): if not self.options['without_building']: - self.print_progress("Building... (1/3)") + self.print_progress("Building...") target = ["__test"] if self.options['without_uploading']: target.append("checkprogsize") @@ -36,7 +36,7 @@ def process(self): return False if not self.options['without_uploading']: - self.print_progress("Uploading... (2/3)") + self.print_progress("Uploading...") target = ["upload"] if self.options['without_building']: target.append("nobuild") @@ -48,7 +48,7 @@ def process(self): if self.options['without_testing']: return None - self.print_progress("Testing... (3/3)") + self.print_progress("Testing...") return self.run() def run(self): diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index 7367094f9a..e5ba8f3df6 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -14,7 +14,7 @@ from os.path import join -from platformio import util +from platformio import fs, proc from platformio.commands.test.processor import TestProcessorBase from platformio.proc import LineBufferedAsyncPipe from platformio.project.helpers import get_project_build_dir @@ -24,18 +24,18 @@ class NativeTestProcessor(TestProcessorBase): def process(self): if not self.options['without_building']: - self.print_progress("Building... (1/2)") + self.print_progress("Building...") if not self.build_or_upload(["__test"]): return False if self.options['without_testing']: return None - self.print_progress("Testing... (2/2)") + self.print_progress("Testing...") return self.run() def run(self): - with util.cd(self.options['project_dir']): + with fs.cd(self.options['project_dir']): build_dir = get_project_build_dir() - result = util.exec_command( + result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), stderr=LineBufferedAsyncPipe(self.on_run_out)) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index d3029a3412..085ffb2c6c 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -20,7 +20,6 @@ import click from platformio import exception -from platformio.commands.run.helpers import print_header from platformio.project.helpers import get_project_test_dir TRANSPORT_OPTIONS = { @@ -40,14 +39,6 @@ "begin": "pc.baud($baudrate)", "end": "" }, - "energia": { - "include": "#include ", - "object": "", - "putchar": "Serial.write(c)", - "flush": "Serial.flush()", - "begin": "Serial.begin($baudrate)", - "end": "Serial.end()" - }, "espidf": { "include": "#include ", "object": "", @@ -108,12 +99,8 @@ def get_transport(self): def get_baudrate(self): return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE)) - def print_progress(self, text, is_error=False): - click.echo() - print_header("[test/%s > %s] %s" % - (click.style(self.test_name, fg="yellow"), - click.style(self.env_name, fg="cyan"), text), - is_error=is_error) + def print_progress(self, text): + click.secho(text, bold=self.options.get("verbose")) def build_or_upload(self, target): if not self._outputcpp_generated: @@ -123,9 +110,6 @@ def build_or_upload(self, target): if self.test_name != "*": self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name - if not self.options['verbose']: - click.echo("Please wait...") - try: from platformio.commands.run import cli as cmd_run return self.cmd_ctx.invoke(cmd_run, @@ -164,7 +148,11 @@ def generate_outputcpp(self, test_dir): "", "$object", "", + "#ifdef __GNUC__", + "void output_start(unsigned int baudrate __attribute__((unused)))", + "#else", "void output_start(unsigned int baudrate)", + "#endif", "{", " $begin;", "}", diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 91286230c9..a70e778040 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -21,7 +21,6 @@ from platformio import VERSION, __version__, exception, util from platformio.compat import WINDOWS -from platformio.managers.core import shutdown_piohome_servers from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -38,9 +37,6 @@ def cli(dev): click.secho("Please wait while upgrading PlatformIO ...", fg="yellow") - # kill all PIO Home servers, they block `pioplus` binary - shutdown_piohome_servers() - to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") cmds = (["pip", "install", "--upgrade", get_pip_package(to_develop)], ["platformio", "--version"]) diff --git a/platformio/fs.py b/platformio/fs.py new file mode 100644 index 0000000000..a5f61ce590 --- /dev/null +++ b/platformio/fs.py @@ -0,0 +1,163 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import shutil +import stat +import sys +from glob import glob + +import click + +from platformio import exception +from platformio.compat import get_file_contents, glob_escape + + +class cd(object): + + def __init__(self, new_path): + self.new_path = new_path + self.prev_path = os.getcwd() + + def __enter__(self): + os.chdir(self.new_path) + + def __exit__(self, etype, value, traceback): + os.chdir(self.prev_path) + + +def get_source_dir(): + curpath = os.path.abspath(__file__) + if not os.path.isfile(curpath): + for p in sys.path: + if os.path.isfile(os.path.join(p, __file__)): + curpath = os.path.join(p, __file__) + break + return os.path.dirname(curpath) + + +def load_json(file_path): + try: + with open(file_path, "r") as f: + return json.load(f) + except ValueError: + raise exception.InvalidJSONFile(file_path) + + +def format_filesize(filesize): + base = 1024 + unit = 0 + suffix = "B" + filesize = float(filesize) + if filesize < base: + return "%d%s" % (filesize, suffix) + for i, suffix in enumerate("KMGTPEZY"): + unit = base**(i + 2) + if filesize >= unit: + continue + if filesize % (base**(i + 1)): + return "%.2f%sB" % ((base * filesize / unit), suffix) + break + return "%d%sB" % ((base * filesize / unit), suffix) + + +def ensure_udev_rules(): + from platformio.util import get_systype + + def _rules_to_set(rules_path): + return set(l.strip() for l in get_file_contents(rules_path).split("\n") + if l.strip() and not l.startswith("#")) + + if "linux" not in get_systype(): + return None + installed_rules = [ + "/etc/udev/rules.d/99-platformio-udev.rules", + "/lib/udev/rules.d/99-platformio-udev.rules" + ] + if not any(os.path.isfile(p) for p in installed_rules): + raise exception.MissedUdevRules + + origin_path = os.path.abspath( + os.path.join(get_source_dir(), "..", "scripts", + "99-platformio-udev.rules")) + if not os.path.isfile(origin_path): + return None + + origin_rules = _rules_to_set(origin_path) + for rules_path in installed_rules: + if not os.path.isfile(rules_path): + continue + current_rules = _rules_to_set(rules_path) + if not origin_rules <= current_rules: + raise exception.OutdatedUdevRules(rules_path) + + return True + + +def path_endswith_ext(path, extensions): + if not isinstance(extensions, (list, tuple)): + extensions = [extensions] + for ext in extensions: + if path.endswith("." + ext): + return True + return False + + +def match_src_files(src_dir, src_filter=None, src_exts=None): + + def _append_build_item(items, item, src_dir): + if not src_exts or path_endswith_ext(item, src_exts): + items.add(item.replace(src_dir + os.sep, "")) + + src_filter = src_filter or "" + if isinstance(src_filter, (list, tuple)): + src_filter = " ".join(src_filter) + + matches = set() + # correct fs directory separator + src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) + for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): + items = set() + for item in glob(os.path.join(glob_escape(src_dir), pattern)): + if os.path.isdir(item): + for root, _, files in os.walk(item, followlinks=True): + for f in files: + _append_build_item(items, os.path.join(root, f), + src_dir) + else: + _append_build_item(items, item, src_dir) + if action == "+": + matches |= items + else: + matches -= items + return sorted(list(matches)) + + +def rmtree(path): + + def _onerror(func, path, __): + try: + st_mode = os.stat(path).st_mode + if st_mode & stat.S_IREAD: + os.chmod(path, st_mode | stat.S_IWRITE) + func(path) + except Exception as e: # pylint: disable=broad-except + click.secho("%s \nPlease manually remove the file `%s`" % + (str(e), path), + fg="red", + err=True) + + return shutil.rmtree(path, onerror=_onerror) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index debdeeaba2..afc544734b 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -20,7 +20,7 @@ import bottle -from platformio import util +from platformio import fs, util from platformio.compat import WINDOWS, get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig @@ -32,54 +32,90 @@ class ProjectGenerator(object): - def __init__(self, project_dir, ide, env_name): + def __init__(self, project_dir, ide, boards): + self.config = ProjectConfig.get_instance( + join(project_dir, "platformio.ini")) + self.config.validate() self.project_dir = project_dir self.ide = str(ide) - self.env_name = str(env_name) + self.env_name = str(self.get_best_envname(boards)) @staticmethod def get_supported_ides(): - tpls_dir = join(util.get_source_dir(), "ide", "tpls") + tpls_dir = join(fs.get_source_dir(), "ide", "tpls") return sorted( [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) + def get_best_envname(self, boards=None): + envname = None + default_envs = self.config.default_envs() + if default_envs: + envname = default_envs[0] + if not boards: + return envname + + for env in self.config.envs(): + if not boards: + return env + if not envname: + envname = env + items = self.config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in boards: + return env + + return envname + def _load_tplvars(self): - tpl_vars = {"env_name": self.env_name} + tpl_vars = { + "config": self.config, + "systype": util.get_systype(), + "project_name": basename(self.project_dir), + "project_dir": self.project_dir, + "env_name": self.env_name, + "user_home_dir": abspath(expanduser("~")), + "platformio_path": + sys.argv[0] if isfile(sys.argv[0]) + else where_is_program("platformio"), + "env_path": os.getenv("PATH"), + "env_pathsep": os.pathsep + } # yapf: disable + # default env configuration - tpl_vars.update( - ProjectConfig.get_instance(join( - self.project_dir, "platformio.ini")).items(env=self.env_name, - as_dict=True)) + tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) # build data tpl_vars.update( load_project_ide_data(self.project_dir, self.env_name) or {}) - with util.cd(self.project_dir): + with fs.cd(self.project_dir): tpl_vars.update({ - "project_name": basename(self.project_dir), "src_files": self.get_src_files(), - "user_home_dir": abspath(expanduser("~")), - "project_dir": self.project_dir, "project_src_dir": get_project_src_dir(), "project_lib_dir": get_project_lib_dir(), "project_libdeps_dir": join( - get_project_libdeps_dir(), self.env_name), - "systype": util.get_systype(), - "platformio_path": self._fix_os_path( - sys.argv[0] if isfile(sys.argv[0]) - else where_is_program("platformio")), - "env_pathsep": os.pathsep, - "env_path": self._fix_os_path(os.getenv("PATH")) + get_project_libdeps_dir(), self.env_name) + }) # yapf: disable + + for key, value in tpl_vars.items(): + if key.endswith(("_path", "_dir")): + tpl_vars[key] = self.to_unix_path(value) + for key in ("includes", "src_files", "libsource_dirs"): + if key not in tpl_vars: + continue + tpl_vars[key] = [self.to_unix_path(inc) for inc in tpl_vars[key]] + + tpl_vars['to_unix_path'] = self.to_unix_path return tpl_vars @staticmethod - def _fix_os_path(path): - return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) + def to_unix_path(path): + if not WINDOWS or not path: + return path + return re.sub(r"[\\]+", "/", path) def get_src_files(self): result = [] - with util.cd(self.project_dir): + with fs.cd(self.project_dir): for root, _, files in os.walk(get_project_src_dir()): for f in files: result.append(relpath(join(root, f))) @@ -87,7 +123,7 @@ def get_src_files(self): def get_tpls(self): tpls = [] - tpls_dir = join(util.get_source_dir(), "ide", "tpls", self.ide) + tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): diff --git a/platformio/ide/tpls/atom/.gcc-flags.json.tpl b/platformio/ide/tpls/atom/.gcc-flags.json.tpl index 20942a7aae..cbb474186f 100644 --- a/platformio/ide/tpls/atom/.gcc-flags.json.tpl +++ b/platformio/ide/tpls/atom/.gcc-flags.json.tpl @@ -1,9 +1,9 @@ % _defines = " ".join(["-D%s" % d for d in defines]) { - "execPath": "{{ cxx_path.replace("\\", "/") }}", + "execPath": "{{ cxx_path }}", "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccErrorLimit": 15, - "gccIncludePaths": "{{ ','.join(includes).replace("\\", "/") }}", + "gccIncludePaths": "{{ ','.join(includes) }}", "gccSuppressWarnings": false } diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index b3d84a750c..5d69c4d8de 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -17,7 +17,7 @@ % path = path.replace(user_home_dir, "$ENV{HOME}") % end % end -% return path.replace("\\", "/") +% return path % end set(PLATFORMIO_CMD "{{ _normalize_path(platformio_path) }}") diff --git a/platformio/ide/tpls/codeblocks/platformio.cbp.tpl b/platformio/ide/tpls/codeblocks/platformio.cbp.tpl index fc18253179..ccbe4d7f11 100644 --- a/platformio/ide/tpls/codeblocks/platformio.cbp.tpl +++ b/platformio/ide/tpls/codeblocks/platformio.cbp.tpl @@ -53,12 +53,12 @@ % end % for include in includes: - - % end + + % end % for file in src_files: - + % end diff --git a/platformio/ide/tpls/vim/.gcc-flags.json.tpl b/platformio/ide/tpls/vim/.gcc-flags.json.tpl index b904e1a36e..b9b29fdba2 100644 --- a/platformio/ide/tpls/vim/.gcc-flags.json.tpl +++ b/platformio/ide/tpls/vim/.gcc-flags.json.tpl @@ -1,9 +1,9 @@ % _defines = " ".join(["-D%s" % d for d in defines]) { - "execPath": "{{ cxx_path.replace("\\", "/") }}", + "execPath": "{{ cxx_path }}", "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccErrorLimit": 15, - "gccIncludePaths": "{{! ','.join("'{}'".format(w.replace("\\", '/')) for w in includes)}}", + "gccIncludePaths": "{{! ','.join("'{}'".format(inc) for inc in includes)}}", "gccSuppressWarnings": false } diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 1658bcb439..7dd647fcf0 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -10,7 +10,7 @@ % systype = platform.system().lower() % % def _escape(text): -% return text.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') +% return to_unix_path(text).replace('"', '\\"') % end % % cleaned_includes = [] @@ -30,16 +30,15 @@ % end "includePath": [ % for include in cleaned_includes: - "{{! _escape(include) }}", + "{{ include }}", % end "" ], "browse": { "limitSymbolsToIncludedHeaders": true, - "databaseFilename": "${workspaceRoot}/.vscode/.browse.c_cpp.db", "path": [ % for include in cleaned_includes: - "{{! _escape(include) }}", + "{{ include }}", % end "" ] @@ -65,7 +64,7 @@ % if cxx_stds: "cppStandard": "c++{{ cxx_stds[-1] }}", % end - "compilerPath": "\"{{! _escape(cc_path) }}\" {{! _escape(cc_m_flags) }}" + "compilerPath": "\"{{cc_path}}\" {{! _escape(cc_m_flags) }}" } ], "version": 4 diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 56712c25ca..012644d1d6 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -19,7 +19,7 @@ import click import semantic_version -from platformio import __version__, app, exception, telemetry, util +from platformio import __version__, app, exception, fs, telemetry, util from platformio.commands import PlatformioCLI from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update @@ -208,7 +208,7 @@ def check_platformio_upgrade(): fg="cyan", nl=False) click.secho("`.", fg="yellow") - elif join("Cellar", "platformio") in util.get_source_dir(): + elif join("Cellar", "platformio") in fs.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) click.secho("` command.", fg="yellow") else: diff --git a/platformio/managers/core.py b/platformio/managers/core.py index f93145fb6d..74d8afe35e 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -16,11 +16,8 @@ import subprocess import sys from os.path import dirname, join -from time import sleep -import requests - -from platformio import __version__, exception, util +from platformio import __version__, exception, fs from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path @@ -99,25 +96,10 @@ def update_core_packages(only_check=False, silent=False): if not pkg_dir: continue if not silent or pm.outdated(pkg_dir, requirements): - if name == "tool-pioplus" and not only_check: - shutdown_piohome_servers() - if WINDOWS: - sleep(1) pm.update(name, requirements, only_check=only_check) return True -def shutdown_piohome_servers(): - port = 8010 - while port < 8050: - try: - requests.get("http://127.0.0.1:%d?__shutdown__=1" % port, - timeout=0.01) - except: # pylint: disable=bare-except - pass - port += 1 - - def inject_contrib_pysite(): from site import addsitedir contrib_pysite_dir = get_core_package_dir("contrib-pysite") @@ -138,7 +120,7 @@ def pioplus_call(args, **kwargs): pythonexe_path = get_pythonexe_path() os.environ['PYTHONEXEPATH'] = pythonexe_path os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") - os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "") + os.environ['PIOCOREPYSITEDIR'] = dirname(fs.get_source_dir() or "") if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep): os.environ['PATH'] = (os.pathsep).join( [dirname(pythonexe_path), os.environ['PATH']]) diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index b6f630c8ba..b7fc8e51b0 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -211,7 +211,7 @@ def _install_from_piorepo(self, name, requirements): return self._install_from_url( name, dl_data['url'].replace("http://", "https://") - if app.get_setting("enable_ssl") else dl_data['url'], requirements) + if app.get_setting("strict_ssl") else dl_data['url'], requirements) def search_lib_id( # pylint: disable=too-many-branches self, diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 82043114fb..9d99b1a366 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -25,7 +25,7 @@ import requests import semantic_version -from platformio import __version__, app, exception, telemetry, util +from platformio import __version__, app, exception, fs, telemetry, util from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile @@ -359,13 +359,13 @@ def load_manifest(self, pkg_dir): manifest_path = self.get_manifest_path(pkg_dir) src_manifest_path = self.get_src_manifest_path(pkg_dir) if src_manifest_path: - src_manifest = util.load_json(src_manifest_path) + src_manifest = fs.load_json(src_manifest_path) if not manifest_path and not src_manifest_path: return None if manifest_path and manifest_path.endswith(".json"): - manifest = util.load_json(manifest_path) + manifest = fs.load_json(manifest_path) elif manifest_path and manifest_path.endswith(".properties"): with codecs.open(manifest_path, encoding="utf-8") as fp: for line in fp.readlines(): @@ -498,7 +498,7 @@ def _install_from_url(self, if isfile(_url): self.unpack(_url, tmp_dir) else: - util.rmtree_(tmp_dir) + fs.rmtree(tmp_dir) shutil.copytree(_url, tmp_dir) elif url.startswith(("http://", "https://")): dlpath = self.download(url, tmp_dir, sha1) @@ -523,7 +523,7 @@ def _install_from_url(self, return self._install_from_tmp_dir(_tmp_dir, requirements) finally: if isdir(tmp_dir): - util.rmtree_(tmp_dir) + fs.rmtree(tmp_dir) return None def _update_src_manifest(self, data, src_dir): @@ -532,7 +532,7 @@ def _update_src_manifest(self, data, src_dir): src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME) _data = {} if isfile(src_manifest_path): - _data = util.load_json(src_manifest_path) + _data = fs.load_json(src_manifest_path) _data.update(data) with open(src_manifest_path, "w") as fp: json.dump(_data, fp) @@ -602,7 +602,7 @@ def _install_from_tmp_dir( # pylint: disable=too-many-branches # remove previous/not-satisfied package if isdir(pkg_dir): - util.rmtree_(pkg_dir) + fs.rmtree(pkg_dir) shutil.move(tmp_dir, pkg_dir) assert isdir(pkg_dir) self.cache_reset() @@ -768,7 +768,7 @@ def uninstall(self, package, requirements=None, after_update=False): if islink(pkg_dir): os.unlink(pkg_dir) else: - util.rmtree_(pkg_dir) + fs.rmtree(pkg_dir) self.cache_reset() # unfix package with the same name diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 1a45e433c0..996d7162d7 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -22,7 +22,7 @@ import click import semantic_version -from platformio import __version__, app, exception, util +from platformio import __version__, app, exception, fs, util from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager @@ -47,7 +47,7 @@ def __init__(self, package_dir=None, repositories=None): repositories = [ "https://dl.bintray.com/platformio/dl-platforms/manifest.json", "{0}://dl.platformio.org/platforms/manifest.json".format( - "https" if app.get_setting("enable_ssl") else "http") + "https" if app.get_setting("strict_ssl") else "http") ] BasePkgManager.__init__(self, package_dir or get_project_platforms_dir(), repositories) @@ -237,7 +237,7 @@ def newPlatform(cls, name, requirements=None): name = pm.load_manifest(platform_dir)['name'] elif name.endswith("platform.json") and isfile(name): platform_dir = dirname(name) - name = util.load_json(name)['name'] + name = fs.load_json(name)['name'] else: name, requirements, url = pm.parse_pkg_uri(name, requirements) platform_dir = pm.get_package_dir(name, requirements, url) @@ -404,7 +404,7 @@ def _run_scons(self, variables, targets, jobs): join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", "--warn=no-no-parallel-support", "--jobs", str(jobs), - "--sconstruct", join(util.get_source_dir(), "builder", "main.py") + "--sconstruct", join(fs.get_source_dir(), "builder", "main.py") ] # yapf: disable args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) # pylint: disable=protected-access @@ -494,7 +494,7 @@ def __init__(self, manifest_path): self.verbose = False self._BOARDS_CACHE = {} - self._manifest = util.load_json(manifest_path) + self._manifest = fs.load_json(manifest_path) self._custom_packages = None self.pm = PackageManager(get_project_packages_dir(), @@ -693,7 +693,7 @@ def __init__(self, manifest_path): assert isfile(manifest_path) self.manifest_path = manifest_path try: - self._manifest = util.load_json(manifest_path) + self._manifest = fs.load_json(manifest_path) except ValueError: raise exception.InvalidBoardManifest(manifest_path) if not set(["name", "url", "vendor"]) <= set(self._manifest): diff --git a/platformio/project/config.py b/platformio/project/config.py index dc4558097f..e98ace551d 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -16,7 +16,7 @@ import json import os import re -from os.path import isfile +from os.path import expanduser, isfile import click @@ -106,6 +106,8 @@ def read(self, path, parse_extra=True): # load extra configs for pattern in self.get("platformio", "extra_configs", []): + if pattern.startswith("~"): + pattern = expanduser(pattern) for item in glob.glob(pattern): self.read(item) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 3c01766637..0be42893a2 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -165,23 +165,33 @@ def get_project_shared_dir(): join(get_project_dir(), "shared")) -def calculate_project_hash(): +def compute_project_checksum(config): + # rebuild when PIO Core version changes + checksum = sha1(hashlib_encode_data(__version__)) + + # configuration file state + checksum.update(hashlib_encode_data(config.to_json())) + + # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - chunks = [__version__] - for d in (get_project_src_dir(), get_project_lib_dir()): + for d in (get_project_include_dir(), get_project_src_dir(), + get_project_lib_dir()): if not isdir(d): continue + chunks = [] for root, _, files in walk(d): for f in files: path = join(root, f) if path.endswith(check_suffixes): chunks.append(path) - chunks_to_str = ",".join(sorted(chunks)) - if WINDOWS: - # Fix issue with useless project rebuilding for case insensitive FS. - # A case of disk drive can differ... - chunks_to_str = chunks_to_str.lower() - return sha1(hashlib_encode_data(chunks_to_str)).hexdigest() + if not chunks: + continue + chunks_to_str = ",".join(sorted(chunks)) + if WINDOWS: # case insensitive OS + chunks_to_str = chunks_to_str.lower() + checksum.update(hashlib_encode_data(chunks_to_str)) + + return checksum.hexdigest() def load_project_ide_data(project_dir, env_name): diff --git a/platformio/telemetry.py b/platformio/telemetry.py index d35aac40e4..4325dd15af 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -284,32 +284,12 @@ def on_command(): def measure_ci(): event = {"category": "CI", "action": "NoName", "label": None} - - envmap = { - "APPVEYOR": { - "label": getenv("APPVEYOR_REPO_NAME") - }, - "CIRCLECI": { - "label": - "%s/%s" % (getenv("CIRCLE_PROJECT_USERNAME"), - getenv("CIRCLE_PROJECT_REPONAME")) - }, - "TRAVIS": { - "label": getenv("TRAVIS_REPO_SLUG") - }, - "SHIPPABLE": { - "label": getenv("REPO_NAME") - }, - "DRONE": { - "label": getenv("DRONE_REPO_SLUG") - } - } - - for key, value in envmap.items(): - if getenv(key, "").lower() != "true": - continue - event.update({"action": key, "label": value['label']}) - + known_cis = ("TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE", + "DRONE") + for name in known_cis: + if getenv(name, "false").lower() == "true": + event['action'] = name + break on_event(**event) diff --git a/platformio/util.py b/platformio/util.py index 377160b18a..1903f04cd0 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -13,39 +13,30 @@ # limitations under the License. import json +import math import os import platform import re import socket -import stat import sys import time from contextlib import contextmanager from functools import wraps from glob import glob -from os.path import abspath, basename, dirname, isfile, join -from shutil import rmtree import click import requests from platformio import __apiurl__, __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import PY2, WINDOWS, get_file_contents -from platformio.proc import exec_command, is_ci +from platformio.compat import PY2, WINDOWS +from platformio.fs import cd # pylint: disable=unused-import +from platformio.fs import load_json # pylint: disable=unused-import +from platformio.fs import rmtree as rmtree_ # pylint: disable=unused-import +from platformio.proc import exec_command # pylint: disable=unused-import +from platformio.proc import is_ci # pylint: disable=unused-import - -class cd(object): - - def __init__(self, new_path): - self.new_path = new_path - self.prev_path = os.getcwd() - - def __enter__(self): - os.chdir(self.new_path) - - def __exit__(self, etype, value, traceback): - os.chdir(self.prev_path) +# KEEP unused imports for backward compatibility with PIO Core 3.0 API class memoized(object): @@ -119,14 +110,6 @@ def capture_std_streams(stdout, stderr=None): sys.stderr = _stderr -def load_json(file_path): - try: - with open(file_path, "r") as f: - return json.load(f) - except ValueError: - raise exception.InvalidJSONFile(file_path) - - def get_systype(): type_ = platform.system().lower() arch = platform.machine().lower() @@ -141,16 +124,6 @@ def pioversion_to_intstr(): return [int(i) for i in vermatch.group(1).split(".")[:3]] -def get_source_dir(): - curpath = abspath(__file__) - if not isfile(curpath): - for p in sys.path: - if isfile(join(p, __file__)): - curpath = join(p, __file__) - break - return dirname(curpath) - - def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) @@ -221,7 +194,7 @@ def get_logical_devices(): continue items.append({ "path": match.group(1), - "name": basename(match.group(1)) + "name": os.path.basename(match.group(1)) }) return items @@ -333,7 +306,7 @@ def _get_api_result( headers = get_request_defheaders() if not url.startswith("http"): url = __apiurl__ + url - if not get_setting("enable_ssl"): + if not get_setting("strict_ssl"): url = url.replace("https://", "http://") try: @@ -461,23 +434,6 @@ def parse_date(datestr): return time.strptime(datestr) -def format_filesize(filesize): - base = 1024 - unit = 0 - suffix = "B" - filesize = float(filesize) - if filesize < base: - return "%d%s" % (filesize, suffix) - for i, suffix in enumerate("KMGTPEZY"): - unit = base**(i + 2) - if filesize >= unit: - continue - if filesize % (base**(i + 1)): - return "%.2f%sB" % ((base * filesize / unit), suffix) - break - return "%d%sB" % ((base * filesize / unit), suffix) - - def merge_dicts(d1, d2, path=None): if path is None: path = [] @@ -490,35 +446,25 @@ def merge_dicts(d1, d2, path=None): return d1 -def ensure_udev_rules(): - - def _rules_to_set(rules_path): - return set(l.strip() for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#")) +def print_labeled_bar(label, is_error=False, fg=None): + terminal_width, _ = click.get_terminal_size() + width = len(click.unstyle(label)) + half_line = "=" * int((terminal_width - width - 2) / 2) + click.secho("%s %s %s" % (half_line, label, half_line), + fg=fg, + err=is_error) - if "linux" not in get_systype(): - return None - installed_rules = [ - "/etc/udev/rules.d/99-platformio-udev.rules", - "/lib/udev/rules.d/99-platformio-udev.rules" - ] - if not any(isfile(p) for p in installed_rules): - raise exception.MissedUdevRules - - origin_path = abspath( - join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules")) - if not isfile(origin_path): - return None - origin_rules = _rules_to_set(origin_path) - for rules_path in installed_rules: - if not isfile(rules_path): - continue - current_rules = _rules_to_set(rules_path) - if not origin_rules <= current_rules: - raise exception.OutdatedUdevRules(rules_path) - - return True +def humanize_duration_time(duration): + if duration is None: + return duration + duration = duration * 1000 + tokens = [] + for multiplier in (3600000, 60000, 1000, 1): + fraction = math.floor(duration / multiplier) + tokens.append(int(round(duration) if multiplier == 1 else fraction)) + duration -= fraction * multiplier + return "{:02d}:{:02d}:{:02d}.{:03d}".format(*tokens) def get_original_version(version): @@ -530,18 +476,3 @@ def get_original_version(version): if int(raw) <= 9999: return "%s.%s" % (raw[:-2], int(raw[-2:])) return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) - - -def rmtree_(path): - - def _onerror(_, name, __): - try: - os.chmod(name, stat.S_IWRITE) - os.remove(name) - except Exception as e: # pylint: disable=broad-except - click.secho("%s \nPlease manually remove the file `%s`" % - (str(e), name), - fg="red", - err=True) - - return rmtree(path, onerror=_onerror) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 51a4c9eb93..be7c3e852d 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -30,6 +30,9 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="066 # FT232R USB UART SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE:="0666" +# FT231XS USB UART +SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666" + # Prolific Technology, Inc. PL2303 Serial Port SUBSYSTEMS=="usb", ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666" diff --git a/scripts/docspregen.py b/scripts/docspregen.py index b4e5a67f1b..0a3cf437ca 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -20,7 +20,7 @@ path.append("..") -from platformio import util +from platformio import fs, util from platformio.managers.platform import PlatformFactory, PlatformManager RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO @@ -97,8 +97,8 @@ def generate_boards_table(boards, skip_columns=None): debug=debug, mcu=data['mcu'].upper(), f_cpu=int(data['fcpu']) / 1000000, - ram=util.format_filesize(data['ram']), - rom=util.format_filesize(data['rom'])) + ram=fs.format_filesize(data['ram']), + rom=fs.format_filesize(data['rom'])) for (name, template) in columns: if skip_columns and name in skip_columns: @@ -280,7 +280,7 @@ def generate_packages(platform, packagenames, is_embedded): def generate_platform(name, rst_dir): - print "Processing platform: %s" % name + print("Processing platform: %s" % name) compatible_boards = [ board for board in BOARDS if name == board['platform'] @@ -439,7 +439,7 @@ def update_platform_docs(): def generate_framework(type_, data, rst_dir=None): - print "Processing framework: %s" % type_ + print("Processing framework: %s" % type_) compatible_platforms = [ m for m in PLATFORM_MANIFESTS @@ -614,8 +614,8 @@ def update_embedded_board(rst_path, board): mcu_upper=board['mcu'].upper(), f_cpu=board['fcpu'], f_cpu_mhz=int(board['fcpu']) / 1000000, - ram=util.format_filesize(board['ram']), - rom=util.format_filesize(board['rom']), + ram=fs.format_filesize(board['ram']), + rom=fs.format_filesize(board['rom']), vendor=board['vendor'], board_manifest_url=board_manifest_url, upload_protocol=board_config.get("upload.protocol", "")) @@ -811,7 +811,7 @@ def update_debugging(): # save with open( - join(util.get_source_dir(), "..", "docs", "plus", "debugging.rst"), + join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+") as fp: content = fp.read() fp.seek(0) @@ -880,7 +880,7 @@ def update_project_examples(): {examples} """ - project_examples_dir = join(util.get_source_dir(), "..", "examples") + project_examples_dir = join(fs.get_source_dir(), "..", "examples") framework_examples_md_lines = {} embedded = [] desktop = [] diff --git a/scripts/fixsymlink.py b/scripts/fixsymlink.py index a73a010907..0f9daa2e7c 100644 --- a/scripts/fixsymlink.py +++ b/scripts/fixsymlink.py @@ -18,7 +18,7 @@ def fix_symlink(root, fname, brokenlink): - print root, fname, brokenlink + print(root, fname, brokenlink) prevcwd = getcwd() chdir(root) diff --git a/setup.py b/setup.py index eb667d72bc..66e0f9c893 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,8 @@ "colorama", "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", - "semantic_version>=2.5.0,<3" + "semantic_version>=2.5.0,<3", + "tabulate>=0.8.3" ] setup( @@ -65,6 +66,7 @@ "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", diff --git a/tests/commands/test_boards.py b/tests/commands/test_boards.py index c44ff43846..cd0041c512 100644 --- a/tests/commands/test_boards.py +++ b/tests/commands/test_boards.py @@ -27,9 +27,9 @@ def test_board_json_output(clirunner, validate_cliresult): def test_board_raw_output(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_boards, ["energia"]) + result = clirunner.invoke(cmd_boards, ["espidf"]) validate_cliresult(result) - assert "titiva" in result.output + assert "espressif32" in result.output def test_board_options(clirunner, validate_cliresult): diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 6346c7887f..ca7ce99d19 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -26,5 +26,5 @@ def test_local_env(): ]) if result['returncode'] != 1: pytest.fail(result) - assert all([s in result['out'] + assert all([s in result['err'] for s in ("PASSED", "IGNORED", "FAILED")]), result['out']