Add proto2ros machinery (#44)

Signed-off-by: Michel Hidalgo <[email protected]>
Co-authored-by: Gustavo Goretkin <[email protected]>

proto2ros/CMakeLists.txt

@@ -0,0 +1,35 @@
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
+cmake_minimum_required(VERSION 3.8)
+project(proto2ros)
+
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic)
+endif()
+
+# find dependencies
+find_package(ament_cmake REQUIRED)
+find_package(ament_cmake_python REQUIRED)
+find_package(rosidl_default_generators REQUIRED)
+
+rosidl_generate_interfaces(${PROJECT_NAME}
+  msg/Any.msg msg/AnyProto.msg msg/Bytes.msg msg/List.msg
+  msg/Struct.msg msg/StructEntry.msg msg/Value.msg
+)
+
+include(cmake/rosidl_helpers.cmake)
+rosidl_generated_python_package_add(
+  ${PROJECT_NAME}_additional_modules
+  PACKAGES ${PROJECT_NAME}
+  DESTINATION ${PROJECT_NAME}
+)
+
+install(
+  DIRECTORY cmake
+  DESTINATION share/${PROJECT_NAME}
+)
+
+ament_export_dependencies(builtin_interfaces)
+ament_export_dependencies(rosidl_default_runtime)
+ament_export_dependencies(std_msgs)
+
+ament_package(CONFIG_EXTRAS "proto2ros-extras.cmake")

proto2ros/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Boston Dynamics AI Institute
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

proto2ros/README.md

@@ -0,0 +1,5 @@
+# Protobuf to ROS 2 interoperability 
+
+`proto2ros` helps maintain an interoperability layer between Protobuf dependent and ROS 2 aware code by generating equivalent ROS 2 message definitions given source Protobuf message definitions, as well bi-directional conversion APIs in relevant languages (such as Python). To date, Protobuf syntax versions 2 and 3 are supported but only syntax version 3 has been extensively tested.
+
+Continue on to the [`ros_utilities` wiki](https://github.com/bdaiinstitute/ros_utilities/wiki) for further reference.

proto2ros/cmake/proto2ros_generate.cmake

@@ -0,0 +1,159 @@
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
+
+# Generates Protobuf <-> ROS 2 interoperability interfaces.
+#
+# :param target: the target name for the generation process, so it can be depended on.
+# :param PACKAGE_NAME: name of the package that will host the generated interfaces.
+#   Defaults to the current project name.
+# :param PROTOS: Protobuf message files to generate interoperability interfaces for.
+#   These are compiled to a single Protobuf descriptor file for further processing.
+# :param IMPORT_DIRS: optional import paths to pass to protoc. Only used when PROTOS
+#   are provided. If none is given, parent directories of PROTOS are used instead.
+# :param PROTO_DESCRIPTORS: Protobuf descriptor files for the Protobuf messages
+#   that the generation process will produce interoperability interfaces for.
+# :param CONFIG_FILE: optional base configuration file for the generation process.
+# :param CONFIG_OVERLAYS: optional configuration file overlays to be applied sequentially
+#   over the (provided or default) base configuration file.
+# :param INTERFACES_OUT_VAR: the name of the variable to yield ROS 2 interface tuples.
+#   Defaults to the target name with an `_interfaces` suffix.
+# :param PYTHON_OUT_VAR: the name of the variable to yield generated Python sources.
+#   Defaults to the target name with a `_python_sources` suffix.
+# :param APPEND_PYTHONPATH: optional paths to append to the PYTHONPATH that applies
+#   to the generation process.
+# :param NO_LINT: if provided, no lint tests are added for generated code.
+function(proto2ros_generate target)
+  cmake_parse_arguments(
+    ARG "NO_LINT"
+    "PACKAGE_NAME;CONFIG_FILE;INTERFACES_OUT_VAR;PYTHON_OUT_VAR"
+    "PROTOS;PROTO_DESCRIPTORS;IMPORT_DIRS;CONFIG_OVERLAYS;APPEND_PYTHONPATH" ${ARGN})
+  if(NOT ARG_PACKAGE_NAME)
+    set(ARG_PACKAGE_NAME ${PROJECT_NAME})
+  endif()
+  if(NOT ARG_INTERFACES_OUT_VAR)
+    set(ARG_INTERFACES_OUT_VAR ${target}_interfaces)
+  endif()
+  if(NOT ARG_PYTHON_OUT_VAR)
+    set(ARG_INTERFACES_OUT_VAR ${target}_python_sources)
+  endif()
+  list(APPEND ARG_APPEND_PYTHONPATH "${PROJECT_SOURCE_DIR}")
+  string(REPLACE ";" ":" APPEND_PYTHONPATH "${ARG_APPEND_PYTHONPATH}")
+
+  set(BASE_PATH "${CMAKE_CURRENT_BINARY_DIR}/proto2ros_generate")
+  set(OUTPUT_PATH "${BASE_PATH}/${ARG_PACKAGE_NAME}")
+  file(REMOVE_RECURSE "${OUTPUT_PATH}")
+  file(MAKE_DIRECTORY "${OUTPUT_PATH}")
+
+  foreach(path ${ARG_PROTO_DESCRIPTORS})
+    get_filename_component(path "${path}" ABSOLUTE)
+    list(APPEND PROTO_DESCRIPTORS ${path})
+  endforeach()
+
+  if(ARG_PROTOS)
+    set(protoc_options --include_source_info)
+    set(proto_descriptor "${CMAKE_CURRENT_BINARY_DIR}/${target}.desc")
+    foreach(proto ${ARG_PROTOS})
+      get_filename_component(proto_path "${proto}" ABSOLUTE)
+      if(IS_DIRECTORY "${proto_path}")
+        file(GLOB_RECURSE nested_files "${proto_path}" *.proto)
+        if(NOT ARG_IMPORT_DIRS)
+          list(APPEND protoc_options "-I${proto_path}")
+        endif()
+        list(APPEND proto_files ${nested_files})
+      else()
+        get_filename_component(proto_dir "${proto_path}" DIRECTORY)
+        if(NOT ARG_IMPORT_DIRS)
+          list(APPEND protoc_options "-I${proto_dir}")
+        endif()
+        list(APPEND proto_files "${proto_path}")
+      endif()
+    endforeach()
+    foreach(path ${ARG_IMPORT_DIRS})
+      get_filename_component(path "${path}" ABSOLUTE)
+      list(APPEND protoc_options "-I${path}")
+    endforeach()
+    list(REMOVE_DUPLICATES protoc_options)
+
+    # Generate the implicit descriptor file at configuration time
+    # so that configuration code below can run.
+    get_executable_path(PROTOC_EXECUTABLE protobuf::protoc CONFIGURE)
+    execute_process(
+      COMMAND
+        ${PROTOC_EXECUTABLE} ${protoc_options} -o${proto_descriptor} ${proto_files}
+      COMMAND_ERROR_IS_FATAL ANY
+    )
+
+    add_custom_command(
+      OUTPUT ${proto_descriptor}
+      COMMAND ${PROTOC_EXECUTABLE} ${protoc_options} -o${proto_descriptor} ${proto_files}
+      DEPENDS ${proto_files}
+      COMMENT "Compile descriptor from .proto files"
+      VERBATIM
+    )
+    list(APPEND PROTO_DESCRIPTORS ${proto_descriptor})
+  endif()
+
+  if(NOT PROTO_DESCRIPTORS)
+    message(FATAL_ERROR "No Protobuf descriptors to process")
+  endif()
+
+  if(ARG_CONFIG_FILE)
+    get_filename_component(ARG_CONFIG_FILE "${ARG_CONFIG_FILE}" ABSOLUTE)
+    list(APPEND PROTO2ROS_GENERATE_OPTIONS "-c" "${ARG_CONFIG_FILE}")
+  endif()
+  foreach(overlay ${ARG_CONFIG_OVERLAYS})
+    get_filename_component(overlay "${overlay}" ABSOLUTE)
+    list(APPEND PROTO2ROS_GENERATE_OPTIONS "-a" "${overlay}")
+  endforeach()
+  list(APPEND PROTO2ROS_GENERATE_OPTIONS "-m" "${OUTPUT_PATH}/manifest.txt")
+  list(APPEND PROTO2ROS_GENERATE_OPTIONS "-O" "${OUTPUT_PATH}")
+
+  # As we cannot deduce what files will result from the generation process ahead of time,
+  # we perform a dry run at configuration time to determine these files. Build will be
+  # forced to fail until reconfiguration whenever the set of output files changes. Note
+  # that messages are also generated, as the rosidl pipeline requires them to exist at
+  # configuration time.
+  get_executable_path(PYTHON_EXECUTABLE Python3::Interpreter CONFIGURE)
+  execute_process(
+    COMMAND
+      ${CMAKE_COMMAND} -E env
+        "PYTHONPATH=${APPEND_PYTHONPATH}:$ENV{PYTHONPATH}"
+        "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python"
+      ${PYTHON_EXECUTABLE} -m proto2ros.cli.generate --dry --force-message-gen
+        ${PROTO2ROS_GENERATE_OPTIONS} ${ARG_PACKAGE_NAME} ${PROTO_DESCRIPTORS}
+      COMMAND_ERROR_IS_FATAL ANY
+    )
+  file(STRINGS "${OUTPUT_PATH}/manifest.txt" output_files)
+  file(RENAME "${OUTPUT_PATH}/manifest.txt" "${OUTPUT_PATH}/manifest.orig.txt")
+
+  add_custom_command(
+    OUTPUT ${output_files}
+    COMMAND
+      ${CMAKE_COMMAND} -E env "PYTHONPATH=${APPEND_PYTHONPATH}:$ENV{PYTHONPATH}" "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python"
+      ${PYTHON_EXECUTABLE} -m proto2ros.cli.generate ${PROTO2ROS_GENERATE_OPTIONS} ${ARG_PACKAGE_NAME} ${PROTO_DESCRIPTORS}
+    COMMAND ${CMAKE_COMMAND} -E compare_files "${OUTPUT_PATH}/manifest.txt" "${OUTPUT_PATH}/manifest.orig.txt"
+    COMMENT "Generate Protobuf <-> ROS interop interfaces (must reconfigure if the cardinality of the output set changes)"
+    DEPENDS ${PROTO_DESCRIPTORS}
+    VERBATIM
+  )
+  add_custom_target(${target} DEPENDS ${output_files})
+
+  set(interface_files ${output_files})
+  list(FILTER interface_files INCLUDE REGEX ".*\.msg$")
+  string(REPLACE "${OUTPUT_PATH}/" "${OUTPUT_PATH}:" interface_tuples "${interface_files}")
+  set(python_sources ${output_files})
+  list(FILTER python_sources INCLUDE REGEX ".*\.py$")
+
+  if(BUILD_TESTING AND NOT ARG_NO_LINT AND ament_cmake_mypy_FOUND)
+    set(MYPY_PATH "${APPEND_PYTHONPATH}:$ENV{PYTHONPATH}")
+    configure_file(
+      "${proto2ros_DIR}/templates/mypy.ini.in"
+      "${BASE_PATH}/mypy.ini" @ONLY
+    )
+    ament_mypy(${python_sources}
+      TESTNAME ${target}_mypy
+      CONFIG_FILE "${BASE_PATH}/mypy.ini"
+    )
+  endif()
+  set(${ARG_INTERFACES_OUT_VAR} ${interface_tuples} PARENT_SCOPE)
+  set(${ARG_PYTHON_OUT_VAR} ${python_sources} PARENT_SCOPE)
+endfunction()

proto2ros/cmake/proto2ros_vendor_package.cmake

@@ -0,0 +1,64 @@
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
+
+# Vendors a package providing Protobuf messages, adding ROS 2 interoperability interfaces.
+#
+# :param target: the target name for the process, so it can be depended on.
+# :param PACKAGE_NAME: name of the package that will host the generated interfaces.
+#   Defaults to the current project name.
+# :param PROTOS: Protobuf message files to generate interoperability interfaces for.
+#   These are compiled to a single Protobuf descriptor file for further processing.
+# :param IMPORT_DIRS: optional import paths to pass to protoc. Only used when PROTOS
+#   are provided. If none is given, parent directories of PROTOS are used instead.
+# :param CONFIG_OVERLAYS: optional configuration file overlays to be applied sequentially
+#   over the default base configuration file.
+#
+macro(proto2ros_vendor_package target)
+  set(options NO_LINT)
+  set(one_value_keywords PACKAGE_NAME)
+  set(multi_value_keywords PROTOS IMPORT_DIRS CONFIG_OVERLAYS ROS_DEPENDENCIES PYTHON_MODULES PYTHON_PACKAGES)
+  cmake_parse_arguments(ARG "${options}" "${one_value_keywords}" "${multi_value_keywords}" ${ARGN})
+
+  if(NOT ARG_PACKAGE_NAME)
+    set(ARG_PACKAGE_NAME ${PROJECT_NAME})
+  endif()
+
+  set(proto2ros_generate_OPTIONS)
+  if(ARG_NO_LINT)
+    list(APPEND proto2ros_generate_OPTIONS NO_LINT)
+  endif()
+
+  proto2ros_generate(
+    ${target}_messages_gen
+    PROTOS ${ARG_PROTOS}
+    IMPORT_DIRS ${ARG_IMPORT_DIRS}
+    PACKAGE_NAME ${ARG_PACKAGE_NAME}
+    CONFIG_OVERLAYS ${ARG_CONFIG_OVERLAYS}
+    INTERFACES_OUT_VAR ros_messages
+    PYTHON_OUT_VAR py_sources
+    ${proto2ros_generate_OPTIONS}
+  )
+
+  set(rosidl_generate_interfaces_OPTIONS)
+  if(NOT ARG_NO_LINT)
+    list(APPEND rosidl_generate_interfaces_OPTIONS ADD_LINTER_TESTS)
+  endif()
+
+  rosidl_generate_interfaces(
+    ${target} ${ros_messages}
+    DEPENDENCIES ${ARG_ROS_DEPENDENCIES} builtin_interfaces proto2ros
+    ${rosidl_generate_interfaces_OPTIONS}
+  )
+  add_dependencies(${target} ${target}_messages_gen)
+
+  get_filename_component(package_path "${ARG_PACKAGE_NAME}" ABSOLUTE)
+  if(EXISTS "${package_path}/__init__.py")
+    list(APPEND ARG_PYTHON_PACKAGES ${ARG_PACKAGE_NAME})
+  endif()
+
+  rosidl_generated_python_package_add(
+    ${target}_additional_modules
+    MODULES ${ARG_PYTHON_MODULES} ${py_sources}
+    PACKAGES ${ARG_PYTHON_PACKAGES}
+    DESTINATION ${target}
+  )
+endmacro()

proto2ros/cmake/rosidl_helpers.cmake

@@ -0,0 +1,81 @@
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
+
+# Adds Python sources to an existing rosidl generated Python package.
+#
+# :param target: the target name for the file transfer, so it can be depended on.
+# :param MODULES: Python modules (i.e. files) to be added. These will copied
+#   under the rosidl generated Python package. No directory structure will
+#   be kept.
+# :param PACKAGES: Python packages (i.e. directories) to be added. Their content
+#   will be copied under the rosidl generated Python Package. Their directory
+#   structure will be kept.
+# :param DESTINATION: the rosidl_generate_interfaces target that the destination
+#   Python package is associated with. Defaults to the current project name.
+function(rosidl_generated_python_package_add target)
+  cmake_parse_arguments(ARG "" "DESTINATION" "MODULES;PACKAGES" ${ARGN})
+  if(NOT ARG_DESTINATION)
+    set(ARG_DESTINATION ${PROJECT_NAME})
+  endif()
+  if(NOT ARG_MODULES AND NOT ARG_PACKAGES)
+    message(FATAL_ERROR "No modules nor packages to add")
+  endif()
+  set(OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_py/${ARG_DESTINATION}")
+
+  if(ARG_MODULES)
+    unset(input_files)
+    unset(output_files)
+    foreach(module ${ARG_MODULES})
+      get_filename_component(module_name "${module}" NAME)
+      list(APPEND output_files "${OUTPUT_DIR}/${module_name}")
+      get_filename_component(module_path "${module}" ABSOLUTE)
+      list(APPEND input_files "${module_path}")
+    endforeach()
+    add_custom_command(
+      OUTPUT ${output_files}
+      COMMAND ${CMAKE_COMMAND} -E copy ${input_files} ${OUTPUT_DIR}/.
+      DEPENDS ${input_files}
+    )
+    list(APPEND OUTPUT_FILES ${output_files})
+  endif()
+
+  if(ARG_PACKAGES)
+    unset(input_files)
+    unset(output_files)
+    foreach(package ${ARG_PACKAGES})
+      get_filename_component(package_path "${package}" ABSOLUTE)
+      file(GLOB_RECURSE package_files "${package_path}" *.py)
+      foreach(input_file ${package_files})
+        get_filename_component(module_name "${module}" NAME)
+        file(RELATIVE_PATH output_file ${package_path} "${input_file}")
+        list(APPEND output_files "${OUTPUT_DIR}/${output_file}")
+      endforeach()
+      list(APPEND input_dirs "${package_path}")
+      list(APPEND input_files "${package_files}")
+    endforeach()
+    add_custom_command(
+      OUTPUT ${output_files}
+      COMMAND ${CMAKE_COMMAND} -E copy_directory ${input_dirs} ${OUTPUT_DIR}/.
+      DEPENDS ${input_files} ${input_dirs}
+    )
+    list(APPEND OUTPUT_FILES ${output_files})
+  endif()
+
+  add_custom_target(${target} ALL DEPENDS ${OUTPUT_FILES})
+  add_dependencies(${target} ${ARG_DESTINATION})
+endfunction()
+
+# Exports relevant variables to setup tests that depend on rosidl generated artifacts.
+#
+# :param LIBRARY_DIRS: the name of the variable to yield the paths
+#   where generated shared library may be found.
+# :param ENV: the name of the variable to yield relevant environment
+#   variable values pointing to generated artifacts (such as PYTHONPATH).
+function(get_rosidl_generated_interfaces_test_setup)
+  cmake_parse_arguments(ARG "" "LIBRARY_DIRS;ENV" "" ${ARGN})
+  if(ARG_LIBRARY_DIRS)
+    set(${ARG_LIBRARY_DIRS} "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE)
+  endif()
+  if(ARG_ENV)
+    set(${ARG_ENV} "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/rosidl_generator_py" PARENT_SCOPE)
+  endif()
+endfunction()

proto2ros/cmake/templates/mypy.ini.in

@@ -0,0 +1,10 @@
+# Generated from cmake/templates/mypy.ini.in in the proto2ros package.
+[mypy]
+ignore_missing_imports = true
+mypy_path = "@MYPY_PATH@"
+# Ignore missing type annotations, mainly coming from ROS 2 imports.
+disable_error_code = var-annotated
+
+[mypy-google.*]
+# Ignore google.* modules explicitly to avoid missing type hints errors.
+ignore_missing_imports = true

proto2ros/msg/Any.msg

@@ -0,0 +1,6 @@
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
+
+# A dynamically typed ROS 2 message.
+
+string  type_name  # ROS 2 message type name
+uint8[] value      # Serialized ROS 2 message instance

proto2ros/msg/AnyProto.msg

@@ -0,0 +1,7 @@
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
+
+# A dynamically typed Protobuf message. Equivalent to the google.protobuf.Any message.
+# See https://protobuf.dev/reference/protobuf/google.protobuf/#any for further reference.
+
+string  type_url    # Protobuf message type URL.
+uint8[] value       # Packed Protobuf message instance.