From 4ebafd88a253f2b7ec7bd5c3ae0c012aab4368e7 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Wed, 15 Jan 2025 15:15:13 +0100 Subject: [PATCH 01/17] Prepare nexus agent (python) skeleton --- .github/workflows/build-and-publish.yml | 4 +-- pytest.ini | 1 + requirements.txt | 4 +++ .../PackageReferencesController.cs | 2 +- src/agent/python/main.py | 27 +++++++++++++++++++ tests/agent/python-tests/agent-tests.py | 2 ++ 6 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 requirements.txt create mode 100644 src/agent/python/main.py create mode 100644 tests/agent/python-tests/agent-tests.py diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f4622e0..256c355 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -71,8 +71,8 @@ jobs: - name: Test run: | dotnet test -c Release - # pyright - # pytest + pyright + pytest - name: Docker Build run: | diff --git a/pytest.ini b/pytest.ini index a212f5e..e6396ca 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,4 +8,5 @@ python_functions=*_test pythonpath = src/remoting/python testpaths = + tests/agent/python-tests tests/remoting/python-tests \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..18bfb86 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +fastapi[standard] +nexus_extensibility==2.0.0-beta.38 +nexus_api==2.0.0-beta.38 +pytest-asyncio \ No newline at end of file diff --git a/src/agent/dotnet/Controllers/PackageReferencesController.cs b/src/agent/dotnet/Controllers/PackageReferencesController.cs index 1637619..bf2d1b7 100644 --- a/src/agent/dotnet/Controllers/PackageReferencesController.cs +++ b/src/agent/dotnet/Controllers/PackageReferencesController.cs @@ -43,7 +43,7 @@ public Task> GetAsync() /// The package reference to create. [HttpPost] public Task CreateAsync( - [FromBody] PackageReference packageReference) + PackageReference packageReference) { return _packageService.PutAsync(packageReference); } diff --git a/src/agent/python/main.py b/src/agent/python/main.py new file mode 100644 index 0000000..f6bf931 --- /dev/null +++ b/src/agent/python/main.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass +from uuid import UUID + +@dataclass +class PackageReference: + provider: str + configuration: dict[str, str] + +from fastapi import FastAPI + +app = FastAPI() + +@app.get("/api/v1/packagereferences", tags=["PackageReferences"], summary="Gets the list of package references.") +async def get() -> dict[UUID, PackageReference]: + return {} + +@app.post("/api/v1/packagereferences", tags=["PackageReferences"], summary="Creates a package reference.") +async def create(packageReference: PackageReference): + return 0 + +@app.delete("/api/v1/packagereferences/{id}", tags=["PackageReferences"], summary="Deletes a package reference.") +async def delete(id: UUID): + return 0 + +@app.get("/api/v1/packagereferences/{id}/versions", tags=["PackageReferences"], summary="Gets package versions.") +async def get_versions(id: UUID): + return 0 \ No newline at end of file diff --git a/tests/agent/python-tests/agent-tests.py b/tests/agent/python-tests/agent-tests.py new file mode 100644 index 0000000..886236c --- /dev/null +++ b/tests/agent/python-tests/agent-tests.py @@ -0,0 +1,2 @@ +def dummy_test(): + pass \ No newline at end of file From 69e04836f408a04bf0c939a19b2385ff1abf8e63 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Wed, 15 Jan 2025 21:36:13 +0100 Subject: [PATCH 02/17] Update remoting/python/nexus_remoting --- .github/workflows/build-and-publish.yml | 40 ++++---- requirements.txt | 4 +- src/remoting/dotnet/Remoting.cs | 10 +- .../python/nexus_remoting/_encoder.py | 8 +- .../python/nexus_remoting/_remoting.py | 98 +++++++++++-------- src/remoting/python/setup.py | 2 +- .../python/remote.py | 21 +--- 7 files changed, 93 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 256c355..22fa411 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -45,19 +45,19 @@ jobs: dotnet-version: "9.0.x" dotnet-quality: "preview" - # - name: Set up Python - # uses: actions/setup-python@v3 - # with: - # python-version: '3.9' + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.9' - name: Create Docker Output Folder run: mkdir --parent artifacts/images - # - name: Install - # run: | - # npm install -g pyright - # python -m pip install build wheel pytest pytest-asyncio - # python -m pip install --pre --index-url https://www.myget.org/F/apollo3zehn-dev/python/ nexus-extensibility + - name: Install + run: | + npm install -g pyright + python -m pip install build wheel pytest pytest-asyncio + python -m pip install --pre --index-url https://www.myget.org/F/apollo3zehn-dev/python/ nexus-extensibility - name: Docker Setup id: buildx @@ -116,10 +116,10 @@ jobs: env: MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} - # - name: Python package (MyGet) - # run: 'for filePath in artifacts/package/*.whl; do curl -k -X POST https://www.myget.org/F/apollo3zehn-dev/python/upload -H "Authorization: Bearer ${MYGET_API_KEY}" -F "data=@$filePath"; done' - # env: - # MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} + - name: Python package (MyGet) + run: 'for filePath in artifacts/package/*.whl; do curl -k -X POST https://www.myget.org/F/apollo3zehn-dev/python/upload -H "Authorization: Bearer ${MYGET_API_KEY}" -F "data=@$filePath"; done' + env: + MYGET_API_KEY: ${{ secrets.MYGET_API_KEY }} - name: Docker Login (Github Container Registry) uses: docker/login-action@v1 @@ -143,9 +143,9 @@ jobs: steps: - # - name: Install - # run: | - # python -m pip install twine + - name: Install + run: | + python -m pip install twine - name: Download Artifacts uses: actions/download-artifact@v3 @@ -166,10 +166,10 @@ jobs: env: NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} - # - name: Python Package (PyPI) - # run: twine upload artifacts/package/*.whl -u__token__ -p"${PYPI_API_KEY}" - # env: - # PYPI_API_KEY: ${{ secrets.PYPI_API_KEY }} + - name: Python Package (PyPI) + run: twine upload artifacts/package/*.whl -u__token__ -p"${PYPI_API_KEY}" + env: + PYPI_API_KEY: ${{ secrets.PYPI_API_KEY }} - name: Docker Login (Docker Hub) uses: docker/login-action@v1 diff --git a/requirements.txt b/requirements.txt index 18bfb86..0037f49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ fastapi[standard] -nexus_extensibility==2.0.0-beta.38 -nexus_api==2.0.0-beta.38 +nexus_extensibility==2.0.0-beta.41 +nexus_api==2.0.0-beta.41 pytest-asyncio \ No newline at end of file diff --git a/src/remoting/dotnet/Remoting.cs b/src/remoting/dotnet/Remoting.cs index 3bee3f8..b9e0cea 100644 --- a/src/remoting/dotnet/Remoting.cs +++ b/src/remoting/dotnet/Remoting.cs @@ -12,7 +12,11 @@ namespace Nexus.Remoting; -internal class Logger(NetworkStream commStream, Stopwatch watchdogTimer, CancellationToken cancellationToken) : ILogger +internal class Logger( + NetworkStream commStream, + Stopwatch watchdogTimer, + CancellationToken cancellationToken +) : ILogger { private readonly Stopwatch _watchdogTimer = watchdogTimer; @@ -74,8 +78,8 @@ public class RemoteCommunicator /// The network stream for data. /// A func to get a new data source instance by its type name. public RemoteCommunicator( - NetworkStream commStream, - NetworkStream dataStream, + NetworkStream commStream, + NetworkStream dataStream, Func getDataSource ) { diff --git a/src/remoting/python/nexus_remoting/_encoder.py b/src/remoting/python/nexus_remoting/_encoder.py index a297551..002f8e4 100644 --- a/src/remoting/python/nexus_remoting/_encoder.py +++ b/src/remoting/python/nexus_remoting/_encoder.py @@ -7,12 +7,10 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta from enum import Enum -from typing import (Any, Callable, ClassVar, Optional, Type, Union, +from typing import (Any, Callable, ClassVar, Optional, Type, TypeVar, Union, cast) from uuid import UUID -from typing import TypeVar - T = TypeVar("T") @dataclass(frozen=True) @@ -102,7 +100,7 @@ def _decode(typeCls: Type[T], data: Any, options: JsonEncoderOptions) -> T: return cast(T, instance3) # list - elif issubclass(origin, list): + elif issubclass(cast(type, origin), list): listType = args[0] instance1: list = list() @@ -113,7 +111,7 @@ def _decode(typeCls: Type[T], data: Any, options: JsonEncoderOptions) -> T: return cast(T, instance1) # dict - elif issubclass(origin, dict): + elif issubclass(cast(type, origin), dict): # keyType = args[0] valueType = args[1] diff --git a/src/remoting/python/nexus_remoting/_remoting.py b/src/remoting/python/nexus_remoting/_remoting.py index 87a00b3..fabfff1 100644 --- a/src/remoting/python/nexus_remoting/_remoting.py +++ b/src/remoting/python/nexus_remoting/_remoting.py @@ -1,14 +1,15 @@ import json import socket import struct -from datetime import datetime +import time +from datetime import datetime, timedelta from threading import Lock -from typing import Any, Dict, Optional, Tuple, cast +from typing import Any, Callable, Dict, Optional, Tuple, cast from urllib.parse import urlparse from nexus_extensibility import (CatalogItem, DataSourceContext, ExtensibilityUtilities, IDataSource, ILogger, - LogLevel, ReadRequest) + LogLevel, ReadRequest, ResourceCatalog) from ._encoder import (JsonEncoder, JsonEncoderOptions, to_camel_case, to_snake_case) @@ -22,10 +23,10 @@ class _Logger(ILogger): - _tcp_comm_socket: socket.socket + _comm_socket: socket.socket def __init__(self, tcp_comm_socket: socket.socket): - self._tcp_comm_socket = tcp_comm_socket + self._comm_socket = tcp_comm_socket def log(self, log_level: LogLevel, message: str): @@ -35,54 +36,53 @@ def log(self, log_level: LogLevel, message: str): "params": [log_level.name, message] } - _send_to_server(notification, self._tcp_comm_socket) + _send_to_server(notification, self._comm_socket) class RemoteCommunicator: """A remote communicator.""" + _watchdog_timer = time.time() _logger: ILogger - - def __init__(self, data_source: IDataSource, address: str, port: int): + _data_source: IDataSource + + def __init__( + self, + comm_socket: socket.socket, + data_socket: socket.socket, + get_data_source: Callable[[str], IDataSource] + ): """ Initializes a new instance of the RemoteCommunicator. Args: - data_source: The data source. - address: The address to connect to. - port: The port to connect to. + comm_stream: The network stream for communications. + data_stream: The network stream for data. + get_data_source: A func to get a new data source instance by its type name. """ - self._address: str = address - self._port: int = port - self._tcp_comm_socket: socket.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._tcp_data_socket: socket.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._comm_socket = comm_socket + self._data_socket = data_socket - if not (0 < port and port < 65536): - raise Exception(f"The port {port} is not a valid port number.") + self._get_data_source = get_data_source - self._data_source: IDataSource = data_source + @property + def last_communication(self) -> timedelta: + end = time.time() + return timedelta(seconds=end - self._watchdog_timer) async def run(self): """ Starts the remoting operation. """ - # comm connection - self._tcp_comm_socket.connect((self._address, self._port)) - self._tcp_comm_socket.sendall("comm".encode()) - - # data connection - self._tcp_data_socket.connect((self._address, self._port)) - self._tcp_data_socket.sendall("data".encode()) - # loop while (True): # https://www.jsonrpc.org/specification # get request message - size = self._read_size(self._tcp_comm_socket) - json_request = self._tcp_comm_socket.recv(size, socket.MSG_WAITALL) + size = self._read_size(self._comm_socket) + json_request = self._comm_socket.recv(size, socket.MSG_WAITALL) if len(json_request) == 0: _shutdown() @@ -99,6 +99,7 @@ async def run(self): if "id" in request: try: + (result, data, status) = await self._process_invocation(request) response = { @@ -124,12 +125,12 @@ async def run(self): response["id"] = request["id"] # send response - _send_to_server(response, self._tcp_comm_socket) + _send_to_server(response, self._comm_socket) # send data if data is not None and status is not None: - self._tcp_data_socket.sendall(data) - self._tcp_data_socket.sendall(status) + self._data_socket.sendall(data) + self._data_socket.sendall(status) async def _process_invocation(self, request: dict[str, Any]) \ -> Tuple[Optional[Dict[str, Any]], Optional[memoryview], Optional[memoryview]]: @@ -155,6 +156,8 @@ async def _process_invocation(self, request: dict[str, Any]) \ resource_locator_string = cast(str, raw_context["resourceLocator"]) if "resourceLocator" in raw_context else None resource_locator = None if resource_locator_string is None else urlparse(resource_locator_string) + self._data_source = self._get_data_source(type) + system_configuration = raw_context["systemConfiguration"] \ if "systemConfiguration" in raw_context else None @@ -164,7 +167,7 @@ async def _process_invocation(self, request: dict[str, Any]) \ request_configuration = raw_context["requestConfiguration"] \ if "requestConfiguration" in raw_context else None - self._logger = _Logger(self._tcp_comm_socket) + self._logger = _Logger(self._comm_socket) context = DataSourceContext( resource_locator, @@ -176,6 +179,9 @@ async def _process_invocation(self, request: dict[str, Any]) \ elif method_name == "getCatalogRegistrations": + if self._data_source is None: + raise Exception("The data source context must be set before invoking other methods.") + path = cast(str, params[0]) registrations = await self._data_source.get_catalog_registrations(path) @@ -183,10 +189,13 @@ async def _process_invocation(self, request: dict[str, Any]) \ "registrations": registrations } - elif method_name == "getCatalog": + elif method_name == "enrichCatalog": - catalog_id = params[0] - catalog = await self._data_source.get_catalog(catalog_id) + if self._data_source is None: + raise Exception("The data source context must be set before invoking other methods.") + + original_catalog = JsonEncoder().decode(ResourceCatalog, params[0]) + catalog = await self._data_source.enrich_catalog(original_catalog) result = { "catalog": catalog @@ -194,6 +203,9 @@ async def _process_invocation(self, request: dict[str, Any]) \ elif method_name == "getTimeRange": + if self._data_source is None: + raise Exception("The data source context must be set before invoking other methods.") + catalog_id = params[0] (begin, end) = await self._data_source.get_time_range(catalog_id) @@ -204,6 +216,9 @@ async def _process_invocation(self, request: dict[str, Any]) \ elif method_name == "getAvailability": + if self._data_source is None: + raise Exception("The data source context must be set before invoking other methods.") + catalog_id = params[0] begin = datetime.strptime(params[1], "%Y-%m-%dT%H:%M:%SZ") end = datetime.strptime(params[2], "%Y-%m-%dT%H:%M:%SZ") @@ -215,6 +230,9 @@ async def _process_invocation(self, request: dict[str, Any]) \ elif method_name == "readSingle": + if self._data_source is None: + raise Exception("The data source context must be set before invoking other methods.") + begin = datetime.strptime(params[0], "%Y-%m-%dT%H:%M:%SZ") end = datetime.strptime(params[1], "%Y-%m-%dT%H:%M:%SZ") original_resource_name = params[2] @@ -258,15 +276,17 @@ async def _handle_read_data(self, resource_path: str, begin: datetime, end: date "params": [resource_path, begin, end] } - _send_to_server(read_data_request, self._tcp_comm_socket) + _send_to_server(read_data_request, self._comm_socket) - size = self._read_size(self._tcp_data_socket) - data = self._tcp_data_socket.recv(size, socket.MSG_WAITALL) + size = self._read_size(self._data_socket) + data = self._data_socket.recv(size, socket.MSG_WAITALL) if len(data) == 0: _shutdown() - return memoryview(data).cast("d") + # 'cast' is required because of https://github.com/python/cpython/issues/126012 + # see also https://github.com/nexus-main/nexus/issues/184 + return cast(memoryview, memoryview(data).cast("d")) def _handle_report_progress(self, progress_value: float): pass # not implemented diff --git a/src/remoting/python/setup.py b/src/remoting/python/setup.py index ec65ed0..74c406f 100644 --- a/src/remoting/python/setup.py +++ b/src/remoting/python/setup.py @@ -47,6 +47,6 @@ }, python_requires=">=3.9", install_requires=[ - "nexus-extensibility>=2.0.0b24" + "nexus-extensibility>=2.0.0b41" ] ) diff --git a/tests/Nexus.Sources.Remote.Tests/python/remote.py b/tests/Nexus.Sources.Remote.Tests/python/remote.py index a14e751..1b3d1bc 100644 --- a/tests/Nexus.Sources.Remote.Tests/python/remote.py +++ b/tests/Nexus.Sources.Remote.Tests/python/remote.py @@ -1,7 +1,5 @@ -import asyncio import glob import os -import sys from datetime import datetime, timedelta, timezone from typing import Callable from urllib.request import url2pathname @@ -10,7 +8,6 @@ IDataSource, LogLevel, NexusDataType, ReadDataHandler, ReadRequest, Representation, ResourceBuilder, ResourceCatalogBuilder) -from nexus_remoting import RemoteCommunicator class PythonDataSource(IDataSource): @@ -206,20 +203,4 @@ async def _read_and_modify_nexus_data( double_data[i] = data_from_nexus[i] * 2 for i in range(0, len(request.status)): - request.status[i] = 1 - -# args -if len(sys.argv) < 3: - raise Exception("No argument for address and/or port was specified.") - -# get address -address = sys.argv[1] - -# get port -try: - port = int(sys.argv[2]) -except Exception as ex: - raise Exception(f"The second command line argument must be a valid port number. Inner error: {str(ex)}") - -# run -asyncio.run(RemoteCommunicator(PythonDataSource(), address, port).run()) + request.status[i] = 1 \ No newline at end of file From d256cfc52f8b6742fa9bd5ffcee67fc69ac968f3 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Wed, 15 Jan 2025 22:46:00 +0100 Subject: [PATCH 03/17] Prepare implementation of Nexus.Agent (python) --- .github/workflows/build-and-publish.yml | 5 +- .../config/packages.json | 0 .nexus-agent-python/config/packages.json | 10 + .vscode/launch.json | 4 +- requirements.txt | 6 +- src/Nexus.Sources.Remote/Remote.cs | 3 + src/agent/dotnet/Core/AgentService.cs | 19 +- src/agent/dotnet/Core/Options.cs | 13 +- src/agent/dotnet/Program.cs | 1 + src/agent/dotnet/appsettings.json | 3 + src/agent/python/main.py | 4 + .../Nexus.Sources.Remote.Tests/RemoteTests.cs | 52 +++-- .../RemoteTestsFixture.cs | 206 +++++++++++------- 13 files changed, 219 insertions(+), 107 deletions(-) rename {.nexus-agent => .nexus-agent-dotnet}/config/packages.json (100%) create mode 100644 .nexus-agent-python/config/packages.json diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 22fa411..f30b3cc 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -56,8 +56,7 @@ jobs: - name: Install run: | npm install -g pyright - python -m pip install build wheel pytest pytest-asyncio - python -m pip install --pre --index-url https://www.myget.org/F/apollo3zehn-dev/python/ nexus-extensibility + python -m pip install -r requirements.txt - name: Docker Setup id: buildx @@ -66,7 +65,7 @@ jobs: - name: Build run: | dotnet publish -c Release -o app /p:GeneratePackage=true src/agent/dotnet/agent.csproj - # python -m build --wheel --outdir artifacts/package --no-isolation src/remoting/python + python -m build --wheel --outdir artifacts/package --no-isolation src/remoting/python - name: Test run: | diff --git a/.nexus-agent/config/packages.json b/.nexus-agent-dotnet/config/packages.json similarity index 100% rename from .nexus-agent/config/packages.json rename to .nexus-agent-dotnet/config/packages.json diff --git a/.nexus-agent-python/config/packages.json b/.nexus-agent-python/config/packages.json new file mode 100644 index 0000000..3967718 --- /dev/null +++ b/.nexus-agent-python/config/packages.json @@ -0,0 +1,10 @@ +{ + "c05b592f-e198-472d-9902-3f60cf0a6332": { + "Provider": "local", + "Configuration": { + "path": "../../../tests/Nexus.Sources.Remote.Tests/python", + "version": "v1", + "csproj": "remote.py" + } + } +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 2f093cd..c12c26e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,7 +2,7 @@ "version": "0.2.0", "configurations": [ { - "name": "Launch Nexus.Agent", + "name": "Launch Nexus.Agent (dotnet)", "type": "coreclr", "request": "launch", "preLaunchTask": "build-nexus-agent", @@ -12,7 +12,7 @@ "stopAtEntry": false, "env": { "ASPNETCORE_ENVIRONMENT": "Development", - "NEXUSAGENT_Paths__Config": "../../.nexus-agent/config" + "NEXUSAGENT_Paths__Config": "../../.nexus-agent-dotnet/config" } } ] diff --git a/requirements.txt b/requirements.txt index 0037f49..afce96c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ +build fastapi[standard] -nexus_extensibility==2.0.0-beta.41 nexus_api==2.0.0-beta.41 -pytest-asyncio \ No newline at end of file +nexus_extensibility==2.0.0-beta.41 +pytest-asyncio +wheel \ No newline at end of file diff --git a/src/Nexus.Sources.Remote/Remote.cs b/src/Nexus.Sources.Remote/Remote.cs index 527678f..da33b7e 100644 --- a/src/Nexus.Sources.Remote/Remote.cs +++ b/src/Nexus.Sources.Remote/Remote.cs @@ -18,8 +18,11 @@ public partial class Remote : IDataSource, IDisposable private const int DEFAULT_AGENT_PORT = 56145; private ReadDataHandler? _readData; + private static readonly int API_LEVEL = 1; + private RemoteCommunicator _communicator = default!; + private IJsonRpcServer _rpcServer = default!; /* Possible features to be implemented for this data source: diff --git a/src/agent/dotnet/Core/AgentService.cs b/src/agent/dotnet/Core/AgentService.cs index f579f6c..97bda50 100644 --- a/src/agent/dotnet/Core/AgentService.cs +++ b/src/agent/dotnet/Core/AgentService.cs @@ -2,6 +2,8 @@ using System.Net; using System.Net.Sockets; using System.Text; +using Microsoft.Extensions.Options; +using Nexus.Core; using Nexus.Extensibility; using Nexus.PackageManagement.Services; using Nexus.Remoting; @@ -33,14 +35,18 @@ internal class AgentService private readonly ILogger _agentLogger; + private readonly SystemOptions _systemOptions; + public AgentService( IExtensionHive extensionHive, IPackageService packageService, - ILogger agentLogger) + ILogger agentLogger, + IOptions systemOptions) { _extensionHive = extensionHive; _packageService = packageService; _agentLogger = agentLogger; + _systemOptions = systemOptions.Value; } public async Task LoadPackagesAsync(CancellationToken cancellationToken) @@ -59,7 +65,16 @@ await _extensionHive.LoadPackagesAsync( public Task AcceptClientsAsync(CancellationToken cancellationToken) { - var tcpListener = new TcpListener(IPAddress.Any, 56145); + _agentLogger.LogInformation( + "Listening for JSON-RPC communication on {JsonRpcListenAddress}:{JsonRpcListenPort}", + _systemOptions.JsonRpcListenAddress, _systemOptions.JsonRpcListenPort + ); + + var tcpListener = new TcpListener( + IPAddress.Parse(_systemOptions.JsonRpcListenAddress), + _systemOptions.JsonRpcListenPort + ); + tcpListener.Start(); // Detect and remove inactivate clients diff --git a/src/agent/dotnet/Core/Options.cs b/src/agent/dotnet/Core/Options.cs index e75b624..f90960a 100644 --- a/src/agent/dotnet/Core/Options.cs +++ b/src/agent/dotnet/Core/Options.cs @@ -26,6 +26,15 @@ internal static IConfiguration BuildConfiguration() } } +internal record SystemOptions : NexusAgentOptions +{ + public const string Section = "System"; + + public string JsonRpcListenAddress { get; set; } = "0.0.0.0"; + + public int JsonRpcListenPort { get; set; } = 56145; +} + internal record PathsOptions : IPackageManagementPathsOptions { public const string Section = "Paths"; @@ -34,11 +43,7 @@ internal record PathsOptions : IPackageManagementPathsOptions public string Packages { get; set; } = Path.Combine(PlatformSpecificRoot, "packages"); - #region Support - private static string PlatformSpecificRoot { get; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "nexus-agent") : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".local", "share", "nexus-agent"); - - #endregion } \ No newline at end of file diff --git a/src/agent/dotnet/Program.cs b/src/agent/dotnet/Program.cs index f658f9a..f7ca826 100644 --- a/src/agent/dotnet/Program.cs +++ b/src/agent/dotnet/Program.cs @@ -36,6 +36,7 @@ builder.Services .AddSingleton(); +builder.Services.Configure(configuration.GetSection(SystemOptions.Section)); builder.Services.Configure(configuration.GetSection(PathsOptions.Section)); // Package management diff --git a/src/agent/dotnet/appsettings.json b/src/agent/dotnet/appsettings.json index 47820ef..a051b23 100644 --- a/src/agent/dotnet/appsettings.json +++ b/src/agent/dotnet/appsettings.json @@ -7,6 +7,9 @@ } }, "AllowedHosts": "*", + "System": { + "BlindSample": "System" + }, "Paths": { "BlindSample": "Paths" } diff --git a/src/agent/python/main.py b/src/agent/python/main.py index f6bf931..fe5e4e7 100644 --- a/src/agent/python/main.py +++ b/src/agent/python/main.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +import os from uuid import UUID @dataclass @@ -8,6 +9,9 @@ class PackageReference: from fastapi import FastAPI +json_rpc_listen_address = os.getenv("NEXUSAGENT_System__JsonRpcListenAddress", default="0.0.0.0") +json_rpc_listen_port = os.getenv("NEXUSAGENT_System__JsonRpcListenPort", default=56145) + app = FastAPI() @app.get("/api/v1/packagereferences", tags=["PackageReferences"], summary="Gets the list of package references.") diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index 59122a3..09c25c0 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -15,14 +15,16 @@ public class RemoteTests(RemoteTestsFixture fixture) { private readonly RemoteTestsFixture _fixture = fixture; - [Fact] - public async Task ProvidesCatalog() + [Theory] + [InlineData(60000 /* dotnet */)] + [InlineData(60001 /* python */)] + public async Task ProvidesCatalog(int port) { await _fixture.Initialize; // Arrange var dataSource = new Remote() as IDataSource; - var context = CreateContext(); + var context = CreateContext(port); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -49,13 +51,15 @@ public async Task ProvidesCatalog() Assert.True(expectedDataTypes.SequenceEqual(actualDataTypes)); } -[Fact] - public async Task CanProvideTimeRange() + [Theory] + [InlineData(60000 /* dotnet */)] + [InlineData(60001 /* python */)] + public async Task CanProvideTimeRange(int port) { await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(); + var context = CreateContext(port); var expectedBegin = new DateTime(2019, 12, 31, 12, 00, 00, DateTimeKind.Utc); var expectedEnd = new DateTime(2020, 01, 02, 09, 50, 00, DateTimeKind.Utc); @@ -68,13 +72,15 @@ public async Task CanProvideTimeRange() Assert.Equal(expectedEnd, end); } -[Fact] - public async Task CanProvideAvailability() + [Theory] + [InlineData(60000 /* dotnet */)] + [InlineData(60001 /* python */)] + public async Task CanProvideAvailability(int port) { await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(); + var context = CreateContext(port); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -85,8 +91,10 @@ public async Task CanProvideAvailability() Assert.Equal(2 / 144.0, actual, precision: 4); } -[Fact] - public async Task CanReadFullDay() + [Theory] + [InlineData(60000 /* dotnet */)] + [InlineData(60001 /* python */)] + public async Task CanReadFullDay(int port) { // TODO fix this var complexData = true; @@ -94,7 +102,7 @@ public async Task CanReadFullDay() await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(); + var context = CreateContext(port); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -149,14 +157,16 @@ void GenerateData(DateTimeOffset dateTime) Assert.True(expectedStatus.SequenceEqual(status.ToArray())); } -[Fact] - public async Task CanLog() + [Theory] + [InlineData(60000 /* dotnet */)] + [InlineData(60001 /* python */)] + public async Task CanLog(int port) { await _fixture.Initialize; var loggerMock = new Mock(); var dataSource = new Remote() as IDataSource; - var context = CreateContext(); + var context = CreateContext(port); await dataSource.SetContextAsync(context, loggerMock.Object, CancellationToken.None); @@ -172,13 +182,15 @@ public async Task CanLog() ); } -[Fact] - public async Task CanReadDataHandler() + [Theory] + [InlineData(60000 /* dotnet */)] + [InlineData(60001 /* python */)] + public async Task CanReadDataHandler(int port) { await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(); + var context = CreateContext(port); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -228,10 +240,10 @@ Task HandleReadDataAsync(string resourcePath, DateTime begin, DateTime end, Memo Assert.True(expectedStatus.SequenceEqual(status.ToArray())); } - private static DataSourceContext CreateContext() + private static DataSourceContext CreateContext(int port) { return new DataSourceContext( - ResourceLocator: new Uri("tcp://127.0.0.1:56145"), + ResourceLocator: new Uri($"tcp://127.0.0.1:{port}"), SystemConfiguration: new Dictionary() { [typeof(Remote).FullName!] = JsonSerializer.SerializeToElement(new JsonObject() diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs index 3919b7a..57a3fb5 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -16,101 +16,159 @@ public class RemoteTestsFixture : IDisposable public RemoteTestsFixture() { - Initialize = Task.Run(async () => + Initialize = Task.Run(() => { - /* Why not `dotnet run`? Because it spawns a child process for which - * we do not know the process ID and so we cannot kill it. - */ + var dotnetTask = RunDotnetAgent(); + var pythonTask = RunPythonAgent(); - // Build Nexus.Agent - var psi_build = new ProcessStartInfo("bash") - { - /* Why `sleep infinity`? Because the test debugger seems to stop whenever a child process stops */ - Arguments = "-c \"dotnet build ../../../../src/agent/dotnet/agent.csproj && sleep infinity\"", - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - _buildProcess = new Process - { - StartInfo = psi_build, - EnableRaisingEvents = true - }; + return Task.WhenAll(dotnetTask, pythonTask); + }); + } - _buildProcess.OutputDataReceived += (sender, e) => - { - if (e.Data is not null && e.Data.Contains("Build succeeded")) - { - _success = true; - _semaphoreBuild.Release(); - } - }; - - _buildProcess.ErrorDataReceived += (sender, e) => + public Task Initialize { get; } + + private async Task RunDotnetAgent() + { + /* Why not `dotnet run`? Because it spawns a child process for which + * we do not know the process ID and so we cannot kill it. + */ + + // Build Nexus.Agent + var psi_build = new ProcessStartInfo("bash") + { + /* Why `sleep infinity`? Because the test debugger seems to stop whenever a child process stops */ + Arguments = "-c \"dotnet build ../../../../src/agent/dotnet/agent.csproj && sleep infinity\"", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + _buildProcess = new Process + { + StartInfo = psi_build, + EnableRaisingEvents = true + }; + + _buildProcess.OutputDataReceived += (sender, e) => + { + if (e.Data is not null && e.Data.Contains("Build succeeded")) { - _success = false; + _success = true; _semaphoreBuild.Release(); - }; + } + }; - _buildProcess.Start(); - _buildProcess.BeginOutputReadLine(); - _buildProcess.BeginErrorReadLine(); + _buildProcess.ErrorDataReceived += (sender, e) => + { + _success = false; + _semaphoreBuild.Release(); + }; - await _semaphoreBuild.WaitAsync(TimeSpan.FromMinutes(1)); + _buildProcess.Start(); + _buildProcess.BeginOutputReadLine(); + _buildProcess.BeginErrorReadLine(); - if (!_success) - throw new Exception("Unable to build Nexus.Agent."); + await _semaphoreBuild.WaitAsync(TimeSpan.FromMinutes(1)); - // Run Nexus.Agent - var psi_run = new ProcessStartInfo("dotnet") - { - Arguments = $"../../../artifacts/bin/agent/debug/Nexus.Agent.dll", - WorkingDirectory="../../../../src/agent/dotnet", - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true - }; + if (!_success) + throw new Exception("Unable to build Nexus.Agent."); - psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../../.nexus-agent/config"; + // Run Nexus.Agent + var psi_run = new ProcessStartInfo("dotnet") + { + Arguments = $"../../../artifacts/bin/agent/debug/Nexus.Agent.dll", + WorkingDirectory="../../../../src/agent/dotnet", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; - _runProcess = new Process - { - StartInfo = psi_run, - EnableRaisingEvents = true - }; + psi_run.Environment["NEXUSAGENT_System__JsonRpcListenPort"] = "60000"; + psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../../.nexus-agent-dotnet/config"; - _runProcess.OutputDataReceived += (sender, e) => - { - File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + _runProcess = new Process + { + StartInfo = psi_run, + EnableRaisingEvents = true + }; - if (e.Data is not null && e.Data.Contains("Now listening on")) - { - _success = true; - _semaphoreRun.Release(); - } - }; + _runProcess.OutputDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); - _runProcess.ErrorDataReceived += (sender, e) => + if (e.Data is not null && e.Data.Contains("Now listening on")) { - File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); - - _success = false; + _success = true; _semaphoreRun.Release(); - }; + } + }; - _runProcess.Start(); - _runProcess.BeginOutputReadLine(); - _runProcess.BeginErrorReadLine(); + _runProcess.ErrorDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); - await _semaphoreRun.WaitAsync(TimeSpan.FromMinutes(1)); + _success = false; + _semaphoreRun.Release(); + }; - if (!_success) - throw new Exception("Unable to launch Nexus.Agent."); - }); + _runProcess.Start(); + _runProcess.BeginOutputReadLine(); + _runProcess.BeginErrorReadLine(); + + await _semaphoreRun.WaitAsync(TimeSpan.FromMinutes(1)); + + if (!_success) + throw new Exception("Unable to launch Nexus.Agent (dotnet)."); } - public Task Initialize { get; } + private async Task RunPythonAgent() + { + var psi_run = new ProcessStartInfo("fastapi") + { + Arguments = $"run main.py", + WorkingDirectory="../../../../src/agent/python", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + psi_run.Environment["NEXUSAGENT_System__JsonRpcListenPort"] = "60001"; + psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../../.nexus-agent-python/config"; + + _runProcess = new Process + { + StartInfo = psi_run, + EnableRaisingEvents = true + }; + + _runProcess.OutputDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + + if (e.Data is not null && e.Data.Contains("Application startup complete.")) + { + _success = true; + _semaphoreRun.Release(); + } + }; + + _runProcess.ErrorDataReceived += (sender, e) => + { + // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + + _success = false; + _semaphoreRun.Release(); + }; + + _runProcess.Start(); + _runProcess.BeginOutputReadLine(); + _runProcess.BeginErrorReadLine(); + + await _semaphoreRun.WaitAsync(TimeSpan.FromMinutes(1)); + + if (!_success) + throw new Exception("Unable to launch Nexus.Agent (python)."); + } public void Dispose() { From 24547912ac165e842f6cae2a969d6a3332fbd196 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Thu, 16 Jan 2025 09:35:57 +0100 Subject: [PATCH 04/17] Follow Nexus changes --- .nexus-agent-dotnet/config/packages.json | 2 +- .vscode/launch.json | 2 +- requirements.txt | 4 ++-- src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj | 2 +- src/agent/dotnet/Program.cs | 5 ++++- src/agent/dotnet/agent.csproj | 3 ++- src/remoting/dotnet/remoting.csproj | 2 +- src/remoting/python/setup.py | 2 +- .../Nexus.Sources.Remote.Tests.csproj | 6 +++--- tests/Nexus.Sources.Remote.Tests/RemoteTests.cs | 2 +- .../Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs | 12 +++++++++--- .../dotnet/v1/{remote.cs => test.cs} | 2 +- .../dotnet/v1/{remote.csproj => test.csproj} | 2 +- .../python/{remote.py => test.py} | 2 +- 14 files changed, 29 insertions(+), 19 deletions(-) rename tests/Nexus.Sources.Remote.Tests/dotnet/v1/{remote.cs => test.cs} (99%) rename tests/Nexus.Sources.Remote.Tests/dotnet/v1/{remote.csproj => test.csproj} (96%) rename tests/Nexus.Sources.Remote.Tests/python/{remote.py => test.py} (99%) diff --git a/.nexus-agent-dotnet/config/packages.json b/.nexus-agent-dotnet/config/packages.json index 9fba7c7..25a9090 100644 --- a/.nexus-agent-dotnet/config/packages.json +++ b/.nexus-agent-dotnet/config/packages.json @@ -4,7 +4,7 @@ "Configuration": { "path": "../../../tests/Nexus.Sources.Remote.Tests/dotnet", "version": "v1", - "csproj": "remote.csproj" + "csproj": "test.csproj" } } } \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index c12c26e..11b6466 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,7 @@ "stopAtEntry": false, "env": { "ASPNETCORE_ENVIRONMENT": "Development", - "NEXUSAGENT_Paths__Config": "../../.nexus-agent-dotnet/config" + "NEXUSAGENT_Paths__Config": "../../../.nexus-agent-dotnet/config" } } ] diff --git a/requirements.txt b/requirements.txt index afce96c..0608a9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ build fastapi[standard] -nexus_api==2.0.0-beta.41 -nexus_extensibility==2.0.0-beta.41 +nexus_api==2.0.0-beta.42 +nexus_extensibility==2.0.0-beta.42 pytest-asyncio wheel \ No newline at end of file diff --git a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj index 3387817..d5df479 100644 --- a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj +++ b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj @@ -11,7 +11,7 @@ - + runtime;native diff --git a/src/agent/dotnet/Program.cs b/src/agent/dotnet/Program.cs index f7ca826..6db2622 100644 --- a/src/agent/dotnet/Program.cs +++ b/src/agent/dotnet/Program.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Options; using Nexus.Agent; using Nexus.Core; +using Nexus.Extensibility; using Nexus.PackageManagement.Core; using Scalar.AspNetCore; @@ -40,13 +41,15 @@ builder.Services.Configure(configuration.GetSection(PathsOptions.Section)); // Package management -builder.Services.AddPackageManagement(); +builder.Services.AddPackageManagement(); builder.Services.AddSingleton( serviceProvider => serviceProvider.GetRequiredService>().Value); var app = builder.Build(); +var b = app.Services.GetRequiredService>().Value; + app.MapGet("/", () => Results.Redirect("/scalar/v1")).ExcludeFromDescription(); app.MapOpenApi(); app.MapScalarApiReference(); diff --git a/src/agent/dotnet/agent.csproj b/src/agent/dotnet/agent.csproj index b4e714a..72e25d1 100644 --- a/src/agent/dotnet/agent.csproj +++ b/src/agent/dotnet/agent.csproj @@ -9,7 +9,8 @@ - + + diff --git a/src/remoting/dotnet/remoting.csproj b/src/remoting/dotnet/remoting.csproj index 2dc0726..fa98095 100644 --- a/src/remoting/dotnet/remoting.csproj +++ b/src/remoting/dotnet/remoting.csproj @@ -26,7 +26,7 @@ - + diff --git a/src/remoting/python/setup.py b/src/remoting/python/setup.py index 74c406f..96906e0 100644 --- a/src/remoting/python/setup.py +++ b/src/remoting/python/setup.py @@ -47,6 +47,6 @@ }, python_requires=">=3.9", install_requires=[ - "nexus-extensibility>=2.0.0b41" + "nexus-extensibility>=2.0.0b42" ] ) diff --git a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj index 50c6d37..bd038f0 100644 --- a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj +++ b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj @@ -11,7 +11,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -24,11 +24,11 @@ - + - + Always diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index 09c25c0..46ed92d 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -256,7 +256,7 @@ private static DataSourceContext CreateContext(int port) }, SourceConfiguration: new Dictionary() { - ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.DotnetDataSource"), + ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.Test"), ["resourceLocator"] = JsonSerializer.SerializeToElement("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")) }, RequestConfiguration: default diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs index 57a3fb5..628eb29 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -19,7 +19,7 @@ public RemoteTestsFixture() Initialize = Task.Run(() => { var dotnetTask = RunDotnetAgent(); - var pythonTask = RunPythonAgent(); + var pythonTask = Task.CompletedTask; // RunPythonAgent(); return Task.WhenAll(dotnetTask, pythonTask); }); @@ -107,8 +107,11 @@ private async Task RunDotnetAgent() { // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + var oldSuccess = _success; _success = false; - _semaphoreRun.Release(); + + if (oldSuccess) + _semaphoreRun.Release(); }; _runProcess.Start(); @@ -156,8 +159,11 @@ private async Task RunPythonAgent() { // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + var oldSuccess = _success; _success = false; - _semaphoreRun.Release(); + + if (oldSuccess) + _semaphoreRun.Release(); }; _runProcess.Start(); diff --git a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.cs b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.cs similarity index 99% rename from tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.cs rename to tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.cs index 10d6868..b660917 100644 --- a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.cs +++ b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.cs @@ -11,7 +11,7 @@ namespace Nexus.Sources; * but collides with ReadAndModifyNexusData method */ -public class DotnetDataSource : IDataSource +public class Test : IDataSource { private DataSourceContext _context = default!; diff --git a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj similarity index 96% rename from tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj rename to tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj index 96a84c4..3213d8a 100644 --- a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/remote.csproj +++ b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj @@ -5,7 +5,7 @@ - + runtime;native diff --git a/tests/Nexus.Sources.Remote.Tests/python/remote.py b/tests/Nexus.Sources.Remote.Tests/python/test.py similarity index 99% rename from tests/Nexus.Sources.Remote.Tests/python/remote.py rename to tests/Nexus.Sources.Remote.Tests/python/test.py index 1b3d1bc..6aff78a 100644 --- a/tests/Nexus.Sources.Remote.Tests/python/remote.py +++ b/tests/Nexus.Sources.Remote.Tests/python/test.py @@ -10,7 +10,7 @@ ResourceBuilder, ResourceCatalogBuilder) -class PythonDataSource(IDataSource): +class Test(IDataSource): _root: str From cb64915a16d67e7c57789dc119628670784fef5e Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Thu, 16 Jan 2025 14:59:59 +0100 Subject: [PATCH 05/17] Switch to central package management --- Directory.Build.props | 1 + Directory.Packages.props | 20 +++++++++++++++++++ .../Nexus.Benchmarks/Nexus.Benchmarks.csproj | 2 +- .../Nexus.Sources.Remote.csproj | 4 ++-- src/agent/dotnet/agent.csproj | 14 ++++++------- src/remoting/dotnet/remoting.csproj | 2 +- tests/Directory.Build.props | 2 +- .../Nexus.Sources.Remote.Tests.csproj | 10 +++++----- .../dotnet/v1/test.csproj | 2 +- 9 files changed, 39 insertions(+), 18 deletions(-) create mode 100644 Directory.Packages.props diff --git a/Directory.Build.props b/Directory.Build.props index eeb7e32..219e9e5 100755 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,6 +12,7 @@ true $(MSBuildThisFileDirectory)artifacts + NU1507 \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 0000000..3c098ab --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,20 @@ + + + true + + + + + + + + + + + + + + + + + diff --git a/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj b/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj index a34fbf9..f818e33 100644 --- a/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj +++ b/benchmarks/Nexus.Benchmarks/Nexus.Benchmarks.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj index d5df479..601af96 100644 --- a/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj +++ b/src/Nexus.Sources.Remote/Nexus.Sources.Remote.csproj @@ -11,11 +11,11 @@ - + runtime;native - + diff --git a/src/agent/dotnet/agent.csproj b/src/agent/dotnet/agent.csproj index 72e25d1..acc09f7 100644 --- a/src/agent/dotnet/agent.csproj +++ b/src/agent/dotnet/agent.csproj @@ -7,13 +7,13 @@ - - - - - - - + + + + + + + diff --git a/src/remoting/dotnet/remoting.csproj b/src/remoting/dotnet/remoting.csproj index fa98095..56ce09a 100644 --- a/src/remoting/dotnet/remoting.csproj +++ b/src/remoting/dotnet/remoting.csproj @@ -26,7 +26,7 @@ - + diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 3e6025f..a372527 100755 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -3,7 +3,7 @@ - VSTHRD200 + $(NoWarn);VSTHRD200 \ No newline at end of file diff --git a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj index bd038f0..a9b0540 100644 --- a/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj +++ b/tests/Nexus.Sources.Remote.Tests/Nexus.Sources.Remote.Tests.csproj @@ -9,11 +9,11 @@ - - - - - + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj index 3213d8a..8ddaa9b 100644 --- a/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj +++ b/tests/Nexus.Sources.Remote.Tests/dotnet/v1/test.csproj @@ -5,7 +5,7 @@ - + runtime;native From 8ec13f42eab59f923be16f98f8af41850e17c9e8 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Thu, 16 Jan 2025 21:45:40 +0100 Subject: [PATCH 06/17] REST api nearly works --- .nexus-agent-python/config/packages.json | 8 +- .vscode/launch.json | 14 ++- src/agent/python/__init__.py | 0 src/agent/python/main.py | 31 +----- src/agent/python/options.py | 16 +++ src/agent/python/routers/__init__.py | 0 .../python/routers/package_references.py | 41 ++++++++ src/agent/python/services.py | 99 +++++++++++++++++++ src/agent/python/typedefs.py | 7 ++ .../python/nexus_remoting/_encoder.py | 8 +- ...t-tests.py => package-management-tests.py} | 0 11 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 src/agent/python/__init__.py create mode 100644 src/agent/python/options.py create mode 100644 src/agent/python/routers/__init__.py create mode 100644 src/agent/python/routers/package_references.py create mode 100644 src/agent/python/services.py create mode 100644 src/agent/python/typedefs.py rename tests/agent/python-tests/{agent-tests.py => package-management-tests.py} (100%) diff --git a/.nexus-agent-python/config/packages.json b/.nexus-agent-python/config/packages.json index 3967718..73d6b4b 100644 --- a/.nexus-agent-python/config/packages.json +++ b/.nexus-agent-python/config/packages.json @@ -1,10 +1,10 @@ { "c05b592f-e198-472d-9902-3f60cf0a6332": { - "Provider": "local", - "Configuration": { - "path": "../../../tests/Nexus.Sources.Remote.Tests/python", + "provider": "local", + "configuration": { + "path": "../../../tests/Nexus.Sources.Remote.Tests/dotnet", "version": "v1", - "csproj": "remote.py" + "csproj": "main.py" } } } \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 11b6466..7e142e5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,19 @@ "stopAtEntry": false, "env": { "ASPNETCORE_ENVIRONMENT": "Development", - "NEXUSAGENT_Paths__Config": "../../../.nexus-agent-dotnet/config" + "NEXUSAGENT_PATHS__Config": "../../../.nexus-agent-dotnet/config" + } + }, + { + "name": "Launch Nexus.Agent (python)", + "type": "debugpy", + "request": "launch", + "module": "uvicorn", + "args": ["main:app", "--reload"], + "cwd": "${workspaceFolder}/src/agent/python", + "env": { + "PYTHONPATH": "${workspaceFolder}/src/remoting/python", + "NEXUSAGENT_PATHS__Config": "../../../.nexus-agent-python/config" } } ] diff --git a/src/agent/python/__init__.py b/src/agent/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agent/python/main.py b/src/agent/python/main.py index fe5e4e7..4053b67 100644 --- a/src/agent/python/main.py +++ b/src/agent/python/main.py @@ -1,31 +1,6 @@ -from dataclasses import dataclass -import os -from uuid import UUID - -@dataclass -class PackageReference: - provider: str - configuration: dict[str, str] - +import uuid from fastapi import FastAPI - -json_rpc_listen_address = os.getenv("NEXUSAGENT_System__JsonRpcListenAddress", default="0.0.0.0") -json_rpc_listen_port = os.getenv("NEXUSAGENT_System__JsonRpcListenPort", default=56145) +from routers import package_references app = FastAPI() - -@app.get("/api/v1/packagereferences", tags=["PackageReferences"], summary="Gets the list of package references.") -async def get() -> dict[UUID, PackageReference]: - return {} - -@app.post("/api/v1/packagereferences", tags=["PackageReferences"], summary="Creates a package reference.") -async def create(packageReference: PackageReference): - return 0 - -@app.delete("/api/v1/packagereferences/{id}", tags=["PackageReferences"], summary="Deletes a package reference.") -async def delete(id: UUID): - return 0 - -@app.get("/api/v1/packagereferences/{id}/versions", tags=["PackageReferences"], summary="Gets package versions.") -async def get_versions(id: UUID): - return 0 \ No newline at end of file +app.include_router(package_references.router) \ No newline at end of file diff --git a/src/agent/python/options.py b/src/agent/python/options.py new file mode 100644 index 0000000..775d5b3 --- /dev/null +++ b/src/agent/python/options.py @@ -0,0 +1,16 @@ +import os +import platform +from typing import cast + +# Paths options +if platform.system() == "Windows": + platform_specific_root = os.path.join(cast(str, os.getenv('LOCALAPPDATA')), "nexus-agent") + +else: + platform_specific_root = os.path.join(cast(str, os.getenv("HOME")), ".local", "share", "nexus-agent") + +config_folder_path = os.getenv("NEXUSAGENT_PATHS__Config", default=os.path.join(platform_specific_root, "config")) + +# System options +json_rpc_listen_address = os.getenv("NEXUSAGENT_System__JsonRpcListenAddress", default="0.0.0.0") +json_rpc_listen_port = os.getenv("NEXUSAGENT_System__JsonRpcListenPort", default=56145) \ No newline at end of file diff --git a/src/agent/python/routers/__init__.py b/src/agent/python/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agent/python/routers/package_references.py b/src/agent/python/routers/package_references.py new file mode 100644 index 0000000..c914604 --- /dev/null +++ b/src/agent/python/routers/package_references.py @@ -0,0 +1,41 @@ +from uuid import UUID + +from fastapi import APIRouter, HTTPException +from options import config_folder_path +from services import PackageService +from typedefs import PackageReference + +router = APIRouter( + prefix="/api/v1/packagereferences", + tags=["PackageReferences"], +) + +_package_service = PackageService(config_folder_path) + +@router.get("/", tags=["PackageReferences"], summary="Gets the list of package references.") +async def get() -> dict[UUID, PackageReference]: + return await _package_service.get_all() + +@router.post("/", tags=["PackageReferences"], summary="Creates a package reference.") +async def create(package_reference: PackageReference) -> UUID: + return await _package_service.put(package_reference) + +@router.delete("/{id}", tags=["PackageReferences"], summary="Deletes a package reference.") +async def delete(id: UUID): + return await _package_service.delete(id) + +@router.get("/{id}/versions", tags=["PackageReferences"], summary="Gets package versions.") +async def get_versions(id: UUID) -> list[str]: + + package_reference_map = await _package_service.get_all() + + if id in package_reference_map: + package_reference = package_reference_map[id] + + else: + raise HTTPException(status_code=404, detail=f"Unable to find package reference with ID {id}.") + + # result = await _extension_hive.get_versions(package_reference) + + # return result + return [] \ No newline at end of file diff --git a/src/agent/python/services.py b/src/agent/python/services.py new file mode 100644 index 0000000..7ebdf7f --- /dev/null +++ b/src/agent/python/services.py @@ -0,0 +1,99 @@ +import asyncio +import json +import os +import uuid +from pathlib import Path +from typing import Awaitable, Callable, Optional, TypeVar +from uuid import UUID + +from nexus_remoting._encoder import JsonEncoder +from typedefs import PackageReference + +T = TypeVar("T") + +class PackageService: + + _lock = asyncio.Lock() + _cache: Optional[dict[UUID, PackageReference]] = None + + def __init__(self, config_folder_path: str): + self._config_folder_path = config_folder_path + + def put(self, package_reference: PackageReference) -> Awaitable[UUID]: + + return self._interact_with_package_reference_map( + lambda package_reference_map: self._put_internal(package_reference, package_reference_map), + save_changes=True + ) + + def _put_internal(self, package_reference: PackageReference, package_reference_map: dict[UUID, PackageReference]) -> UUID: + + id = uuid.uuid4() + package_reference_map[id] = package_reference + + return id + + def get(self, package_reference_id: UUID) -> Awaitable[Optional[PackageReference]]: + + return self._interact_with_package_reference_map( + lambda package_reference_map: package_reference_map.get(package_reference_id), + save_changes=False + ) + + def delete(self, package_reference_id: UUID) -> Awaitable: + + return self._interact_with_package_reference_map( + lambda package_reference_map: package_reference_map.pop(package_reference_id, None), + save_changes=True + ) + + def get_all(self) -> Awaitable[dict[UUID, PackageReference]]: + + return self._interact_with_package_reference_map( + lambda package_reference_map: package_reference_map, + save_changes=False + ) + + def _get_package_reference_map(self) -> dict[UUID, PackageReference]: + + if self._cache is None: + + folder_path = self._config_folder_path + package_references_file_path = os.path.join(folder_path, "packages.json") + + if (os.path.exists(package_references_file_path)): + + with open(package_references_file_path, "r") as file: + json_value = json.load(file) + + self._cache = JsonEncoder().decode(dict[UUID, PackageReference], json_value) + + else: + return {} + + return self._cache + + async def _interact_with_package_reference_map( + self, + func: Callable[[dict[UUID, PackageReference]], T], + save_changes: bool + ) -> T: + + async with self._lock: + + package_reference_map = self._get_package_reference_map() + result = func(package_reference_map) + + if save_changes: + + folder_path = self._config_folder_path + package_references_file_path = os.path.join(folder_path, "packages.json") + + Path(folder_path).mkdir(parents=True, exist_ok=True) + + json_value = JsonEncoder().encode(package_reference_map) + + with open(package_references_file_path, "w") as file: + json.dump(json_value, file, indent=2) + + return result \ No newline at end of file diff --git a/src/agent/python/typedefs.py b/src/agent/python/typedefs.py new file mode 100644 index 0000000..8e6b727 --- /dev/null +++ b/src/agent/python/typedefs.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class PackageReference: + provider: str + configuration: dict[str, str] \ No newline at end of file diff --git a/src/remoting/python/nexus_remoting/_encoder.py b/src/remoting/python/nexus_remoting/_encoder.py index 002f8e4..b044770 100644 --- a/src/remoting/python/nexus_remoting/_encoder.py +++ b/src/remoting/python/nexus_remoting/_encoder.py @@ -55,7 +55,8 @@ def _try_encode(value: Any, options: JsonEncoderOptions) -> Any: # dict elif isinstance(value, dict): - value = {key:JsonEncoder._try_encode(current_value, options) for key, current_value in value.items()} + # also encode key, it could be a UUID + value = {JsonEncoder._try_encode(key, options):JsonEncoder._try_encode(current_value, options) for key, current_value in value.items()} elif dataclasses.is_dataclass(value): # dataclasses.asdict(value) would be good choice here, but it also converts nested dataclasses into @@ -113,13 +114,14 @@ def _decode(typeCls: Type[T], data: Any, options: JsonEncoderOptions) -> T: # dict elif issubclass(cast(type, origin), dict): - # keyType = args[0] + keyType = args[0] valueType = args[1] instance2: dict = dict() for key, value in data.items(): - instance2[key] = JsonEncoder._decode(valueType, value, options) + # also decode key, it could be a UUID + instance2[JsonEncoder._decode(keyType, key, options)] = JsonEncoder._decode(valueType, value, options) return cast(T, instance2) diff --git a/tests/agent/python-tests/agent-tests.py b/tests/agent/python-tests/package-management-tests.py similarity index 100% rename from tests/agent/python-tests/agent-tests.py rename to tests/agent/python-tests/package-management-tests.py From 191d0f1deecd869401afafe856174957b078461e Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Fri, 17 Jan 2025 15:16:27 +0100 Subject: [PATCH 07/17] Externalize the package management --- .github/workflows/build-and-publish.yml | 1 - Directory.Packages.props | 2 +- pyrightconfig.json | 1 + pytest.ini | 1 - requirements.txt | 5 +- .../PackageReferencesController.cs | 4 +- src/agent/dotnet/Core/AgentService.cs | 2 +- .../Core/InternalControllerFeatureProvider.cs | 2 +- src/agent/dotnet/Program.cs | 2 +- src/agent/dotnet/agent.csproj | 2 +- src/agent/python/main.py | 1 - .../python/routers/package_references.py | 3 +- src/agent/python/services.py | 99 ------------------- src/agent/python/typedefs.py | 7 -- src/remoting/python/nexus_remoting/readme.txt | 1 - src/remoting/python/setup.py | 2 +- .../python-tests/package-management-tests.py | 2 - 17 files changed, 13 insertions(+), 124 deletions(-) delete mode 100644 src/agent/python/services.py delete mode 100644 src/agent/python/typedefs.py delete mode 100644 src/remoting/python/nexus_remoting/readme.txt delete mode 100644 tests/agent/python-tests/package-management-tests.py diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f30b3cc..2aec2b6 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -43,7 +43,6 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: "9.0.x" - dotnet-quality: "preview" - name: Set up Python uses: actions/setup-python@v3 diff --git a/Directory.Packages.props b/Directory.Packages.props index 3c098ab..27e4da8 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,11 +3,11 @@ true + - diff --git a/pyrightconfig.json b/pyrightconfig.json index 00a5481..32fa7d8 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,5 +1,6 @@ { "include": [ + "src/agent/python", "src/remoting/python", "tests/remoting/python-tests", "tests/Nexus.Sources.Remote.Tests/python" diff --git a/pytest.ini b/pytest.ini index e6396ca..a212f5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,5 +8,4 @@ python_functions=*_test pythonpath = src/remoting/python testpaths = - tests/agent/python-tests tests/remoting/python-tests \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0608a9a..1fc9473 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ +apollo3zehn-package-management==1.0.0-b1011 build fastapi[standard] -nexus_api==2.0.0-beta.42 -nexus_extensibility==2.0.0-beta.42 +nexus-api==2.0.0-beta.42 +nexus-extensibility==2.0.0-beta.42 pytest-asyncio wheel \ No newline at end of file diff --git a/src/agent/dotnet/Controllers/PackageReferencesController.cs b/src/agent/dotnet/Controllers/PackageReferencesController.cs index bf2d1b7..d55648c 100644 --- a/src/agent/dotnet/Controllers/PackageReferencesController.cs +++ b/src/agent/dotnet/Controllers/PackageReferencesController.cs @@ -3,8 +3,8 @@ using Asp.Versioning; using Microsoft.AspNetCore.Mvc; -using Nexus.PackageManagement.Services; -using Nexus.PackageManagement; +using Apollo3zehn.PackageManagement.Services; +using Apollo3zehn.PackageManagement; namespace Nexus.Controllers; diff --git a/src/agent/dotnet/Core/AgentService.cs b/src/agent/dotnet/Core/AgentService.cs index 97bda50..88c3fc8 100644 --- a/src/agent/dotnet/Core/AgentService.cs +++ b/src/agent/dotnet/Core/AgentService.cs @@ -5,7 +5,7 @@ using Microsoft.Extensions.Options; using Nexus.Core; using Nexus.Extensibility; -using Nexus.PackageManagement.Services; +using Apollo3zehn.PackageManagement.Services; using Nexus.Remoting; namespace Nexus.Agent; diff --git a/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs b/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs index 2bc8aa8..abd3e9b 100644 --- a/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs +++ b/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs @@ -6,7 +6,7 @@ using Microsoft.AspNetCore.Mvc.Controllers; using System.Reflection; -namespace Nexus.PackageManagement.Core; +namespace Apollo3zehn.PackageManagement.Core; internal class InternalControllerFeatureProvider : IApplicationFeatureProvider { diff --git a/src/agent/dotnet/Program.cs b/src/agent/dotnet/Program.cs index 6db2622..e0c4e04 100644 --- a/src/agent/dotnet/Program.cs +++ b/src/agent/dotnet/Program.cs @@ -3,7 +3,7 @@ using Nexus.Agent; using Nexus.Core; using Nexus.Extensibility; -using Nexus.PackageManagement.Core; +using Apollo3zehn.PackageManagement.Core; using Scalar.AspNetCore; var builder = WebApplication.CreateBuilder(); diff --git a/src/agent/dotnet/agent.csproj b/src/agent/dotnet/agent.csproj index acc09f7..52ea09d 100644 --- a/src/agent/dotnet/agent.csproj +++ b/src/agent/dotnet/agent.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/agent/python/main.py b/src/agent/python/main.py index 4053b67..5bc6b77 100644 --- a/src/agent/python/main.py +++ b/src/agent/python/main.py @@ -1,4 +1,3 @@ -import uuid from fastapi import FastAPI from routers import package_references diff --git a/src/agent/python/routers/package_references.py b/src/agent/python/routers/package_references.py index c914604..6aad788 100644 --- a/src/agent/python/routers/package_references.py +++ b/src/agent/python/routers/package_references.py @@ -1,9 +1,8 @@ from uuid import UUID +from apollo3zehn_package_management import PackageReference, PackageService from fastapi import APIRouter, HTTPException from options import config_folder_path -from services import PackageService -from typedefs import PackageReference router = APIRouter( prefix="/api/v1/packagereferences", diff --git a/src/agent/python/services.py b/src/agent/python/services.py deleted file mode 100644 index 7ebdf7f..0000000 --- a/src/agent/python/services.py +++ /dev/null @@ -1,99 +0,0 @@ -import asyncio -import json -import os -import uuid -from pathlib import Path -from typing import Awaitable, Callable, Optional, TypeVar -from uuid import UUID - -from nexus_remoting._encoder import JsonEncoder -from typedefs import PackageReference - -T = TypeVar("T") - -class PackageService: - - _lock = asyncio.Lock() - _cache: Optional[dict[UUID, PackageReference]] = None - - def __init__(self, config_folder_path: str): - self._config_folder_path = config_folder_path - - def put(self, package_reference: PackageReference) -> Awaitable[UUID]: - - return self._interact_with_package_reference_map( - lambda package_reference_map: self._put_internal(package_reference, package_reference_map), - save_changes=True - ) - - def _put_internal(self, package_reference: PackageReference, package_reference_map: dict[UUID, PackageReference]) -> UUID: - - id = uuid.uuid4() - package_reference_map[id] = package_reference - - return id - - def get(self, package_reference_id: UUID) -> Awaitable[Optional[PackageReference]]: - - return self._interact_with_package_reference_map( - lambda package_reference_map: package_reference_map.get(package_reference_id), - save_changes=False - ) - - def delete(self, package_reference_id: UUID) -> Awaitable: - - return self._interact_with_package_reference_map( - lambda package_reference_map: package_reference_map.pop(package_reference_id, None), - save_changes=True - ) - - def get_all(self) -> Awaitable[dict[UUID, PackageReference]]: - - return self._interact_with_package_reference_map( - lambda package_reference_map: package_reference_map, - save_changes=False - ) - - def _get_package_reference_map(self) -> dict[UUID, PackageReference]: - - if self._cache is None: - - folder_path = self._config_folder_path - package_references_file_path = os.path.join(folder_path, "packages.json") - - if (os.path.exists(package_references_file_path)): - - with open(package_references_file_path, "r") as file: - json_value = json.load(file) - - self._cache = JsonEncoder().decode(dict[UUID, PackageReference], json_value) - - else: - return {} - - return self._cache - - async def _interact_with_package_reference_map( - self, - func: Callable[[dict[UUID, PackageReference]], T], - save_changes: bool - ) -> T: - - async with self._lock: - - package_reference_map = self._get_package_reference_map() - result = func(package_reference_map) - - if save_changes: - - folder_path = self._config_folder_path - package_references_file_path = os.path.join(folder_path, "packages.json") - - Path(folder_path).mkdir(parents=True, exist_ok=True) - - json_value = JsonEncoder().encode(package_reference_map) - - with open(package_references_file_path, "w") as file: - json.dump(json_value, file, indent=2) - - return result \ No newline at end of file diff --git a/src/agent/python/typedefs.py b/src/agent/python/typedefs.py deleted file mode 100644 index 8e6b727..0000000 --- a/src/agent/python/typedefs.py +++ /dev/null @@ -1,7 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class PackageReference: - provider: str - configuration: dict[str, str] \ No newline at end of file diff --git a/src/remoting/python/nexus_remoting/readme.txt b/src/remoting/python/nexus_remoting/readme.txt deleted file mode 100644 index 10b0fd9..0000000 --- a/src/remoting/python/nexus_remoting/readme.txt +++ /dev/null @@ -1 +0,0 @@ -_encoder.py is a copy from Nexus main repo! \ No newline at end of file diff --git a/src/remoting/python/setup.py b/src/remoting/python/setup.py index 96906e0..f7c7dfa 100644 --- a/src/remoting/python/setup.py +++ b/src/remoting/python/setup.py @@ -5,7 +5,7 @@ source_dir = os.getcwd() -build_dir = "../../../artifacts/obj/python-client" +build_dir = "../../../artifacts/obj/python-remoting" Path(build_dir).mkdir(parents=True, exist_ok=True) os.chdir(build_dir) diff --git a/tests/agent/python-tests/package-management-tests.py b/tests/agent/python-tests/package-management-tests.py deleted file mode 100644 index 886236c..0000000 --- a/tests/agent/python-tests/package-management-tests.py +++ /dev/null @@ -1,2 +0,0 @@ -def dummy_test(): - pass \ No newline at end of file From 81516eb29aa8ad29a4c316409a78512ad8dd794b Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Sun, 19 Jan 2025 21:14:40 +0100 Subject: [PATCH 08/17] Switch back to Nswag --- .vscode/launch.json | 4 +- Directory.Packages.props | 8 +- .../PackageReferencesController.cs | 33 +------- .../dotnet/Core/AgentOpenApiExtensions.cs | 80 +++++++++++++++++++ src/agent/dotnet/Core/AgentService.cs | 17 ++-- .../Core/InternalControllerFeatureProvider.cs | 5 +- src/agent/dotnet/Core/Options.cs | 2 +- src/agent/dotnet/Program.cs | 49 +++--------- src/agent/dotnet/agent.csproj | 6 +- 9 files changed, 112 insertions(+), 92 deletions(-) create mode 100644 src/agent/dotnet/Core/AgentOpenApiExtensions.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index 7e142e5..b3f79ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,7 +2,7 @@ "version": "0.2.0", "configurations": [ { - "name": "Launch Nexus.Agent (dotnet)", + "name": "Nexus.Agent (dotnet)", "type": "coreclr", "request": "launch", "preLaunchTask": "build-nexus-agent", @@ -16,7 +16,7 @@ } }, { - "name": "Launch Nexus.Agent (python)", + "name": "Nexus.Agent (python)", "type": "debugpy", "request": "launch", "module": "uvicorn", diff --git a/Directory.Packages.props b/Directory.Packages.props index 27e4da8..b9fef5d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,18 +3,20 @@ true - + + + - + - + \ No newline at end of file diff --git a/src/agent/dotnet/Controllers/PackageReferencesController.cs b/src/agent/dotnet/Controllers/PackageReferencesController.cs index d55648c..106d36b 100644 --- a/src/agent/dotnet/Controllers/PackageReferencesController.cs +++ b/src/agent/dotnet/Controllers/PackageReferencesController.cs @@ -1,12 +1,8 @@ -// MIT License -// Copyright (c) [2024] [nexus-main] - -using Asp.Versioning; using Microsoft.AspNetCore.Mvc; using Apollo3zehn.PackageManagement.Services; using Apollo3zehn.PackageManagement; -namespace Nexus.Controllers; +namespace Nexus.Agent.Controllers; /// /// Provides access to package references. @@ -15,18 +11,14 @@ namespace Nexus.Controllers; [ApiVersion("1.0")] [Route("api/v{version:apiVersion}/[controller]")] internal class PackageReferencesController( - IPackageService packageService, - IExtensionHive extensionHive) : ControllerBase + IPackageService packageService) : ControllerBase { // GET /api/packagereferences // POST /api/packagereferences // DELETE /api/packagereferences/{id} - // GET /api/packagereferences/{id}/versions private readonly IPackageService _packageService = packageService; - private readonly IExtensionHive _extensionHive = extensionHive; - /// /// Gets the list of package references. /// @@ -58,25 +50,4 @@ public Task DeleteAsync( { return _packageService.DeleteAsync(id); } - - /// - /// Gets package versions. - /// - /// The ID of the package reference. - /// A token to cancel the current operation. - [HttpGet("{id}/versions")] - public async Task> GetVersionsAsync( - Guid id, - CancellationToken cancellationToken) - { - var packageReferenceMap = await _packageService.GetAllAsync(); - - if (!packageReferenceMap.TryGetValue(id, out var packageReference)) - return NotFound($"Unable to find package reference with ID {id}."); - - var result = await _extensionHive - .GetVersionsAsync(packageReference, cancellationToken); - - return result; - } } diff --git a/src/agent/dotnet/Core/AgentOpenApiExtensions.cs b/src/agent/dotnet/Core/AgentOpenApiExtensions.cs new file mode 100644 index 0000000..a11e89f --- /dev/null +++ b/src/agent/dotnet/Core/AgentOpenApiExtensions.cs @@ -0,0 +1,80 @@ +using Microsoft.AspNetCore.Mvc.ApiExplorer; +using Nexus.Agent.Core; +using NJsonSchema.Generation; +using NSwag.AspNetCore; +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.DependencyInjection; + +internal static class AgentOpenApiExtensions +{ + public static IServiceCollection AddAgentOpenApi( + this IServiceCollection services + ) + { + // https://github.com/dotnet/aspnet-api-versioning/tree/master/samples/aspnetcore/SwaggerSample + services + .AddControllers() + .AddJsonOptions(options => options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter())) + .ConfigureApplicationPartManager( + manager => manager.FeatureProviders.Add(new InternalControllerFeatureProvider()) + ); + + services.AddApiVersioning( + options => + { + options.ReportApiVersions = true; + }); + + services.AddVersionedApiExplorer( + options => + { + options.GroupNameFormat = "'v'VVV"; + options.SubstituteApiVersionInUrl = true; + }); + + /* not optimal */ + var provider = services.BuildServiceProvider().GetRequiredService(); + + foreach (var description in provider.ApiVersionDescriptions) + { + services.AddOpenApiDocument(config => + { + config.SchemaSettings.DefaultReferenceTypeNullHandling = ReferenceTypeNullHandling.NotNull; + + config.Title = "Nexus.Agent REST API"; + config.Version = description.GroupName; + config.Description = "Manage package references." + + (description.IsDeprecated ? " This API version is deprecated." : ""); + + config.ApiGroupNames = [description.GroupName]; + config.DocumentName = description.GroupName; + }); + } + + return services; + } + + public static WebApplication UseAgentOpenApi( + this WebApplication app, + IApiVersionDescriptionProvider provider + ) + { + app.UseOpenApi(settings => settings.Path = "/openapi/{documentName}.json"); + + app.UseSwaggerUi(settings => + { + settings.Path = "/api"; + + foreach (var description in provider.ApiVersionDescriptions) + { + settings.SwaggerRoutes.Add( + new SwaggerUiRoute( + description.GroupName.ToUpperInvariant(), + $"/openapi/{description.GroupName}.json")); + } + }); + + return app; + } +} diff --git a/src/agent/dotnet/Core/AgentService.cs b/src/agent/dotnet/Core/AgentService.cs index 88c3fc8..935f51f 100644 --- a/src/agent/dotnet/Core/AgentService.cs +++ b/src/agent/dotnet/Core/AgentService.cs @@ -1,14 +1,13 @@ +using Apollo3zehn.PackageManagement.Services; +using Microsoft.Extensions.Options; +using Nexus.Extensibility; +using Nexus.Remoting; using System.Collections.Concurrent; using System.Net; using System.Net.Sockets; using System.Text; -using Microsoft.Extensions.Options; -using Nexus.Core; -using Nexus.Extensibility; -using Apollo3zehn.PackageManagement.Services; -using Nexus.Remoting; -namespace Nexus.Agent; +namespace Nexus.Agent.Core; public class TcpClientPair { @@ -29,7 +28,7 @@ internal class AgentService private readonly ConcurrentDictionary _tcpClientPairs = new(); - private readonly IExtensionHive _extensionHive; + private readonly IExtensionHive _extensionHive; private readonly IPackageService _packageService; @@ -38,7 +37,7 @@ internal class AgentService private readonly SystemOptions _systemOptions; public AgentService( - IExtensionHive extensionHive, + IExtensionHive extensionHive, IPackageService packageService, ILogger agentLogger, IOptions systemOptions) @@ -194,7 +193,7 @@ public Task AcceptClientsAsync(CancellationToken cancellationToken) pair.RemoteCommunicator = new RemoteCommunicator( pair.Comm, pair.Data, - getDataSource: type => _extensionHive.GetInstance(type) + getDataSource: type => _extensionHive.GetInstance(type) ); _ = pair.RemoteCommunicator.RunAsync(pair.CancellationTokenSource.Token); diff --git a/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs b/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs index abd3e9b..8be658b 100644 --- a/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs +++ b/src/agent/dotnet/Core/InternalControllerFeatureProvider.cs @@ -1,12 +1,9 @@ -// MIT License -// Copyright (c) [2024] [nexus-main] - using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.ApplicationParts; using Microsoft.AspNetCore.Mvc.Controllers; using System.Reflection; -namespace Apollo3zehn.PackageManagement.Core; +namespace Nexus.Agent.Core; internal class InternalControllerFeatureProvider : IApplicationFeatureProvider { diff --git a/src/agent/dotnet/Core/Options.cs b/src/agent/dotnet/Core/Options.cs index f90960a..1076fc9 100644 --- a/src/agent/dotnet/Core/Options.cs +++ b/src/agent/dotnet/Core/Options.cs @@ -1,6 +1,6 @@ using System.Runtime.InteropServices; -namespace Nexus.Core; +namespace Nexus.Agent.Core; internal abstract record NexusAgentOptions() { diff --git a/src/agent/dotnet/Program.cs b/src/agent/dotnet/Program.cs index e0c4e04..fdf453f 100644 --- a/src/agent/dotnet/Program.cs +++ b/src/agent/dotnet/Program.cs @@ -1,59 +1,32 @@ -using Asp.Versioning; +using Microsoft.AspNetCore.Mvc.ApiExplorer; using Microsoft.Extensions.Options; -using Nexus.Agent; -using Nexus.Core; +using Nexus.Agent.Core; using Nexus.Extensibility; -using Apollo3zehn.PackageManagement.Core; -using Scalar.AspNetCore; var builder = WebApplication.CreateBuilder(); var configuration = NexusAgentOptions.BuildConfiguration(); builder.Configuration.AddConfiguration(configuration); -builder.Services +builder.Services.AddAgentOpenApi(); +builder.Services.AddPackageManagement(); +builder.Services.AddExtensionHive(); - .AddOpenApi() - // .AddOpenApi("v2") +builder.Services.AddSingleton(); - .AddApiVersioning(config => - { - config.ReportApiVersions = true; - config.ApiVersionReader = new UrlSegmentApiVersionReader(); - }) - - .AddApiExplorer(config => - { - config.GroupNameFormat = "'v'VVV"; - config.SubstituteApiVersionInUrl = true; - }); - -builder.Services - .AddControllers() - .ConfigureApplicationPartManager( - manager => manager.FeatureProviders.Add(new InternalControllerFeatureProvider()) - ); - -builder.Services - .AddSingleton(); +builder.Services.AddSingleton( + serviceProvider => serviceProvider.GetRequiredService>().Value); builder.Services.Configure(configuration.GetSection(SystemOptions.Section)); builder.Services.Configure(configuration.GetSection(PathsOptions.Section)); -// Package management -builder.Services.AddPackageManagement(); - -builder.Services.AddSingleton( - serviceProvider => serviceProvider.GetRequiredService>().Value); - var app = builder.Build(); -var b = app.Services.GetRequiredService>().Value; +var provider = app.Services.GetRequiredService(); +app.UseAgentOpenApi(provider); -app.MapGet("/", () => Results.Redirect("/scalar/v1")).ExcludeFromDescription(); -app.MapOpenApi(); -app.MapScalarApiReference(); app.MapControllers(); +app.MapGet("/", () => Results.Redirect("/api")); var pathsOptions = app.Services.GetRequiredService>(); diff --git a/src/agent/dotnet/agent.csproj b/src/agent/dotnet/agent.csproj index 52ea09d..200e60f 100644 --- a/src/agent/dotnet/agent.csproj +++ b/src/agent/dotnet/agent.csproj @@ -7,12 +7,10 @@ - - - + - + From 48efd3a02a51cfc9edebf1fe91340c704ec07489 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 14:09:34 +0100 Subject: [PATCH 09/17] Intermediate commit --- .nexus-agent-dotnet/config/packages.json | 2 +- .nexus-agent-python/config/packages.json | 5 +- .vscode/launch.json | 8 +- Directory.Packages.props | 2 +- requirements.txt | 2 +- src/agent/dotnet/Core/AgentService.cs | 148 +++++++---------- src/agent/python/main.py | 38 ++++- src/agent/python/options.py | 8 +- .../python/routers/package_references.py | 23 +-- src/agent/python/services.py | 156 ++++++++++++++++++ .../python/nexus_remoting/_remoting.py | 15 +- .../RemoteTestsFixture.cs | 19 ++- .../python/{ => v1}/test.py | 0 13 files changed, 292 insertions(+), 134 deletions(-) create mode 100644 src/agent/python/services.py rename tests/Nexus.Sources.Remote.Tests/python/{ => v1}/test.py (100%) diff --git a/.nexus-agent-dotnet/config/packages.json b/.nexus-agent-dotnet/config/packages.json index 25a9090..6194cbc 100644 --- a/.nexus-agent-dotnet/config/packages.json +++ b/.nexus-agent-dotnet/config/packages.json @@ -4,7 +4,7 @@ "Configuration": { "path": "../../../tests/Nexus.Sources.Remote.Tests/dotnet", "version": "v1", - "csproj": "test.csproj" + "entrypoint": "test.csproj" } } } \ No newline at end of file diff --git a/.nexus-agent-python/config/packages.json b/.nexus-agent-python/config/packages.json index 73d6b4b..4b2a81b 100644 --- a/.nexus-agent-python/config/packages.json +++ b/.nexus-agent-python/config/packages.json @@ -2,9 +2,10 @@ "c05b592f-e198-472d-9902-3f60cf0a6332": { "provider": "local", "configuration": { - "path": "../../../tests/Nexus.Sources.Remote.Tests/dotnet", + "path": "../../../tests/Nexus.Sources.Remote.Tests/python", "version": "v1", - "csproj": "main.py" + "module-name": "foo", + "entrypoint": "test.py" } } } \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index b3f79ba..c873b09 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,19 +12,19 @@ "stopAtEntry": false, "env": { "ASPNETCORE_ENVIRONMENT": "Development", - "NEXUSAGENT_PATHS__Config": "../../../.nexus-agent-dotnet/config" + "NEXUSAGENT_PATHS__CONFIG": "../../../.nexus-agent-dotnet/config" } }, { "name": "Nexus.Agent (python)", "type": "debugpy", "request": "launch", - "module": "uvicorn", - "args": ["main:app", "--reload"], + "module": "fastapi", + "args": ["run"], "cwd": "${workspaceFolder}/src/agent/python", "env": { "PYTHONPATH": "${workspaceFolder}/src/remoting/python", - "NEXUSAGENT_PATHS__Config": "../../../.nexus-agent-python/config" + "NEXUSAGENT_PATHS__CONFIG": "../../../.nexus-agent-python/config" } } ] diff --git a/Directory.Packages.props b/Directory.Packages.props index b9fef5d..a076738 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,7 +3,7 @@ true - + diff --git a/requirements.txt b/requirements.txt index 1fc9473..00cfd00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apollo3zehn-package-management==1.0.0-b1011 +apollo3zehn-package-management==1.0.0-b1019 build fastapi[standard] nexus-api==2.0.0-beta.42 diff --git a/src/agent/dotnet/Core/AgentService.cs b/src/agent/dotnet/Core/AgentService.cs index 935f51f..11fdc3c 100644 --- a/src/agent/dotnet/Core/AgentService.cs +++ b/src/agent/dotnet/Core/AgentService.cs @@ -3,6 +3,7 @@ using Nexus.Extensibility; using Nexus.Remoting; using System.Collections.Concurrent; +using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Text; @@ -17,6 +18,8 @@ public class TcpClientPair public RemoteCommunicator? RemoteCommunicator { get; set; } + public Stopwatch WatchdogTimer = new(); + public CancellationTokenSource CancellationTokenSource { get; } = new(); } @@ -32,25 +35,25 @@ internal class AgentService private readonly IPackageService _packageService; - private readonly ILogger _agentLogger; + private readonly ILogger _logger; private readonly SystemOptions _systemOptions; public AgentService( IExtensionHive extensionHive, IPackageService packageService, - ILogger agentLogger, + ILogger logger, IOptions systemOptions) { _extensionHive = extensionHive; _packageService = packageService; - _agentLogger = agentLogger; + _logger = logger; _systemOptions = systemOptions.Value; } public async Task LoadPackagesAsync(CancellationToken cancellationToken) { - _agentLogger.LogInformation("Load packages"); + _logger.LogInformation("Load packages"); var packageReferenceMap = await _packageService.GetAllAsync(); var progress = new Progress(); @@ -64,7 +67,7 @@ await _extensionHive.LoadPackagesAsync( public Task AcceptClientsAsync(CancellationToken cancellationToken) { - _agentLogger.LogInformation( + _logger.LogInformation( "Listening for JSON-RPC communication on {JsonRpcListenAddress}:{JsonRpcListenPort}", _systemOptions.JsonRpcListenAddress, _systemOptions.JsonRpcListenPort ); @@ -85,7 +88,11 @@ public Task AcceptClientsAsync(CancellationToken cancellationToken) foreach (var (key, pair) in _tcpClientPairs) { - if (pair.RemoteCommunicator!.LastCommunication >= CLIENT_TIMEOUT) + var isDead = + (pair.Comm is null || pair.Data is null) && pair.WatchdogTimer.Elapsed > CLIENT_TIMEOUT || + pair.RemoteCommunicator?.LastCommunication >= CLIENT_TIMEOUT; + + if (isDead) { if (_tcpClientPairs.TryRemove(key, out var _)) pair.CancellationTokenSource.Cancel(); @@ -109,103 +116,76 @@ public Task AcceptClientsAsync(CancellationToken cancellationToken) var streamReadCts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); var networkStream = client.GetStream(); /* no 'using' because it would close the TCP client */ - // get connection id + // Get connection id var buffer1 = new byte[36]; await networkStream.ReadExactlyAsync(buffer1, streamReadCts.Token); var idString = Encoding.UTF8.GetString(buffer1); - // get connection type + // Get connection type var buffer2 = new byte[4]; await networkStream.ReadExactlyAsync(buffer2, streamReadCts.Token); var typeString = Encoding.UTF8.GetString(buffer2); - if (Guid.TryParse(Encoding.UTF8.GetString(buffer1), out var id)) + if (!Guid.TryParse(idString, out var id)) { - _agentLogger.LogDebug("Accept TCP client with connection ID {ConnectionId} and communication type {CommunicationType}", idString, typeString); + client.Dispose(); + return; + } - var tcpPairCts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + _logger.LogDebug("Accept TCP client with connection ID {ConnectionId} and communication type {CommunicationType}", idString, typeString); - // Handle the timeout event - tcpPairCts.Token.Register(() => - { - // If TCP client pair can be found ... - if (_tcpClientPairs.TryGetValue(id, out var pair)) + TcpClientPair pair; + + // We got a "comm" tcp connection + if (typeString == "comm") + { + pair = _tcpClientPairs.AddOrUpdate( + id, + addValueFactory: id => new TcpClientPair { Comm = networkStream }, + updateValueFactory: (id, pair) => { - // and if TCP client pair is not yet complete ... - if (pair.Comm is null || pair.Data is null) - { - // then dispose and remove the clients and the pair - pair.Comm?.Dispose(); - pair.Data?.Dispose(); - - _tcpClientPairs.Remove(id, out _); - } + pair.Comm?.Dispose(); + pair.Comm = networkStream; + return pair; } - }); + ); + } - // We got a "comm" tcp connection - if (typeString == "comm") - { - _tcpClientPairs.AddOrUpdate( - id, - addValueFactory: id => new TcpClientPair { Comm = networkStream }, - updateValueFactory: (id, pair) => - { - pair.Comm?.Dispose(); - pair.Comm = networkStream; - return pair; - } - ); - } + // We got a "data" tcp connection + else if (typeString == "data") + { + pair = _tcpClientPairs.AddOrUpdate( + id, + addValueFactory: id => new TcpClientPair { Data = networkStream }, + updateValueFactory: (id, pair) => + { + pair.Data?.Dispose(); + pair.Data = networkStream; + return pair; + } + ); + } - // We got a "data" tcp connection - else if (typeString == "data") - { - _tcpClientPairs.AddOrUpdate( - id, - addValueFactory: id => new TcpClientPair { Data = networkStream }, - updateValueFactory: (id, pair) => - { - pair.Data?.Dispose(); - pair.Data = networkStream; - return pair; - } - ); - } + // Something went wrong, dispose the network stream and return + else + { + networkStream.Dispose(); + return; + } - // Something went wrong, dispose the network stream and return - else + lock (_lock) + { + if (pair.Comm is not null && pair.Data is not null && pair.RemoteCommunicator is null) { - networkStream.Dispose(); - return; - } + _logger.LogDebug("Accept remoting client with connection ID {ConnectionId}", id); - var pair = _tcpClientPairs[id]; + pair.RemoteCommunicator = new RemoteCommunicator( + pair.Comm, + pair.Data, + getDataSource: type => _extensionHive.GetInstance(type) + ); - lock (_lock) - { - if (pair.Comm is not null && pair.Data is not null && pair.RemoteCommunicator is null) - { - _agentLogger.LogDebug("Accept remoting client with connection ID {ConnectionId}", id); - - try - { - pair.RemoteCommunicator = new RemoteCommunicator( - pair.Comm, - pair.Data, - getDataSource: type => _extensionHive.GetInstance(type) - ); - - _ = pair.RemoteCommunicator.RunAsync(pair.CancellationTokenSource.Token); - } - catch - { - pair.Comm?.Dispose(); - pair.Data?.Dispose(); - - throw; - } - } + _ = pair.RemoteCommunicator.RunAsync(pair.CancellationTokenSource.Token); } } }); diff --git a/src/agent/python/main.py b/src/agent/python/main.py index 5bc6b77..6b38c90 100644 --- a/src/agent/python/main.py +++ b/src/agent/python/main.py @@ -1,5 +1,37 @@ +import asyncio +import logging + +from apollo3zehn_package_management._services import (ExtensionHive, + PackageService) from fastapi import FastAPI -from routers import package_references +from nexus_extensibility import IDataSource + +from .options import (config_folder_path, json_rpc_listen_address, + json_rpc_listen_port, packages_folder_path) +from .routers import package_references +from .services import AgentService + + +async def main(): + + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger() + + extension_hive = ExtensionHive[IDataSource](packages_folder_path, logger) + package_service = PackageService(config_folder_path) + + agent_service = AgentService( + extension_hive, + package_service, + logger, + json_rpc_listen_address, + json_rpc_listen_port + ) + + await agent_service.load_packages() + agent_service.accept_clients() + + app = FastAPI() + app.include_router(package_references.router) -app = FastAPI() -app.include_router(package_references.router) \ No newline at end of file +asyncio.run(main()) \ No newline at end of file diff --git a/src/agent/python/options.py b/src/agent/python/options.py index 775d5b3..b6b9139 100644 --- a/src/agent/python/options.py +++ b/src/agent/python/options.py @@ -9,8 +9,8 @@ else: platform_specific_root = os.path.join(cast(str, os.getenv("HOME")), ".local", "share", "nexus-agent") -config_folder_path = os.getenv("NEXUSAGENT_PATHS__Config", default=os.path.join(platform_specific_root, "config")) +config_folder_path = os.getenv("NEXUSAGENT_PATHS__CONFIG", default=os.path.join(platform_specific_root, "config")) +packages_folder_path = os.getenv("NEXUSAGENT_PATHS__PACKAGES", default=os.path.join(platform_specific_root, "packages")) -# System options -json_rpc_listen_address = os.getenv("NEXUSAGENT_System__JsonRpcListenAddress", default="0.0.0.0") -json_rpc_listen_port = os.getenv("NEXUSAGENT_System__JsonRpcListenPort", default=56145) \ No newline at end of file +json_rpc_listen_address = os.getenv("NEXUSAGENT_SYSTEM__JSONRPCLISTENADDRESS", default="0.0.0.0") +json_rpc_listen_port = int(os.getenv("NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT", default="56145")) \ No newline at end of file diff --git a/src/agent/python/routers/package_references.py b/src/agent/python/routers/package_references.py index 6aad788..925bd64 100644 --- a/src/agent/python/routers/package_references.py +++ b/src/agent/python/routers/package_references.py @@ -1,8 +1,9 @@ from uuid import UUID from apollo3zehn_package_management import PackageReference, PackageService -from fastapi import APIRouter, HTTPException -from options import config_folder_path +from fastapi import APIRouter + +from ..options import config_folder_path router = APIRouter( prefix="/api/v1/packagereferences", @@ -21,20 +22,4 @@ async def create(package_reference: PackageReference) -> UUID: @router.delete("/{id}", tags=["PackageReferences"], summary="Deletes a package reference.") async def delete(id: UUID): - return await _package_service.delete(id) - -@router.get("/{id}/versions", tags=["PackageReferences"], summary="Gets package versions.") -async def get_versions(id: UUID) -> list[str]: - - package_reference_map = await _package_service.get_all() - - if id in package_reference_map: - package_reference = package_reference_map[id] - - else: - raise HTTPException(status_code=404, detail=f"Unable to find package reference with ID {id}.") - - # result = await _extension_hive.get_versions(package_reference) - - # return result - return [] \ No newline at end of file + return await _package_service.delete(id) \ No newline at end of file diff --git a/src/agent/python/services.py b/src/agent/python/services.py new file mode 100644 index 0000000..6619404 --- /dev/null +++ b/src/agent/python/services.py @@ -0,0 +1,156 @@ +import asyncio +import socket +import time +import uuid +from datetime import timedelta +from logging import Logger +from typing import Optional + +from apollo3zehn_package_management._services import (ExtensionHive, + PackageService) +from nexus_remoting._remoting import RemoteCommunicator + + +class TcpClientPair: + comm: Optional[socket.socket] + data: Optional[socket.socket] + remote_communicator: Optional[RemoteCommunicator] + watchdog_timer = time.time() + task: Optional[asyncio.Task] + +class AgentService: + + CLIENT_TIMEOUT = timedelta(minutes=1) + + _tcp_client_pairs: dict[uuid.UUID, TcpClientPair] = {} + _lock = asyncio.Lock() + + def __init__( + self, + extension_hive: ExtensionHive, + package_service: PackageService, + logger: Logger, + json_rpc_listen_address: str, + json_rpc_listen_port: int + ): + + self._extension_hive = extension_hive + self._package_service = package_service + self._logger = logger + self._json_rpc_listen_address = json_rpc_listen_address + self._json_rpc_listen_port = json_rpc_listen_port + + async def load_packages(self): + + self._logger.info("Load packages") + package_reference_map = await self._package_service.get_all() + await self._extension_hive.load_packages(package_reference_map) + + def accept_clients(self): + + self._logger.info( + "Listening for JSON-RPC communication on %s:%d", + self._json_rpc_listen_address, + self._json_rpc_listen_port + ) + + tcp_listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp_listener.bind((self._json_rpc_listen_address, self._json_rpc_listen_port)) + tcp_listener.listen() + + async def detect_and_remove_inactive_clients(): + + while True: + + await asyncio.sleep(600) + + for key, pair in list(self._tcp_client_pairs.items()): + + now = time.time() + watchdog_timer_elasped = timedelta(seconds=now - pair.watchdog_timer) + + is_dead =\ + (pair.comm is None or pair.data is None) and watchdog_timer_elasped >= self.CLIENT_TIMEOUT or \ + pair.remote_communicator is not None and pair.remote_communicator.last_communication >= self.CLIENT_TIMEOUT + + if is_dead: + + if key in self._tcp_client_pairs: + + # TODO: Add proper cancellation https://medium.com/nuculabs/cancellation-token-pattern-in-python-b549d894e244 + # Otherwise sockets may stay open + if pair.task is not None: + pair.task.cancel() + + del self._tcp_client_pairs[key] + + asyncio.create_task(detect_and_remove_inactive_clients()) + + async def accept_new_clients(): + + loop = asyncio.get_event_loop() + + while True: + client, _ = await loop.sock_accept(tcp_listener) + asyncio.create_task(self._handle_client(client)) + + asyncio.create_task(accept_new_clients()) + + async def _handle_client(self, client: socket.socket): + + stream_read_timeout = 1 + client.settimeout(stream_read_timeout) + + # Get connection id + buffer1 = client.recv(36) + id_string = buffer1.decode("utf-8") + + # Get connection type + buffer2 = client.recv(4) + type_string = buffer2.decode("utf-8") + + try: + id = uuid.UUID(id_string) + + except: + client.close() + return + + self._logger.debug("Accept TCP client with connection ID %s and communication type %s", id_string, type_string) + + async with self._lock: + + # We got a "comm" tcp connection + if type_string == "comm": + + if id not in self._tcp_client_pairs: + self._tcp_client_pairs[id] = TcpClientPair() + + self._tcp_client_pairs[id].comm = client + + # We got a "data" tcp connection + elif type_string == "data": + + if id not in self._tcp_client_pairs: + self._tcp_client_pairs[id] = TcpClientPair() + + self._tcp_client_pairs[id].data = client + + # Something went wrong, close the socket and return + else: + client.close() + return + + pair = self._tcp_client_pairs[id] + + if pair.comm and pair.data and not pair.remote_communicator: + + self._logger.debug("Accept remoting client with connection ID %s", id) + + pair.remote_communicator = RemoteCommunicator( + pair.comm, + pair.data, + get_data_source=lambda type: self._extension_hive.get_instance(type) + ) + + pair.task = asyncio.create_task(pair.remote_communicator.run()) \ No newline at end of file diff --git a/src/remoting/python/nexus_remoting/_remoting.py b/src/remoting/python/nexus_remoting/_remoting.py index fabfff1..066ec29 100644 --- a/src/remoting/python/nexus_remoting/_remoting.py +++ b/src/remoting/python/nexus_remoting/_remoting.py @@ -4,7 +4,7 @@ import time from datetime import datetime, timedelta from threading import Lock -from typing import Any, Callable, Dict, Optional, Tuple, cast +from typing import Any, Awaitable, Callable, Dict, Optional, Tuple, cast from urllib.parse import urlparse from nexus_extensibility import (CatalogItem, DataSourceContext, @@ -62,15 +62,14 @@ def __init__( self._comm_socket = comm_socket self._data_socket = data_socket - self._get_data_source = get_data_source @property def last_communication(self) -> timedelta: - end = time.time() - return timedelta(seconds=end - self._watchdog_timer) + now = time.time() + return timedelta(seconds=now - self._watchdog_timer) - async def run(self): + async def run(self) -> Awaitable: """ Starts the remoting operation. """ @@ -133,7 +132,11 @@ async def run(self): self._data_socket.sendall(status) async def _process_invocation(self, request: dict[str, Any]) \ - -> Tuple[Optional[Dict[str, Any]], Optional[memoryview], Optional[memoryview]]: + -> Tuple[ + Optional[Dict[str, Any]], + Optional[memoryview], + Optional[memoryview] + ]: result: Optional[Dict[str, Any]] = None data: Optional[memoryview] = None diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs index 628eb29..22609a6 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -19,7 +19,7 @@ public RemoteTestsFixture() Initialize = Task.Run(() => { var dotnetTask = RunDotnetAgent(); - var pythonTask = Task.CompletedTask; // RunPythonAgent(); + var pythonTask = RunPythonAgent(); return Task.WhenAll(dotnetTask, pythonTask); }); @@ -83,8 +83,8 @@ private async Task RunDotnetAgent() RedirectStandardError = true }; - psi_run.Environment["NEXUSAGENT_System__JsonRpcListenPort"] = "60000"; - psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../../.nexus-agent-dotnet/config"; + psi_run.Environment["NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT"] = "60000"; + psi_run.Environment["NEXUSAGENT_PATHS__CONFIG"] = "../../../.nexus-agent-dotnet/config"; _runProcess = new Process { @@ -126,17 +126,18 @@ private async Task RunDotnetAgent() private async Task RunPythonAgent() { - var psi_run = new ProcessStartInfo("fastapi") + var psi_run = new ProcessStartInfo("/usr/bin/bash") { - Arguments = $"run main.py", + Arguments = $@"-c ""source ../../../.venv/bin/activate; fastapi run main.py""", WorkingDirectory="../../../../src/agent/python", UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true }; - psi_run.Environment["NEXUSAGENT_System__JsonRpcListenPort"] = "60001"; - psi_run.Environment["NEXUSAGENT_Paths__Config"] = "../../../.nexus-agent-python/config"; + psi_run.Environment["PYTHONPATH"] = "../../remoting/python"; + psi_run.Environment["NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT"] = "60001"; + psi_run.Environment["NEXUSAGENT_PATHS__CONFIG"] = "../../../.nexus-agent-python/config"; _runProcess = new Process { @@ -146,7 +147,7 @@ private async Task RunPythonAgent() _runProcess.OutputDataReceived += (sender, e) => { - // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); if (e.Data is not null && e.Data.Contains("Application startup complete.")) { @@ -157,7 +158,7 @@ private async Task RunPythonAgent() _runProcess.ErrorDataReceived += (sender, e) => { - // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); var oldSuccess = _success; _success = false; diff --git a/tests/Nexus.Sources.Remote.Tests/python/test.py b/tests/Nexus.Sources.Remote.Tests/python/v1/test.py similarity index 100% rename from tests/Nexus.Sources.Remote.Tests/python/test.py rename to tests/Nexus.Sources.Remote.Tests/python/v1/test.py From 9c952705d08fd1e2a96c94b91b235f4eea57016a Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 21:14:09 +0100 Subject: [PATCH 10/17] All tests work --- .vscode/launch.json | 6 +- src/agent/python/main.py | 37 +++---- src/agent/python/services.py | 18 ++-- .../Nexus.Sources.Remote.Tests/RemoteTests.cs | 70 ++++++++----- .../RemoteTestsFixture.cs | 98 ++++++++++--------- .../python/v1/test.py | 11 ++- 6 files changed, 140 insertions(+), 100 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c873b09..2ef5f52 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,8 @@ "stopAtEntry": false, "env": { "ASPNETCORE_ENVIRONMENT": "Development", - "NEXUSAGENT_PATHS__CONFIG": "../../../.nexus-agent-dotnet/config" + "NEXUSAGENT_PATHS__CONFIG": "../../../.nexus-agent-dotnet/config", + "NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT": "60000" } }, { @@ -24,7 +25,8 @@ "cwd": "${workspaceFolder}/src/agent/python", "env": { "PYTHONPATH": "${workspaceFolder}/src/remoting/python", - "NEXUSAGENT_PATHS__CONFIG": "../../../.nexus-agent-python/config" + "NEXUSAGENT_PATHS__CONFIG": "../../../.nexus-agent-python/config", + "NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT": "60001" } } ] diff --git a/src/agent/python/main.py b/src/agent/python/main.py index 6b38c90..e370676 100644 --- a/src/agent/python/main.py +++ b/src/agent/python/main.py @@ -1,5 +1,7 @@ import asyncio import logging +import sys +from contextlib import asynccontextmanager from apollo3zehn_package_management._services import (ExtensionHive, PackageService) @@ -11,27 +13,28 @@ from .routers import package_references from .services import AgentService +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger() -async def main(): - - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger() +extension_hive = ExtensionHive[IDataSource](packages_folder_path, logger) +package_service = PackageService(config_folder_path) - extension_hive = ExtensionHive[IDataSource](packages_folder_path, logger) - package_service = PackageService(config_folder_path) - - agent_service = AgentService( - extension_hive, - package_service, - logger, - json_rpc_listen_address, - json_rpc_listen_port - ) +agent_service = AgentService( + extension_hive, + package_service, + logger, + json_rpc_listen_address, + json_rpc_listen_port +) +async def main(): await agent_service.load_packages() agent_service.accept_clients() - app = FastAPI() - app.include_router(package_references.router) +@asynccontextmanager +async def lifespan(app: FastAPI): + asyncio.create_task(main()) + yield -asyncio.run(main()) \ No newline at end of file +app = FastAPI(lifespan=lifespan) +app.include_router(package_references.router) diff --git a/src/agent/python/services.py b/src/agent/python/services.py index 6619404..00d0728 100644 --- a/src/agent/python/services.py +++ b/src/agent/python/services.py @@ -12,11 +12,11 @@ class TcpClientPair: - comm: Optional[socket.socket] - data: Optional[socket.socket] - remote_communicator: Optional[RemoteCommunicator] + comm: Optional[socket.socket] = None + data: Optional[socket.socket] = None + remote_communicator: Optional[RemoteCommunicator] = None watchdog_timer = time.time() - task: Optional[asyncio.Task] + task: Optional[asyncio.Task] = None class AgentService: @@ -55,6 +55,7 @@ def accept_clients(self): ) tcp_listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp_listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcp_listener.bind((self._json_rpc_listen_address, self._json_rpc_listen_port)) tcp_listener.listen() @@ -92,7 +93,11 @@ async def accept_new_clients(): while True: client, _ = await loop.sock_accept(tcp_listener) - asyncio.create_task(self._handle_client(client)) + try: + await self._handle_client(client) + # await loop.create_task(self._handle_client(client)) + except Exception as ex: + b = 1 asyncio.create_task(accept_new_clients()) @@ -153,4 +158,5 @@ async def _handle_client(self, client: socket.socket): get_data_source=lambda type: self._extension_hive.get_instance(type) ) - pair.task = asyncio.create_task(pair.remote_communicator.run()) \ No newline at end of file + # pair.task = asyncio.create_task(pair.remote_communicator.run()) + await pair.remote_communicator.run() \ No newline at end of file diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs index 46ed92d..6c8f3ed 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTests.cs @@ -13,18 +13,33 @@ namespace Nexus.Sources.Tests; public class RemoteTests(RemoteTestsFixture fixture) : IClassFixture { + private const string DOTNET = "DOTNET"; + private const string PYTHON = "PYTHON"; + + private static readonly Dictionary _portMap = new() + { + [DOTNET] = 60000, + [PYTHON] = 60001 + }; + + private static readonly Dictionary _extensionNameMap = new() + { + [DOTNET] = "Nexus.Sources.Test", + [PYTHON] = "foo.Test" + }; + private readonly RemoteTestsFixture _fixture = fixture; [Theory] - [InlineData(60000 /* dotnet */)] - [InlineData(60001 /* python */)] - public async Task ProvidesCatalog(int port) + [InlineData(DOTNET)] + [InlineData(PYTHON)] + public async Task ProvidesCatalog(string language) { await _fixture.Initialize; // Arrange var dataSource = new Remote() as IDataSource; - var context = CreateContext(port); + var context = CreateContext(language); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -52,14 +67,14 @@ public async Task ProvidesCatalog(int port) } [Theory] - [InlineData(60000 /* dotnet */)] - [InlineData(60001 /* python */)] - public async Task CanProvideTimeRange(int port) + [InlineData(DOTNET)] + [InlineData(PYTHON)] + public async Task CanProvideTimeRange(string language) { await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(port); + var context = CreateContext(language); var expectedBegin = new DateTime(2019, 12, 31, 12, 00, 00, DateTimeKind.Utc); var expectedEnd = new DateTime(2020, 01, 02, 09, 50, 00, DateTimeKind.Utc); @@ -73,14 +88,14 @@ public async Task CanProvideTimeRange(int port) } [Theory] - [InlineData(60000 /* dotnet */)] - [InlineData(60001 /* python */)] - public async Task CanProvideAvailability(int port) + [InlineData(DOTNET)] + [InlineData(PYTHON)] + public async Task CanProvideAvailability(string language) { await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(port); + var context = CreateContext(language); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -92,9 +107,9 @@ public async Task CanProvideAvailability(int port) } [Theory] - [InlineData(60000 /* dotnet */)] - [InlineData(60001 /* python */)] - public async Task CanReadFullDay(int port) + [InlineData(DOTNET)] + [InlineData(PYTHON)] + public async Task CanReadFullDay(string language) { // TODO fix this var complexData = true; @@ -102,7 +117,7 @@ public async Task CanReadFullDay(int port) await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(port); + var context = CreateContext(language); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -158,15 +173,15 @@ void GenerateData(DateTimeOffset dateTime) } [Theory] - [InlineData(60000 /* dotnet */)] - [InlineData(60001 /* python */)] - public async Task CanLog(int port) + [InlineData(DOTNET)] + [InlineData(PYTHON)] + public async Task CanLog(string language) { await _fixture.Initialize; var loggerMock = new Mock(); var dataSource = new Remote() as IDataSource; - var context = CreateContext(port); + var context = CreateContext(language); await dataSource.SetContextAsync(context, loggerMock.Object, CancellationToken.None); @@ -183,14 +198,14 @@ public async Task CanLog(int port) } [Theory] - [InlineData(60000 /* dotnet */)] - [InlineData(60001 /* python */)] - public async Task CanReadDataHandler(int port) + [InlineData(DOTNET)] + [InlineData(PYTHON)] + public async Task CanReadDataHandler(string language) { await _fixture.Initialize; var dataSource = new Remote() as IDataSource; - var context = CreateContext(port); + var context = CreateContext(language); await dataSource.SetContextAsync(context, NullLogger.Instance, CancellationToken.None); @@ -240,8 +255,11 @@ Task HandleReadDataAsync(string resourcePath, DateTime begin, DateTime end, Memo Assert.True(expectedStatus.SequenceEqual(status.ToArray())); } - private static DataSourceContext CreateContext(int port) + private static DataSourceContext CreateContext(string language) { + var port = _portMap[language]; + var extensionName = _extensionNameMap[language]; + return new DataSourceContext( ResourceLocator: new Uri($"tcp://127.0.0.1:{port}"), SystemConfiguration: new Dictionary() @@ -256,7 +274,7 @@ private static DataSourceContext CreateContext(int port) }, SourceConfiguration: new Dictionary() { - ["type"] = JsonSerializer.SerializeToElement("Nexus.Sources.Test"), + ["type"] = JsonSerializer.SerializeToElement(extensionName), ["resourceLocator"] = JsonSerializer.SerializeToElement("file:///" + Path.Combine(Directory.GetCurrentDirectory(), "TESTDATA")) }, RequestConfiguration: default diff --git a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs index 22609a6..2a7771c 100644 --- a/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs +++ b/tests/Nexus.Sources.Remote.Tests/RemoteTestsFixture.cs @@ -4,16 +4,16 @@ namespace Nexus.Sources.Tests; public class RemoteTestsFixture : IDisposable { - private Process? _buildProcess; + private Process? _buildProcess_dotnet; - private Process? _runProcess; + private Process? _runProcess_dotnet; + + private Process? _runProcess_python; private readonly SemaphoreSlim _semaphoreBuild = new(0, 1); private readonly SemaphoreSlim _semaphoreRun = new(0, 1); - private bool _success; - public RemoteTestsFixture() { Initialize = Task.Run(() => @@ -43,34 +43,36 @@ private async Task RunDotnetAgent() RedirectStandardError = true }; - _buildProcess = new Process + _buildProcess_dotnet = new Process { StartInfo = psi_build, EnableRaisingEvents = true }; - _buildProcess.OutputDataReceived += (sender, e) => + var success = false; + + _buildProcess_dotnet.OutputDataReceived += (sender, e) => { if (e.Data is not null && e.Data.Contains("Build succeeded")) { - _success = true; + success = true; _semaphoreBuild.Release(); } }; - _buildProcess.ErrorDataReceived += (sender, e) => + _buildProcess_dotnet.ErrorDataReceived += (sender, e) => { - _success = false; + success = false; _semaphoreBuild.Release(); }; - _buildProcess.Start(); - _buildProcess.BeginOutputReadLine(); - _buildProcess.BeginErrorReadLine(); + _buildProcess_dotnet.Start(); + _buildProcess_dotnet.BeginOutputReadLine(); + _buildProcess_dotnet.BeginErrorReadLine(); await _semaphoreBuild.WaitAsync(TimeSpan.FromMinutes(1)); - if (!_success) + if (!success) throw new Exception("Unable to build Nexus.Agent."); // Run Nexus.Agent @@ -83,52 +85,52 @@ private async Task RunDotnetAgent() RedirectStandardError = true }; - psi_run.Environment["NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT"] = "60000"; psi_run.Environment["NEXUSAGENT_PATHS__CONFIG"] = "../../../.nexus-agent-dotnet/config"; + psi_run.Environment["NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT"] = "60000"; - _runProcess = new Process + _runProcess_dotnet = new Process { StartInfo = psi_run, EnableRaisingEvents = true }; - _runProcess.OutputDataReceived += (sender, e) => + _runProcess_dotnet.OutputDataReceived += (sender, e) => { - // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + // File.AppendAllText("/home/vincent/Downloads/output2.txt", e.Data + Environment.NewLine); if (e.Data is not null && e.Data.Contains("Now listening on")) { - _success = true; + success = true; _semaphoreRun.Release(); } }; - _runProcess.ErrorDataReceived += (sender, e) => + _runProcess_dotnet.ErrorDataReceived += (sender, e) => { - // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + // File.AppendAllText("/home/vincent/Downloads/error2.txt", e.Data + Environment.NewLine); - var oldSuccess = _success; - _success = false; + var oldSuccess = success; + success = false; if (oldSuccess) _semaphoreRun.Release(); }; - _runProcess.Start(); - _runProcess.BeginOutputReadLine(); - _runProcess.BeginErrorReadLine(); + _runProcess_dotnet.Start(); + _runProcess_dotnet.BeginOutputReadLine(); + _runProcess_dotnet.BeginErrorReadLine(); await _semaphoreRun.WaitAsync(TimeSpan.FromMinutes(1)); - if (!_success) + if (!success) throw new Exception("Unable to launch Nexus.Agent (dotnet)."); } private async Task RunPythonAgent() { - var psi_run = new ProcessStartInfo("/usr/bin/bash") + var psi_run = new ProcessStartInfo("bash") { - Arguments = $@"-c ""source ../../../.venv/bin/activate; fastapi run main.py""", + Arguments = $"-c \"source ../../../.venv/bin/activate; fastapi run main.py\"", WorkingDirectory="../../../../src/agent/python", UseShellExecute = false, RedirectStandardOutput = true, @@ -136,50 +138,56 @@ private async Task RunPythonAgent() }; psi_run.Environment["PYTHONPATH"] = "../../remoting/python"; - psi_run.Environment["NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT"] = "60001"; psi_run.Environment["NEXUSAGENT_PATHS__CONFIG"] = "../../../.nexus-agent-python/config"; + psi_run.Environment["NEXUSAGENT_SYSTEM__JSONRPCLISTENPORT"] = "60001"; - _runProcess = new Process + _runProcess_python = new Process { StartInfo = psi_run, EnableRaisingEvents = true }; - _runProcess.OutputDataReceived += (sender, e) => + var success = false; + + _runProcess_python.OutputDataReceived += (sender, e) => { - File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); + // File.AppendAllText("/home/vincent/Downloads/output.txt", e.Data + Environment.NewLine); if (e.Data is not null && e.Data.Contains("Application startup complete.")) { - _success = true; + success = true; _semaphoreRun.Release(); } }; - _runProcess.ErrorDataReceived += (sender, e) => + _runProcess_python.ErrorDataReceived += (sender, e) => { - File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); + // File.AppendAllText("/home/vincent/Downloads/error.txt", e.Data + Environment.NewLine); - var oldSuccess = _success; - _success = false; + if (e.Data is not null && e.Data.ToLower().Contains("error")) + { + var oldSuccess = success; + success = false; - if (oldSuccess) - _semaphoreRun.Release(); + if (oldSuccess) + _semaphoreRun.Release(); + } }; - _runProcess.Start(); - _runProcess.BeginOutputReadLine(); - _runProcess.BeginErrorReadLine(); + _runProcess_python.Start(); + _runProcess_python.BeginOutputReadLine(); + _runProcess_python.BeginErrorReadLine(); await _semaphoreRun.WaitAsync(TimeSpan.FromMinutes(1)); - if (!_success) + if (!success) throw new Exception("Unable to launch Nexus.Agent (python)."); } public void Dispose() { - _buildProcess?.Kill(); - _runProcess?.Kill(); + _buildProcess_dotnet?.Kill(); + _runProcess_dotnet?.Kill(); + _runProcess_python?.Kill(); } } diff --git a/tests/Nexus.Sources.Remote.Tests/python/v1/test.py b/tests/Nexus.Sources.Remote.Tests/python/v1/test.py index 6aff78a..e33a10a 100644 --- a/tests/Nexus.Sources.Remote.Tests/python/v1/test.py +++ b/tests/Nexus.Sources.Remote.Tests/python/v1/test.py @@ -7,7 +7,8 @@ from nexus_extensibility import (CatalogRegistration, DataSourceContext, IDataSource, LogLevel, NexusDataType, ReadDataHandler, ReadRequest, Representation, - ResourceBuilder, ResourceCatalogBuilder) + ResourceBuilder, ResourceCatalog, + ResourceCatalogBuilder) class Test(IDataSource): @@ -39,9 +40,11 @@ async def get_catalog_registrations(self, path: str): else: return [] - async def get_catalog(self, catalog_id: str): + async def enrich_catalog(self, catalog: ResourceCatalog): - if (catalog_id == "/A/B/C"): + # TODO: return catalog.merge(new_catalog) + + if (catalog.id == "/A/B/C"): representation = Representation(NexusDataType.INT64, timedelta(seconds=1)) @@ -65,7 +68,7 @@ async def get_catalog(self, catalog_id: str): .add_resources([resource1, resource2]) \ .build() - elif (catalog_id == "/D/E/F"): + elif (catalog.id == "/D/E/F"): representation = Representation(NexusDataType.FLOAT64, timedelta(seconds=1)) From 9e519fe9756da1bd04cfd767c648418bd98f71a5 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 21:20:24 +0100 Subject: [PATCH 11/17] Fix imports --- src/agent/python/main.py | 3 +-- src/agent/python/services.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/agent/python/main.py b/src/agent/python/main.py index e370676..e400bfd 100644 --- a/src/agent/python/main.py +++ b/src/agent/python/main.py @@ -3,8 +3,7 @@ import sys from contextlib import asynccontextmanager -from apollo3zehn_package_management._services import (ExtensionHive, - PackageService) +from apollo3zehn_package_management import ExtensionHive, PackageService from fastapi import FastAPI from nexus_extensibility import IDataSource diff --git a/src/agent/python/services.py b/src/agent/python/services.py index 00d0728..7dc9c43 100644 --- a/src/agent/python/services.py +++ b/src/agent/python/services.py @@ -6,8 +6,7 @@ from logging import Logger from typing import Optional -from apollo3zehn_package_management._services import (ExtensionHive, - PackageService) +from apollo3zehn_package_management import ExtensionHive, PackageService from nexus_remoting._remoting import RemoteCommunicator From d3e479e75e62f59fdc86fe81ea5de8b4f316228c Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 21:31:00 +0100 Subject: [PATCH 12/17] Debug CI --- .github/workflows/build-and-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 2aec2b6..9409aee 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -66,6 +66,9 @@ jobs: dotnet publish -c Release -o app /p:GeneratePackage=true src/agent/dotnet/agent.csproj python -m build --wheel --outdir artifacts/package --no-isolation src/remoting/python + - name: Setup upterm session + uses: lhotari/action-upterm@v1 + - name: Test run: | dotnet test -c Release From 9d7c164296bce998a099b805a38940f1100e5060 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 21:32:30 +0100 Subject: [PATCH 13/17] Revert "Debug CI" This reverts commit d3e479e75e62f59fdc86fe81ea5de8b4f316228c. --- .github/workflows/build-and-publish.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 9409aee..2aec2b6 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -66,9 +66,6 @@ jobs: dotnet publish -c Release -o app /p:GeneratePackage=true src/agent/dotnet/agent.csproj python -m build --wheel --outdir artifacts/package --no-isolation src/remoting/python - - name: Setup upterm session - uses: lhotari/action-upterm@v1 - - name: Test run: | dotnet test -c Release From 67b65b9e4bcae195272a10511fea770388bcad5b Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 21:32:48 +0100 Subject: [PATCH 14/17] Clean up --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 2aec2b6..e5003cf 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: "3.9" - name: Create Docker Output Folder run: mkdir --parent artifacts/images From aaa6e3936e1961b47e9ad31148531cc172f824dc Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 23:30:16 +0100 Subject: [PATCH 15/17] Add Dockerfile for python container --- .github/workflows/build-and-publish.yml | 29 ++++++++++++++++++------- .vscode/launch.json | 2 +- requirements.txt | 4 ++-- src/agent/python/Dockerfile | 14 ++++++++++++ src/agent/python/requirements.txt | 3 +++ 5 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 src/agent/python/Dockerfile create mode 100644 src/agent/python/requirements.txt diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index e5003cf..ab2e2b7 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -74,8 +74,11 @@ jobs: - name: Docker Build run: | - docker build -t nexus-main/nexus-agent:v_next -f src/agent/dotnet/Dockerfile . - docker save --output artifacts/images/nexus_agent_image.tar nexus-main/nexus-agent:v_next + docker build -t nexus-main/nexus-agent-dotnet:v_next -f src/agent/dotnet/Dockerfile . + docker save --output artifacts/images/nexus_agent_dotnet_image.tar nexus-main/nexus-agent-dotnet:v_next + + docker build -t nexus-main/nexus-agent-python:v_next -f src/agent/dotnet/Dockerfile . + docker save --output artifacts/images/nexus_agent_python_image.tar nexus-main/nexus-agent-python:v_next - name: Upload Artifacts uses: actions/upload-artifact@v3 @@ -107,7 +110,9 @@ jobs: path: artifacts - name: Docker Load Image - run: docker load --input artifacts/images/nexus_agent_image.tar + run: | + docker load --input artifacts/images/nexus_agent_dotnet_image.tar + docker load --input artifacts/images/nexus_agent_python_image.tar - name: Nuget package (MyGet) run: dotnet nuget push 'artifacts/package/release/*.nupkg' --api-key ${MYGET_API_KEY} --source https://www.myget.org/F/apollo3zehn-dev/api/v3/index.json @@ -128,8 +133,11 @@ jobs: - name: Docker Push run: | - docker tag nexus-main/nexus-agent:v_next ghcr.io/nexus-main/nexus-agent:${{ needs.build.outputs.version }} - docker push ghcr.io/nexus-main/nexus-agent:${{ needs.build.outputs.version }} + docker tag nexus-main/nexus-agent-dotnet:v_next ghcr.io/nexus-main/nexus-agent-dotnet:${{ needs.build.outputs.version }} + docker push ghcr.io/nexus-main/nexus-agent-dotnet:${{ needs.build.outputs.version }} + + docker tag nexus-main/nexus-agent-python:v_next ghcr.io/nexus-main/nexus-agent-python:${{ needs.build.outputs.version }} + docker push ghcr.io/nexus-main/nexus-agent-python:${{ needs.build.outputs.version }} publish_release: @@ -152,7 +160,9 @@ jobs: path: artifacts - name: Docker Load Image - run: docker load --input artifacts/images/nexus_agent_image.tar + run: | + docker load --input artifacts/images/nexus_agent_dotnet_image.tar + docker load --input artifacts/images/nexus_agent_python_image.tar - name: GitHub Release Artifacts uses: softprops/action-gh-release@v1 @@ -177,5 +187,8 @@ jobs: - name: Docker Push run: | - docker tag nexus-main/nexus-agent:v_next nexusmain/nexus-agent:${{ needs.build.outputs.version }} - docker push nexusmain/nexus-agent:${{ needs.build.outputs.version }} \ No newline at end of file + docker tag nexus-main/nexus-agent-dotnet:v_next nexusmain/nexus-agent-dotnet:${{ needs.build.outputs.version }} + docker push nexusmain/nexus-agent-dotnet:${{ needs.build.outputs.version }} + + docker tag nexus-main/nexus-agent-python:v_next nexusmain/nexus-agent-python:${{ needs.build.outputs.version }} + docker push nexusmain/nexus-agent-python:${{ needs.build.outputs.version }} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 2ef5f52..de39c8a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -21,7 +21,7 @@ "type": "debugpy", "request": "launch", "module": "fastapi", - "args": ["run"], + "args": ["run", "main.py"], "cwd": "${workspaceFolder}/src/agent/python", "env": { "PYTHONPATH": "${workspaceFolder}/src/remoting/python", diff --git a/requirements.txt b/requirements.txt index 00cfd00..190f567 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -apollo3zehn-package-management==1.0.0-b1019 +# keep versions in sync with src/agent/python/requirements.txt +apollo3zehn-package-management==1.0.0-b1 build fastapi[standard] -nexus-api==2.0.0-beta.42 nexus-extensibility==2.0.0-beta.42 pytest-asyncio wheel \ No newline at end of file diff --git a/src/agent/python/Dockerfile b/src/agent/python/Dockerfile new file mode 100644 index 0000000..f13f5dc --- /dev/null +++ b/src/agent/python/Dockerfile @@ -0,0 +1,14 @@ +FROM docker.io/python:3.12-slim + +COPY src/remoting/python /nexus_remoting +COPY src/agent/python /app + +WORKDIR "/app" + +RUN useradd -u 1654 app &&\ + pip install -r requirements.txt + +USER app +ENV PYTHONPATH /nexus_remoting + +CMD ["fastapi", "run", "main.py"] \ No newline at end of file diff --git a/src/agent/python/requirements.txt b/src/agent/python/requirements.txt new file mode 100644 index 0000000..a4b2f80 --- /dev/null +++ b/src/agent/python/requirements.txt @@ -0,0 +1,3 @@ +apollo3zehn-package-management==1.0.0-b1 +fastapi[standard] +nexus-extensibility==2.0.0-beta.42 \ No newline at end of file From f11791bb44e1f74f27e079426ca920ae1094e4a1 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 23:36:42 +0100 Subject: [PATCH 16/17] Fix Python version --- .github/workflows/build-and-publish.yml | 2 +- src/remoting/python/nexus_remoting/_encoder.py | 3 --- src/remoting/python/setup.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index ab2e2b7..37ed80b 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -47,7 +47,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Create Docker Output Folder run: mkdir --parent artifacts/images diff --git a/src/remoting/python/nexus_remoting/_encoder.py b/src/remoting/python/nexus_remoting/_encoder.py index b044770..16eca35 100644 --- a/src/remoting/python/nexus_remoting/_encoder.py +++ b/src/remoting/python/nexus_remoting/_encoder.py @@ -1,6 +1,3 @@ -# Python <= 3.9 -from __future__ import annotations - import dataclasses import re import typing diff --git a/src/remoting/python/setup.py b/src/remoting/python/setup.py index f7c7dfa..6ad59eb 100644 --- a/src/remoting/python/setup.py +++ b/src/remoting/python/setup.py @@ -45,7 +45,7 @@ package_dir={ "nexus_remoting": os.path.join(source_dir, "nexus_remoting") }, - python_requires=">=3.9", + python_requires=">=3.10", install_requires=[ "nexus-extensibility>=2.0.0b42" ] From 514873c53b43541bfa66cf325951c86f9b614f31 Mon Sep 17 00:00:00 2001 From: Vincent Wilms Date: Tue, 21 Jan 2025 23:46:15 +0100 Subject: [PATCH 17/17] Prepare release --- CHANGELOG.md | 4 ++++ version.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecde93e..cb5678e 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.0.0-beta.40 - 2025-01-21 + +- Release nexus-agent-dotnet and nexus-agent-python. + ## v2.0.0-beta.39 - 2025-01-08 - Release Nexus.Agent. diff --git a/version.json b/version.json index eaf8289..337a0bd 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { "version": "2.0.0", - "suffix": "beta.39" + "suffix": "beta.40" } \ No newline at end of file