test_sanity fails

[piotrm-nvidia]

root@ulalegionbuntu:/workspace/examples/hello_world# pytest tests/test_sanity.py 
/usr/local/lib/python3.12/dist-packages/pytest_asyncio/plugin.py:207: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
The event loop scope for asynchronous fixtures will default to the fixture caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to function scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: "function", "class", "module", "package", "session"

  warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
================================== test session starts ==================================
platform linux -- Python 3.12.3, pytest-8.3.4, pluggy-1.5.0
benchmark: 5.1.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /workspace
configfile: pyproject.toml
plugins: anyio-4.8.0, mock-3.14.0, timeout-2.3.1, asyncio-0.25.2, benchmark-5.1.0, cov-6.0.0, md-report-0.6.3, mypy-0.10.3
asyncio: mode=Mode.AUTO, asyncio_default_fixture_loop_scope=None
collected 3 items                                                                       

tests/test_sanity.py ..F                                                          [100%]

======================================= FAILURES ========================================
______________________________________ test_sanity ______________________________________

    def test_sanity():
        deployment_command = [
            "python3",
            "-m",
            "hello_world.deploy",
            "--initialize-request-plane",
        ]
    
        deployment_process = subprocess.Popen(
            deployment_command,
            stdin=subprocess.DEVNULL,
        )
    
        client_command = [
            "python3",
            "-m",
            "hello_world.client",
            "--requests-per-client",
            "10",
        ]
    
        client_process = subprocess.Popen(
            client_command,
            stdin=subprocess.DEVNULL,
        )
        try:
            client_process.wait(timeout=30)
        except subprocess.TimeoutExpired:
            print("Client timed out!")
            client_process.terminate()
            client_process.wait()
    
        client_process.terminate()
        client_process.kill()
        client_process.wait()
        deployment_process.terminate()
        deployment_process.wait()
>       assert client_process.returncode == 0, "Error in clients!"
E       AssertionError: Error in clients!
E       assert 241 == 0
E        +  where 241 = <Popen: returncode: 241 args: ['python3', '-m', 'hello_world.client', '--req...>.returncode

client_command = ['python3', '-m', 'hello_world.client', '--requests-per-client', '10']
client_process = <Popen: returncode: 241 args: ['python3', '-m', 'hello_world.client', '--req...>
deployment_command = ['python3', '-m', 'hello_world.deploy', '--initialize-request-plane']
deployment_process = <Popen: returncode: 1 args: ['python3', '-m', 'hello_world.deploy', '--initi...>

tests/test_sanity.py:63: AssertionError
--------------------------------- Captured stdout call ----------------------------------
Starting Workers
20:29:13 deployment.py:115[triton_distributed.worker.deployment] INFO: 

Starting Worker:

	Config:
	WorkerConfig(request_plane=<class 'triton_distributed.icp.nats_request_plane.NatsRequestPlane'>,
             data_plane=<function UcpDataPlane at 0x7feac4838220>,
             request_plane_args=(['nats://localhost:4223'], {}),
             data_plane_args=([], {}),
             log_level=1,
             operators=[OperatorConfig(name='encoder',
                                       implementation=<class 'triton_distributed.worker.triton_core_operator.TritonCoreOperator'>,
                                       repository='/workspace/examples/hello_world/operators/triton_core_models',
                                       version=1,
                                       max_inflight_requests=1,
                                       parameters={'config': {'instance_group': [{'count': 1,
                                                                                  'kind': 'KIND_CPU'}],
                                                              'parameters': {'delay': {'string_value': '0'},
                                                                             'input_copies': {'string_value': '1'}}}},
                                       log_level=None)],
             triton_log_path=None,
             name='encoder.0',
             log_dir='/workspace/examples/hello_world/logs',
             metrics_port=50000)
	<SpawnProcess name='encoder.0' parent=1172 initial>

20:29:13 deployment.py:115[triton_distributed.worker.deployment] INFO: 

Starting Worker:

	Config:
	WorkerConfig(request_plane=<class 'triton_distributed.icp.nats_request_plane.NatsRequestPlane'>,
             data_plane=<function UcpDataPlane at 0x7feac4838220>,
             request_plane_args=(['nats://localhost:4223'], {}),
             data_plane_args=([], {}),
             log_level=1,
             operators=[OperatorConfig(name='decoder',
                                       implementation=<class 'triton_distributed.worker.triton_core_operator.TritonCoreOperator'>,
                                       repository='/workspace/examples/hello_world/operators/triton_core_models',
                                       version=1,
                                       max_inflight_requests=1,
                                       parameters={'config': {'instance_group': [{'count': 1,
                                                                                  'kind': 'KIND_CPU'}],
                                                              'parameters': {'delay': {'string_value': '0'},
                                                                             'input_copies': {'string_value': '1'}}}},
                                       log_level=None)],
             triton_log_path=None,
             name='decoder.0',
             log_dir='/workspace/examples/hello_world/logs',
             metrics_port=50001)
	<SpawnProcess name='decoder.0' parent=1172 initial>

20:29:13 deployment.py:115[triton_distributed.worker.deployment] INFO: 

Starting Worker:

	Config:
	WorkerConfig(request_plane=<class 'triton_distributed.icp.nats_request_plane.NatsRequestPlane'>,
             data_plane=<function UcpDataPlane at 0x7feac4838220>,
             request_plane_args=(['nats://localhost:4223'], {}),
             data_plane_args=([], {}),
             log_level=1,
             operators=[OperatorConfig(name='encoder_decoder',
                                       implementation='EncodeDecodeOperator',
                                       repository='/workspace/examples/hello_world/operators',
                                       version=1,
                                       max_inflight_requests=1,
                                       parameters={},
                                       log_level=None)],
             triton_log_path=None,
             name='encoder_decoder.0',
             log_dir='/workspace/examples/hello_world/logs',
             metrics_port=50002)
	<SpawnProcess name='encoder_decoder.0' parent=1172 initial>

Client timed out!
Stopping Clients
Clients Stopped Exit Code -15
Workers started ... press Ctrl-C to Exit
Stopping Workers
20:29:42 deployment.py:128[triton_distributed.worker.deployment] INFO: 

Stopping Worker:


	<SpawnProcess name='encoder.0' pid=1248 parent=1172 started>

20:29:42 deployment.py:128[triton_distributed.worker.deployment] INFO: 

Stopping Worker:


	<SpawnProcess name='decoder.0' pid=1249 parent=1172 started>

20:29:42 deployment.py:128[triton_distributed.worker.deployment] INFO: 

Stopping Worker:


	<SpawnProcess name='encoder_decoder.0' pid=1253 parent=1172 started>

20:29:46 deployment.py:137[triton_distributed.worker.deployment] INFO: 

Worker Stopped:


	<SpawnProcess name='encoder.0' pid=1248 parent=1172 stopped exitcode=0>

20:29:46 deployment.py:137[triton_distributed.worker.deployment] INFO: 

Worker Stopped:


	<SpawnProcess name='decoder.0' pid=1249 parent=1172 stopped exitcode=0>

20:29:46 deployment.py:137[triton_distributed.worker.deployment] INFO: 

Worker Stopped:


	<SpawnProcess name='encoder_decoder.0' pid=1253 parent=1172 stopped exitcode=1>

Workers Stopped Exit Code 1
--------------------------------- Captured stderr call ----------------------------------
Client: 0:   0%|          | 0/10 [00:00<?, ?request/s]
========================================= mypy ==========================================
Success: no issues found in 1 source file
================================ short test summary info ================================
FAILED tests/test_sanity.py::test_sanity - AssertionError: Error in clients!
============================= 1 failed, 2 passed in 34.29s ==============================

[tanmayv25]

It does pass on my machine.

root@859ebeb-lcedt:/workspace/examples/hello_world# pytest tests/test_sanity.py
/usr/local/lib/python3.12/dist-packages/pytest_asyncio/plugin.py:207: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
The event loop scope for asynchronous fixtures will default to the fixture caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to function scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: "function", "class", "module", "package", "session"

  warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
============================================================== test session starts ==============================================================
platform linux -- Python 3.12.3, pytest-8.3.4, pluggy-1.5.0
benchmark: 5.1.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /workspace
configfile: pyproject.toml
plugins: mock-3.14.0, timeout-2.3.1, benchmark-5.1.0, cov-6.0.0, asyncio-0.25.2, anyio-4.8.0, mypy-0.10.3, md-report-0.6.3
asyncio: mode=Mode.AUTO, asyncio_default_fixture_loop_scope=None
collected 3 items                                                                                                                               

tests/test_sanity.py ...                                                                                                                  [100%]
===================================================================== mypy ======================================================================

Success: no issues found in 1 source file
============================================================== 3 passed in 12.55s ===============================================================
root@859ebeb-lcedt:/workspace/examples/hello_world#

The client seems to be timing out in your runs.