Update ragger version and use OCR

.github/workflows/ci-workflow.yml

@@ -79,7 +79,7 @@ jobs:
         run: |
           pip install --extra-index-url https://test.pypi.org/simple/ -r tests/python/requirements.txt
       - name: Run tests
-        run: pytest tests/python/ -v
+        run: pytest tests/python/ -v --tb=short --device all
 
   job_C_test:
     name: C tests

.gitignore

@@ -18,3 +18,4 @@ __pycache__
 *~
 
 tests/elfs/
+tests/python/snapshots-tmp/

tests/python/apps/solana.py

@@ -2,9 +2,8 @@
 from enum import IntEnum
 from contextlib import contextmanager
 
-from ragger.backend.interface import BackendInterface, RAPDU
-
-from ..utils import validate_displayed_message
+from ragger.backend import BackendInterface
+from ragger.utils import RAPDU
 
 
 class INS(IntEnum):
@@ -125,14 +124,6 @@ def send_async_sign_message(self,
             yield
 
 
-    def validate_sign_message(self):
-        validate_displayed_message(self._client, 3)
-
-
-    def refuse_to_sign_message(self):
-        validate_displayed_message(self._client, 3 + 1)
-
-
     def get_async_response(self) -> RAPDU:
         return self._client.last_async_response
 

tests/python/apps/solana_utils.py

@@ -1,6 +1,7 @@
 import base58
 
-from ..utils import create_currency_config, pack_derivation_path
+from ragger.bip import pack_derivation_path
+from ragger.utils import create_currency_config
 
 ### Some utilities functions for amounts conversions ###
 

tests/python/conftest.py

@@ -1,90 +1,151 @@
-from dataclasses import dataclass
-from pathlib import Path
 import pytest
-
+from typing import Optional
+from pathlib import Path
 from ragger.firmware import Firmware
 from ragger.backend import SpeculosBackend, LedgerCommBackend, LedgerWalletBackend
-
-from .utils import app_path_from_app_name
-
-def __str__(self):        # also tried __repr__()
-        # Attempt to print the 'select' attribute in "pytest -v" output
-        return self.select
+from ragger.navigator import NanoNavigator
+from ragger.utils import app_path_from_app_name
 
 
+# This variable is needed for Speculos only (physical tests need the application to be already installed)
+# Adapt this path to your 'tests/elfs' directory
 APPS_DIRECTORY = (Path(__file__).parent.parent / "elfs").resolve()
 
+# Adapt this name part of the compiled app <name>_<device>.elf in the APPS_DIRECTORY
 APP_NAME = "solana"
 
 BACKENDS = ["speculos", "ledgercomm", "ledgerwallet"]
 
+DEVICES = ["nanos", "nanox", "nanosp", "all"]
+
 FIRMWARES = [Firmware('nanos', '2.1'),
              Firmware('nanox', '2.0.2'),
              Firmware('nanosp', '1.0.3')]
 
+
 def pytest_addoption(parser):
-    parser.addoption("--backend", action="store", default="speculos")
-    # Enable using --'device' in the pytest command line to restrict testing to specific devices
-    for fw in FIRMWARES:
-        parser.addoption("--"+fw.device, action="store_true", help="run on nanos only")
+    parser.addoption("--device", choices=DEVICES, required=True)
+    parser.addoption("--backend", choices=BACKENDS, default="speculos")
+    parser.addoption("--display", action="store_true", default=False)
+    parser.addoption("--golden_run", action="store_true", default=False)
+    parser.addoption("--log_apdu_file", action="store", default=None)
+
+
+@pytest.fixture(scope="session")
+def backend_name(pytestconfig):
+    return pytestconfig.getoption("backend")
+
+
+@pytest.fixture(scope="session")
+def display(pytestconfig):
+    return pytestconfig.getoption("display")
+
+
+@pytest.fixture(scope="session")
+def golden_run(pytestconfig):
+    return pytestconfig.getoption("golden_run")
+
+
+@pytest.fixture(scope="session")
+def log_apdu_file(pytestconfig):
+    filename = pytestconfig.getoption("log_apdu_file")
+    return Path(filename).resolve() if filename is not None else None
+
+
+@pytest.fixture
+def test_name(request):
+    # Get the name of current pytest test
+    test_name = request.node.name
+
+    # Remove firmware suffix:
+    # -  test_xxx_transaction_ok[nanox 2.0.2]
+    # => test_xxx_transaction_ok
+    return test_name.split("[")[0]
 
 
 # Glue to call every test that depends on the firmware once for each required firmware
 def pytest_generate_tests(metafunc):
     if "firmware" in metafunc.fixturenames:
         fw_list = []
         ids = []
-        # First pass: enable only demanded firmwares
-        for fw in FIRMWARES:
-            if metafunc.config.getoption(fw.device):
-                fw_list.append(fw)
-                ids.append(fw.device + " " + fw.version)
-        # Second pass if no specific firmware demanded: add them all
-        if not fw_list:
+
+        device = metafunc.config.getoption("device")
+        backend_name = metafunc.config.getoption("backend")
+
+        if device == "all":
+            if backend_name != "speculos":
+                raise ValueError("Invalid device parameter on this backend")
+
+            # Add all supported firmwares
             for fw in FIRMWARES:
                 fw_list.append(fw)
                 ids.append(fw.device + " " + fw.version)
-        metafunc.parametrize("firmware", fw_list, ids=ids)
 
-def prepare_speculos_args(firmware):
+        else:
+            # Enable firmware for demanded device
+            for fw in FIRMWARES:
+                if device == fw.device:
+                    fw_list.append(fw)
+                    ids.append(fw.device + " " + fw.version)
+
+        metafunc.parametrize("firmware", fw_list, ids=ids, scope="session")
+
+
+def prepare_speculos_args(firmware: Firmware, display: bool):
     speculos_args = []
-    # Uncomment line below to enable display
-    # speculos_args += ["--display", "qt"]
+
+    if display:
+        speculos_args += ["--display", "qt"]
 
     app_path = app_path_from_app_name(APPS_DIRECTORY, APP_NAME, firmware.device)
 
     return ([app_path], {"args": speculos_args})
 
 
-@pytest.fixture(scope="session")
-def backend(pytestconfig):
-    return pytestconfig.getoption("backend")
+# Depending on the "--backend" option value, a different backend is
+# instantiated, and the tests will either run on Speculos or on a physical
+# device depending on the backend
+def create_backend(backend_name: str, firmware: Firmware, display: bool, log_apdu_file: Optional[Path]):
+    if backend_name.lower() == "ledgercomm":
+        return LedgerCommBackend(firmware=firmware, interface="hid", log_apdu_file=log_apdu_file)
+    elif backend_name.lower() == "ledgerwallet":
+        return LedgerWalletBackend(firmware=firmware, log_apdu_file=log_apdu_file)
+    elif backend_name.lower() == "speculos":
+        args, kwargs = prepare_speculos_args(firmware, display)
+        return SpeculosBackend(*args, firmware=firmware, log_apdu_file=log_apdu_file, **kwargs)
+    else:
+        raise ValueError(f"Backend '{backend_name}' is unknown. Valid backends are: {BACKENDS}")
+
+
+# This fixture will return the properly configured backend, to be used in tests.
+# As Speculos instantiation takes some time, this fixture scope is by default "session".
+# If your tests needs to be run on independent Speculos instances (in case they affect
+# settings for example), then you should change this fixture scope and choose between
+# function, class, module or session.
+# @pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
+def backend(backend_name, firmware, display, log_apdu_file):
+    with create_backend(backend_name, firmware, display, log_apdu_file) as b:
+        yield b
 
 
-def create_backend(backend: str, firmware: Firmware):
-    if backend.lower() == "ledgercomm":
-        return LedgerCommBackend(firmware, interface="hid")
-    elif backend.lower() == "ledgerwallet":
-        return LedgerWalletBackend(firmware)
-    elif backend.lower() == "speculos":
-        args, kwargs = prepare_speculos_args(firmware)
-        return SpeculosBackend(*args, firmware, **kwargs)
+@pytest.fixture
+def navigator(backend, firmware, golden_run):
+    if firmware.device.startswith("nano"):
+        return NanoNavigator(backend, firmware, golden_run)
     else:
-        raise ValueError(f"Backend '{backend}' is unknown. Valid backends are: {BACKENDS}")
+        raise ValueError(f"Device '{firmware.device}' is unsupported.")
 
-@pytest.fixture
-def client(backend, firmware):
-    with create_backend(backend, firmware) as b:
-        yield b
 
 @pytest.fixture(autouse=True)
 def use_only_on_backend(request, backend):
     if request.node.get_closest_marker('use_on_backend'):
         current_backend = request.node.get_closest_marker('use_on_backend').args[0]
         if current_backend != backend:
-            pytest.skip('skipped on this backend: {}'.format(current_backend))
+            pytest.skip(f'skipped on this backend: "{current_backend}"')
+
 
 def pytest_configure(config):
-  config.addinivalue_line(
+    config.addinivalue_line(
         "markers", "use_only_on_backend(backend): skip test if not on the specified backend",
-  )
+    )

tests/python/requirements.txt

@@ -1,3 +1,2 @@
 base58
-bip32
 ragger[tests,speculos]

tests/python/test_solana.py

@@ -1,72 +1,80 @@
-from time import sleep
-import base58
-
-from ragger.backend.interface import RAPDU, RaisePolicy
+from ragger.backend import RaisePolicy
+from ragger.navigator import NavInsID
+from ragger.utils import RAPDU
 
 from .apps.solana import SolanaClient, ErrorType
 from .apps.solana_cmd_builder import SystemInstructionTransfer, Message, verify_signature
 from .apps.solana_utils import FOREIGN_PUBLIC_KEY, FOREIGN_PUBLIC_KEY_2, AMOUNT, AMOUNT_2, SOL_PACKED_DERIVATION_PATH, SOL_PACKED_DERIVATION_PATH_2
 
-def test_solana_simple_transfer_ok_1(client, firmware):
-    sol = SolanaClient(client)
+from .utils import ROOT_SCREENSHOT_PATH
+
+def test_solana_simple_transfer_ok_1(backend, navigator, test_name):
+    sol = SolanaClient(backend)
     from_public_key = sol.get_public_key(SOL_PACKED_DERIVATION_PATH)
 
     # Create instruction
     instruction: SystemInstructionTransfer = SystemInstructionTransfer(from_public_key, FOREIGN_PUBLIC_KEY, AMOUNT)
     message: bytes = Message([instruction]).serialize()
 
     with sol.send_async_sign_message(SOL_PACKED_DERIVATION_PATH, message):
-        sol.validate_sign_message()
+        navigator.navigate_until_text_and_compare(NavInsID.RIGHT_CLICK,
+                                                  [NavInsID.BOTH_CLICK],
+                                                  "Approve",
+                                                  ROOT_SCREENSHOT_PATH,
+                                                  test_name)
+
     signature: bytes = sol.get_async_response().data
 
     verify_signature(from_public_key, message, signature)
 
-    sleep(0.1)
-
 
-def test_solana_simple_transfer_ok_2(client, firmware):
-    sol = SolanaClient(client)
+def test_solana_simple_transfer_ok_2(backend, navigator, test_name):
+    sol = SolanaClient(backend)
     from_public_key = sol.get_public_key(SOL_PACKED_DERIVATION_PATH_2)
 
     # Create instruction
     instruction: SystemInstructionTransfer = SystemInstructionTransfer(from_public_key, FOREIGN_PUBLIC_KEY_2, AMOUNT_2)
     message: bytes = Message([instruction]).serialize()
 
     with sol.send_async_sign_message(SOL_PACKED_DERIVATION_PATH_2, message):
-        sol.validate_sign_message()
+        navigator.navigate_until_text_and_compare(NavInsID.RIGHT_CLICK,
+                                                  [NavInsID.BOTH_CLICK],
+                                                  "Approve",
+                                                  ROOT_SCREENSHOT_PATH,
+                                                  test_name)
+
     signature: bytes = sol.get_async_response().data
 
     verify_signature(from_public_key, message, signature)
 
-    sleep(0.1)
-
 
-def test_solana_simple_transfer_refused(client, firmware):
-    sol = SolanaClient(client)
+def test_solana_simple_transfer_refused(backend, navigator, test_name):
+    sol = SolanaClient(backend)
     from_public_key = sol.get_public_key(SOL_PACKED_DERIVATION_PATH)
 
     instruction: SystemInstructionTransfer = SystemInstructionTransfer(from_public_key, FOREIGN_PUBLIC_KEY, AMOUNT)
     message: bytes = Message([instruction]).serialize()
 
+    backend.raise_policy = RaisePolicy.RAISE_NOTHING
     with sol.send_async_sign_message(SOL_PACKED_DERIVATION_PATH, message):
-        client.raise_policy = RaisePolicy.RAISE_NOTHING
-        sol.refuse_to_sign_message()
+        navigator.navigate_until_text_and_compare(NavInsID.RIGHT_CLICK,
+                                                  [NavInsID.BOTH_CLICK],
+                                                  "Reject",
+                                                  ROOT_SCREENSHOT_PATH,
+                                                  test_name)
 
     rapdu: RAPDU = sol.get_async_response()
     assert rapdu.status == ErrorType.USER_CANCEL
 
-    sleep(0.1)
-
 
-def test_solana_blind_sign_refused(client, firmware):
-    sol = SolanaClient(client)
+def test_solana_blind_sign_refused(backend):
+    sol = SolanaClient(backend)
     from_public_key = sol.get_public_key(SOL_PACKED_DERIVATION_PATH)
 
     instruction: SystemInstructionTransfer = SystemInstructionTransfer(from_public_key, FOREIGN_PUBLIC_KEY, AMOUNT)
     message: bytes = Message([instruction]).serialize()
 
-    client.raise_policy = RaisePolicy.RAISE_NOTHING
+    backend.raise_policy = RaisePolicy.RAISE_NOTHING
     rapdu: RAPDU = sol.send_blind_sign_message(SOL_PACKED_DERIVATION_PATH, message)
     assert rapdu.status == ErrorType.SDK_NOT_SUPPORTED
 
-    sleep(0.1)