Merge branch 'main' into erik/lsps-2-server-rpc-methods

Makefile

@@ -57,14 +57,14 @@ gen: ${GENALL}
 build-self: ensure-docker
 	(cd libs; cargo build --all)
 	(cd libs/gl-client-py && python3 -m maturin develop)
+	pip install -e libs/gl-testing
 
 check-all: check-self check-self-gl-client check-py
 
 check-self: ensure-docker
 	PYTHONPATH=/repo/libs/gl-testing \
 	pytest -vvv \
 	  /repo/libs/gl-testing \
-	  --force-flaky \
 	  ${PYTEST_OPTS}
 
 ensure-docker:

libs/gl-client-py/Cargo.toml

@@ -12,12 +12,17 @@ crate-type = ["cdylib"]
 anyhow = { workspace = true }
 bytes = "1.4"
 env_logger = { workspace = true }
-gl-client = { path = "../gl-client", features = ["permissive"] }
+
+gl-client = { path = "../gl-client", default-features = false, features = [ "export" ] }
 hex = "*"
 log = "*"
 once_cell = "*"
 prost = "0.11"
 pyo3 = {version = "0.18", features = ["extension-module", "serde", "abi3-py37"]}
 tokio = { version = "1", features = ["full"] }
 tonic = { version = "^0.8", features = ["tls", "transport"] }
-serde_json = "^1.0"
+serde_json = "^1.0"
+
+[features]
+default = ["permissive"]
+permissive = ["gl-client/permissive"]

libs/gl-client-py/Makefile

@@ -36,6 +36,8 @@ ${PYPROTOS}: ${PROTOSRC}
 	python -m grpc_tools.protoc ${PYPROTOC_OPTS} scheduler.proto
 	python -m grpc_tools.protoc ${PYPROTOC_OPTS} greenlight.proto
 	sed -i 's/import scheduler_pb2 as scheduler__pb2/from . import scheduler_pb2 as scheduler__pb2/g' ${PYDIR}/glclient/scheduler_pb2_grpc.py
+	sed -i 's/import greenlight_pb2 as greenlight__pb2/from . import greenlight_pb2 as greenlight__pb2/g' ${PYDIR}/glclient/scheduler_pb2_grpc.py
+	sed -i 's/import greenlight_pb2 as greenlight__pb2/from . import greenlight_pb2 as greenlight__pb2/g' ${PYDIR}/glclient/scheduler_pb2.py
 	sed -i 's/import greenlight_pb2 as greenlight__pb2/from . import greenlight_pb2 as greenlight__pb2/g' ${PYDIR}/glclient/greenlight_pb2_grpc.py
 
 check-py:

libs/gl-client-py/glclient/scheduler_pb2.pyi

@@ -8,6 +8,7 @@ import google.protobuf.descriptor
 import google.protobuf.internal.containers
 import google.protobuf.internal.enum_type_wrapper
 import google.protobuf.message
+import greenlight_pb2
 import sys
 import typing
 
@@ -427,3 +428,24 @@ class ExportNodeResponse(google.protobuf.message.Message):
     def ClearField(self, field_name: typing_extensions.Literal["url", b"url"]) -> None: ...
 
 global___ExportNodeResponse = ExportNodeResponse
+
+@typing_extensions.final
+class SignerRejection(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    MSG_FIELD_NUMBER: builtins.int
+    REQUEST_FIELD_NUMBER: builtins.int
+    msg: builtins.str
+    """A human-readable description of what went wrong"""
+    @property
+    def request(self) -> greenlight_pb2.HsmRequest: ...
+    def __init__(
+        self,
+        *,
+        msg: builtins.str = ...,
+        request: greenlight_pb2.HsmRequest | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["request", b"request"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["msg", b"msg", "request", b"request"]) -> None: ...
+
+global___SignerRejection = SignerRejection

libs/gl-client-py/glclient/scheduler_pb2_grpc.py

@@ -2,6 +2,7 @@
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
 
+from . import greenlight_pb2 as greenlight__pb2
 from . import scheduler_pb2 as scheduler__pb2
 
 
@@ -490,3 +491,74 @@ def ExportNode(request,
             scheduler__pb2.ExportNodeResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+
+class DebugStub(object):
+    """A service to collect debugging information from clients.
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.ReportSignerRejection = channel.unary_unary(
+                '/scheduler.Debug/ReportSignerRejection',
+                request_serializer=scheduler__pb2.SignerRejection.SerializeToString,
+                response_deserializer=greenlight__pb2.Empty.FromString,
+                )
+
+
+class DebugServicer(object):
+    """A service to collect debugging information from clients.
+    """
+
+    def ReportSignerRejection(self, request, context):
+        """The signer is designed to fail closed, i.e., we reject requests
+        that do not resolve or that go against one of its policies. This
+        comes with some issues, such as false negatives, where we reject
+        despite the request being valid. As more apps use the API we need
+        to debug these false negatives, hence why we report rejections,
+        so we can investigate the validity of the rejection, and to
+        fine-tine the signer's ruleset.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_DebugServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'ReportSignerRejection': grpc.unary_unary_rpc_method_handler(
+                    servicer.ReportSignerRejection,
+                    request_deserializer=scheduler__pb2.SignerRejection.FromString,
+                    response_serializer=greenlight__pb2.Empty.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'scheduler.Debug', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Debug(object):
+    """A service to collect debugging information from clients.
+    """
+
+    @staticmethod
+    def ReportSignerRejection(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/scheduler.Debug/ReportSignerRejection',
+            scheduler__pb2.SignerRejection.SerializeToString,
+            greenlight__pb2.Empty.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

libs/gl-client-py/pyproject.toml

@@ -31,6 +31,7 @@ pyln-grpc-proto = "^0.1"
 # have an issue with TLS (TSI corruption).
 grpcio-tools = ">=1.56"
 protobuf = ">=3"
+mypy-protobuf = "^3.5.0"
 
 [build-system]
 requires = ["maturin>=0.11,<0.12"]

libs/gl-client/src/lnurl/pay/mod.rs

@@ -12,27 +12,20 @@ use sha256;
 
 pub async fn resolve_lnurl_to_invoice<T: LnUrlHttpClient>(
     http_client: &T,
-    lnurl: &str,
+    lnurl_identifier: &str,
     amount_msats: u64,
 ) -> Result<String> {
-    let url = parse_lnurl(lnurl)?;
+    let url = match is_lnurl(lnurl_identifier) {
+        true => parse_lnurl(lnurl_identifier)?,
+        false => parse_lightning_address(lnurl_identifier)?,
+    };
+
+    debug!("Domain: {}", Url::parse(&url).unwrap().host().unwrap());
 
     let lnurl_pay_request_response: PayRequestResponse =
         http_client.get_pay_request_response(&url).await?;
 
-    if lnurl_pay_request_response.tag != "payRequest" {
-        return Err(anyhow!("Expected tag to say 'payRequest'"));
-    }
-
-    ensure_amount_is_within_range(&lnurl_pay_request_response, amount_msats)?;
-    let description = extract_description(&lnurl_pay_request_response)?;
-
-    debug!("Domain: {}", Url::parse(&url).unwrap().host().unwrap());
-    debug!("Description: {}", description);
-    debug!(
-        "Accepted range (in millisatoshis): {} - {}",
-        lnurl_pay_request_response.min_sendable, lnurl_pay_request_response.max_sendable
-    );
+    validate_pay_request_response(lnurl_identifier, &lnurl_pay_request_response, amount_msats)?;
 
     let callback_url = build_callback_url(&lnurl_pay_request_response, amount_msats)?;
     let callback_response: PayRequestCallbackResponse = http_client
@@ -48,6 +41,60 @@ pub async fn resolve_lnurl_to_invoice<T: LnUrlHttpClient>(
     Ok(invoice.to_string())
 }
 
+fn is_lnurl(lnurl_identifier: &str) -> bool {
+    const LNURL_PREFIX: &str = "LNURL";
+    lnurl_identifier
+        .trim()
+        .to_uppercase()
+        .starts_with(LNURL_PREFIX)
+}
+
+pub fn validate_pay_request_response(
+    lnurl_identifier: &str,
+    lnurl_pay_request_response: &PayRequestResponse,
+    amount_msats: u64,
+) -> Result<()> {
+    if lnurl_pay_request_response.tag != "payRequest" {
+        return Err(anyhow!("Expected tag to say 'payRequest'"));
+    }
+    ensure_amount_is_within_range(&lnurl_pay_request_response, amount_msats)?;
+    let description = extract_description(&lnurl_pay_request_response)?;
+
+    debug!("Description: {}", description);
+    debug!(
+        "Accepted range (in millisatoshis): {} - {}",
+        lnurl_pay_request_response.min_sendable, lnurl_pay_request_response.max_sendable
+    );
+
+    if !is_lnurl(lnurl_identifier) {
+        let deserialized_metadata: Vec<Vec<String>> =
+            serde_json::from_str(&lnurl_pay_request_response.metadata.to_owned())
+                .map_err(|e| anyhow!("Failed to deserialize metadata: {}", e))?;
+
+        let mut identifier = String::new();
+
+        let metadata_entry_types = ["text/email", "text/identifier"];
+
+        for metadata in deserialized_metadata {
+            let x = &*metadata[0].clone();
+            if metadata_entry_types.contains(&x) {
+                identifier = String::from(metadata[1].clone());
+                break;
+            }
+        }
+
+        if identifier.is_empty() {
+            return Err(anyhow!("Could not find an entry of type "));
+        }
+
+        if identifier != lnurl_identifier {
+            return Err(anyhow!("The lightning address specified in the original request does not match what was found in the metadata array"));
+        }
+    }
+
+    Ok(())
+}
+
 // Validates the invoice on the pay request's callback response
 pub fn validate_invoice_from_callback_response(
     invoice: &Bolt11Invoice,
@@ -65,7 +112,9 @@ pub fn validate_invoice_from_callback_response(
 
     ensure!(
         description_hash == sha256::digest(metadata),
-        "description_hash does not match the hash of the metadata"
+        "description_hash {} does not match the hash of the metadata {}",
+        description_hash,
+        sha256::digest(metadata)
     );
 
     Ok(())
@@ -123,6 +172,40 @@ fn ensure_amount_is_within_range(
     Ok(())
 }
 
+//LUD-16: Paying to static internet identifiers.
+pub fn parse_lightning_address(lightning_address: &str) -> Result<String> {
+    let lightning_address_components: Vec<&str> = lightning_address.split("@").collect();
+
+    if lightning_address_components.len() != 2 {
+        return Err(anyhow!("The provided lightning address is improperly formatted"));
+    }
+
+    let username = match lightning_address_components.get(0) {
+        None => return Err(anyhow!("Could not parse username in lightning address")),
+        Some(u) => {
+            if u.is_empty() {
+                return Err(anyhow!("Username can not be empty"))
+            }
+
+            u
+        }
+    };
+
+    let domain = match lightning_address_components.get(1) {
+        None => return Err(anyhow!("Could not parse domain in lightning address")),
+        Some(d) => {
+            if d.is_empty() {
+                return Err(anyhow!("Domain can not be empty"))
+            }
+
+            d
+        }
+    };
+
+    let pay_request_url = ["https://", domain, "/.well-known/lnurlp/", username].concat();
+    return Ok(pay_request_url);
+}
+
 #[cfg(test)]
 mod tests {
     use crate::lnurl::models::MockLnUrlHttpClient;
@@ -169,7 +252,77 @@ mod tests {
         let lnurl = "LNURL1DP68GURN8GHJ7CMFWP5X2UNSW4HXKTNRDAKJ7CTSDYHHVVF0D3H82UNV9UCSAXQZE2";
         let amount = 100000;
 
-        let _ = resolve_lnurl_to_invoice(&mock_http_client, lnurl, amount).await;
+        let invoice = resolve_lnurl_to_invoice(&mock_http_client, &lnurl, amount).await;
+        assert!(invoice.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_lnurl_pay_with_lightning_address() {
+        let mut mock_http_client = MockLnUrlHttpClient::new();
+        let lightning_address_username = "satoshi";
+        let lightning_address_domain = "cipherpunk.com";
+        let lnurl = format!(
+            "{}@{}",
+            lightning_address_username, lightning_address_domain
+        );
+
+        let lnurl_clone = lnurl.clone();
+        mock_http_client.expect_get_pay_request_response().returning(move |url| {
+            let expected_url = format!("https://{}/.well-known/lnurlp/{}", lightning_address_domain, lightning_address_username);
+            assert_eq!(expected_url, url);
+
+            let pay_request_json = format!("{{\"callback\": \"https://cipherpunk.com/lnurlp/api/v1/lnurl/cb/1\", \"maxSendable\": 100000, \"minSendable\": 100, \"tag\": \"payRequest\", \"metadata\": \"[[\\\"text/plain\\\", \\\"Start the CoinTrain\\\"], [\\\"text/identifier\\\", \\\"{}\\\"]]\" }}", lnurl_clone);
+
+            let x: PayRequestResponse = serde_json::from_str(&pay_request_json).unwrap();
+            convert_to_async_return_value(Ok(x))
+        });
+
+        mock_http_client.expect_get_pay_request_callback_response().returning(|_url| {
+            let invoice = "lnbcrt1u1pj0ypx6sp5hzczugdw9eyw3fcsjkssux7awjlt68vpj7uhmen7sup0hdlrqxaqpp5gp5fm2sn5rua2jlzftkf5h22rxppwgszs7ncm73pmwhvjcttqp3qdy2tddjyar90p6z7urvv95kug3vyq39xarpwf6zqargv5syxmmfde28yctfdc396tpqtv38getcwshkjer9de6xjenfv4ezytpqyfekzar0wd5xjsrrd9cxsetjwp6ku6ewvdhk6gjat5xqyjw5qcqp29qxpqysgqujuf5zavazln2q9gks7nqwdgjypg2qlvv7aqwfmwg7xmjt8hy4hx2ctr5fcspjvmz9x5wvmur8vh6nkynsvateafm73zwg5hkf7xszsqajqwcf";
+            let callback_response_json = format!("{{\"pr\":\"{}\",\"routes\":[]}}", invoice).to_string();
+            let x = serde_json::from_str(&callback_response_json).unwrap();
+            convert_to_async_return_value(Ok(x))
+        });
+
+        let amount = 100000;
+
+        let invoice = resolve_lnurl_to_invoice(&mock_http_client, &lnurl, amount).await;
+        assert!(invoice.is_ok());
+    }
+
+
+    #[tokio::test]
+    async fn test_lnurl_pay_with_lightning_address_fails_with_empty_username() {
+        let mock_http_client = MockLnUrlHttpClient::new();
+        let lightning_address_username = "";
+        let lightning_address_domain = "cipherpunk.com";
+        let lnurl = format!(
+            "{}@{}",
+            lightning_address_username, lightning_address_domain
+        );
+
+        let amount = 100000;
+
+        let error = resolve_lnurl_to_invoice(&mock_http_client, &lnurl, amount).await;
+        assert!(error.is_err());
+        assert!(error.unwrap_err().to_string().contains("Username can not be empty"));
+    }
+
+    #[tokio::test]
+    async fn test_lnurl_pay_with_lightning_address_fails_with_empty_domain() {
+        let mock_http_client = MockLnUrlHttpClient::new();
+        let lightning_address_username = "satoshi";
+        let lightning_address_domain = "";
+        let lnurl = format!(
+            "{}@{}",
+            lightning_address_username, lightning_address_domain
+        );
+
+        let amount = 100000;
+
+        let error = resolve_lnurl_to_invoice(&mock_http_client, &lnurl, amount).await;
+        assert!(error.is_err());
+        assert!(error.unwrap_err().to_string().contains("Domain can not be empty"));
     }
 
     #[tokio::test]