Return raw secrets

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 
+## [3.4.1] - 2024-07-12
+
+### Fixed
+- Returning Raw string instead of parsing. This fixes the case where secret has \\$ in - Python would return \$ - therefore we are calling repr 
 
 ## [3.4.0] - 2024-03-04
 

gestalt/vault.py

@@ -43,8 +43,7 @@ def __init__(
 
         self._vault_client: Optional[hvac.Client] = None
         self._secret_expiry_times: Dict[str, datetime] = dict()
-        self._secret_values: Dict[str, Union[str, int, float, bool,
-                                             List[Any]]] = dict()
+        self._secret_values: Dict[str, Union[str, int, float, bool, List[Any]]] = dict()
         self._is_connected: bool = False
         self._role: Optional[str] = role
         self._jwt: Optional[str] = jwt
@@ -59,10 +58,9 @@ def __init__(
     @property
     def vault_client(self) -> hvac.Client:
         if self._vault_client is None:
-            self._vault_client = hvac.Client(url=self._url,
-                                             token=self._token,
-                                             cert=self._cert,
-                                             verify=self._verify)
+            self._vault_client = hvac.Client(
+                url=self._url, token=self._token, cert=self._cert, verify=self._verify
+            )
         return self._vault_client
 
     def connect(self) -> None:
@@ -80,9 +78,9 @@ def connect(self) -> None:
 
         if self._role and self._jwt:
             try:
-                hvac.api.auth_methods.Kubernetes(
-                    self.vault_client.adapter).login(role=self._role,
-                                                     jwt=self._jwt)
+                hvac.api.auth_methods.Kubernetes(self.vault_client.adapter).login(
+                    role=self._role, jwt=self._jwt
+                )
                 token = retry_call(
                     self.vault_client.auth.token.lookup_self,
                     exceptions=(RuntimeError, Timeout),
@@ -99,21 +97,22 @@ def connect(self) -> None:
                     self.kubes_token_queue.put(kubes_token)
             except hvac.exceptions.InvalidPath:
                 raise RuntimeError(
-                    "Gestalt Error: Kubernetes auth couldn't be performed")
+                    "Gestalt Error: Kubernetes auth couldn't be performed"
+                )
             except requests.exceptions.ConnectionError:
                 raise RuntimeError("Gestalt Error: Couldn't connect to Vault")
 
             dynamic_ttl_renew = Thread(
                 name="dynamic-token-renew",
                 target=self.worker,
                 daemon=True,
-                args=(self.dynamic_token_queue, ),
+                args=(self.dynamic_token_queue,),
             )  # noqa: F841
             kubernetes_ttl_renew = Thread(
                 name="kubes-token-renew",
                 target=self.worker,
                 daemon=True,
-                args=(self.kubes_token_queue, ),
+                args=(self.kubes_token_queue,),
             )
             kubernetes_ttl_renew.start()
         self._is_connected = True
@@ -125,11 +124,7 @@ def __del__(self) -> None:
         self.stop()
 
     def get(
-        self,
-        key: str,
-        path: str,
-        filter: str,
-        sep: Optional[str] = "."
+        self, key: str, path: str, filter: str, sep: Optional[str] = "."
     ) -> Union[str, int, float, bool, List[Any]]:
         """Gets secret from vault
         Args:
@@ -147,8 +142,7 @@ def get(
             return self._secret_values[key]
 
         # if the secret can expire but hasn't expired yet
-        if key in self._secret_expiry_times and not self._is_secret_expired(
-                key):
+        if key in self._secret_expiry_times and not self._is_secret_expired(key):
             return self._secret_values[key]
 
         try:
@@ -171,10 +165,10 @@ def get(
             requested_data = response["data"].get("data", response["data"])
         except hvac.exceptions.InvalidPath:
             raise RuntimeError(
-                "Gestalt Error: The secret path or mount is set incorrectly")
+                "Gestalt Error: The secret path or mount is set incorrectly"
+            )
         except requests.exceptions.ConnectionError:
-            raise RuntimeError(
-                "Gestalt Error: Gestalt couldn't connect to Vault")
+            raise RuntimeError("Gestalt Error: Gestalt couldn't connect to Vault")
         except Exception as err:
             raise RuntimeError(f"Gestalt Error: {err}")
         if filter is None:
@@ -192,20 +186,24 @@ def get(
         if "ttl" in requested_data:
             self._set_secrets_ttl(requested_data, key)
 
-        return returned_value_from_secret  # type: ignore
+        # repr is converting the string to RAW string since \\$ was returning $\
+        # then we are removing single quotes in the end to get final value
+        #
+        return str(repr(returned_value_from_secret)).replace("'", "")  # type: ignore
 
     def _is_secret_expired(self, key: str) -> bool:
         now = datetime.now()
         secret_expires_dt = self._secret_expiry_times[key]
         is_expired = now >= secret_expires_dt
         return is_expired
 
-    def _set_secrets_ttl(self, requested_data: Dict[str, Any],
-                         key: str) -> None:
-        last_vault_rotation_str = requested_data["last_vault_rotation"].split(
-            ".")[0]  # to the nearest second
-        last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str,
-                                                   "%Y-%m-%dT%H:%M:%S")
+    def _set_secrets_ttl(self, requested_data: Dict[str, Any], key: str) -> None:
+        last_vault_rotation_str = requested_data["last_vault_rotation"].split(".")[
+            0
+        ]  # to the nearest second
+        last_vault_rotation_dt = datetime.strptime(
+            last_vault_rotation_str, "%Y-%m-%dT%H:%M:%S"
+        )
         ttl = requested_data["ttl"]
         secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl)
         self._secret_expiry_times[key] = secret_expires_dt
@@ -217,8 +215,7 @@ def worker(self, token_queue: Queue) -> None:  # type: ignore
         try:
             while self._run_worker:
                 if not token_queue.empty():
-                    token_type, token_id, token_duration = token = token_queue.get(
-                    )
+                    token_type, token_id, token_duration = token = token_queue.get()
                     if token_type == "kubernetes":
                         self.vault_client.auth.token.renew(token_id)
                         print("kubernetes token for the app has been renewed")
@@ -230,10 +227,10 @@ def worker(self, token_queue: Queue) -> None:  # type: ignore
                     sleep((token_duration / 3) * 2)
         except hvac.exceptions.InvalidPath:
             raise RuntimeError(
-                "Gestalt Error: The lease path or mount is set incorrectly")
+                "Gestalt Error: The lease path or mount is set incorrectly"
+            )
         except requests.exceptions.ConnectionError:
-            raise RuntimeError(
-                "Gestalt Error: Gestalt couldn't connect to Vault")
+            raise RuntimeError("Gestalt Error: Gestalt couldn't connect to Vault")
         except Exception as err:
             raise RuntimeError(f"Gestalt Error: {err}")
 

setup.py

@@ -12,7 +12,7 @@ def readme():
 
 setup(
     name="gestalt-cfg",
-    version="3.4.0",
+    version="3.4.1",
     description="A sensible configuration library for Python",
     long_description=readme(),
     long_description_content_type="text/markdown",

tests/conftest.py

@@ -8,20 +8,20 @@
 class MockSession(requests.Session):
     def request(self, *_, **__):
         resp = {
-            'request_id': '230f5e67-e55d-bdae-bd24-c7bc13c1a3e9',
-            'lease_id': '',
-            'renewable': False,
-            'lease_duration': 0,
-            'data': {
-                'last_vault_rotation': '2023-05-31T14:24:41.724285249Z',
-                'password': 'foo',
-                'rotation_period': 60,
-                'ttl': 0,
-                'username': 'foo'
+            "request_id": "230f5e67-e55d-bdae-bd24-c7bc13c1a3e9",
+            "lease_id": "",
+            "renewable": False,
+            "lease_duration": 0,
+            "data": {
+                "last_vault_rotation": "2023-05-31T14:24:41.724285249Z",
+                "password": "foo",
+                "rotation_period": 60,
+                "ttl": 0,
+                "username": "foo",
             },
-            'wrap_info': None,
-            'warnings': None,
-            'auth': None
+            "wrap_info": None,
+            "warnings": None,
+            "auth": None,
         }
         return MockResponse(resp, 200)
 
@@ -45,41 +45,44 @@ def mock_db_role_request(mocker):
 def secret_setup():
     client = hvac.Client()
     client.secrets.kv.v2.create_or_update_secret(
-        path="test", secret=dict(test_secret="test_secret_password"))
+        path="test", secret=dict(test_secret="test_secret_password")
+    )
 
 
 @pytest.fixture(scope="function")
 def incorrect_env_setup():
-    os.environ['VAULT_ADDR'] = ""
+    os.environ["VAULT_ADDR"] = ""
 
 
 @pytest.fixture(scope="function")
 def mount_setup():
     client = hvac.Client()
-    secret_engines_list = client.sys.list_mounted_secrets_engines(
-    )['data'].keys()
+    secret_engines_list = client.sys.list_mounted_secrets_engines()["data"].keys()
     if "test-mount/" in secret_engines_list:
         client.sys.disable_secrets_engine(path="test-mount")
     client.sys.enable_secrets_engine(backend_type="kv", path="test-mount")
     client.secrets.kv.v2.create_or_update_secret(
         mount_point="test-mount",
         path="test",
-        secret=dict(test_mount="test_mount_password"))
+        secret=dict(test_mount="test_mount_password\\$"),
+    )
 
 
 @pytest.fixture(scope="function")
 def nested_setup():
     client = hvac.Client()
     client.secrets.kv.v2.create_or_update_secret(
-        path="testnested", secret=dict(slack={"token": "random-token"}))
+        path="testnested", secret=dict(slack={"token": "random-token"})
+    )
 
 
 @pytest.fixture
 def mock_vault_workers():
     mock_dynamic_renew = Mock()
     mock_k8s_renew = Mock()
-    with patch("gestalt.vault.Thread",
-               side_effect=[mock_dynamic_renew, mock_k8s_renew]):
+    with patch(
+        "gestalt.vault.Thread", side_effect=[mock_dynamic_renew, mock_k8s_renew]
+    ):
         yield (mock_dynamic_renew, mock_k8s_renew)
 
 

tests/test_gestalt.py

@@ -24,21 +24,9 @@ def test_merge_into():
     merge_into(combine4, combine2)
     merge_into(combine3, combine2)
 
-    assert combine1 == {
-        "local": 1234,
-        "pg": {
-            "host": "dict2_pg",
-            "pass": "dict1_pg"
-        }
-    }
+    assert combine1 == {"local": 1234, "pg": {"host": "dict2_pg", "pass": "dict1_pg"}}
 
-    assert combine2 == {
-        "local": 1234,
-        "pg": {
-            "host": "dict1_pg",
-            "pass": "dict1_pg"
-        }
-    }
+    assert combine2 == {"local": 1234, "pg": {"host": "dict1_pg", "pass": "dict1_pg"}}
 
 
 def test_combine_into_empty_dict():
@@ -256,15 +244,15 @@ def test_get_yaml_nested_default():
     g = gestalt.Gestalt()
     g.add_config_path("./tests/testdata")
     g.build_config()
-    testval = g.get_string("deep_yaml.nest1.nest2.foo", 'default')
+    testval = g.get_string("deep_yaml.nest1.nest2.foo", "default")
     assert testval == "hello"
 
 
 def test_get_yaml_missing_nested_default():
     g = gestalt.Gestalt()
     g.add_config_path("./tests/testdata")
     g.build_config()
-    testval = g.get_string("deep_yaml.nest1.nest2.fob", 'default')
+    testval = g.get_string("deep_yaml.nest1.nest2.fob", "default")
     assert testval == "default"
 
 
@@ -515,7 +503,7 @@ def test_vault_mount_path(mount_setup):
     g.configure_provider("vault", Vault(role=None, jwt=None))
     g.build_config()
     secret = g.get_string("test_mount.test_mount")
-    assert secret == "test_mount_password"
+    assert secret == r"test_mount_password\\$"
 
 
 def test_vault_incorrect_path(mount_setup):
@@ -550,8 +538,7 @@ def test_read_no_nest_db_role(mock_db_role_request):
 def test_set_vault_key(nested_setup):
     g = gestalt.Gestalt()
     g.configure_provider("vault", Vault(role=None, jwt=None))
-    g.set_string(key="test",
-                 value="ref+vault://secret/data/testnested#.slack.token")
+    g.set_string(key="test", value="ref+vault://secret/data/testnested#.slack.token")
     g.build_config()
     secret = g.get_string("test")
     assert secret == "ref+vault://secret/data/testnested#.slack.token"
@@ -576,8 +563,9 @@ def except_once(self, **kwargs):
         if mock_sleep.call_count == 1:
             raise hvac.exceptions.VaultError("some error")
 
-    with patch("gestalt.vault.sleep", side_effect=except_once,
-               autospec=True) as mock_sleep:
+    with patch(
+        "gestalt.vault.sleep", side_effect=except_once, autospec=True
+    ) as mock_sleep:
         with patch("gestalt.vault.hvac.Client") as mock_client:
             v = Vault(role="test-role", jwt="test-jwt")
             v.connect()
@@ -608,8 +596,9 @@ def except_once(self, **kwargs):
         if mock_sleep.call_count == 1:
             raise hvac.exceptions.VaultError("some error")
 
-    with patch("gestalt.vault.sleep", side_effect=except_once,
-               autospec=True) as mock_sleep:
+    with patch(
+        "gestalt.vault.sleep", side_effect=except_once, autospec=True
+    ) as mock_sleep:
         with patch("gestalt.vault.hvac.Client") as mock_client:
             v = Vault(role="test-role", jwt="test-jwt")
             v.connect()
@@ -634,19 +623,18 @@ def test_vault_start_dynamic_lease(mock_vault_workers):
     mock_response = {
         "lease_id": "1",
         "lease_duration": 5,
-        "data": {
-            "data": "mock_data"
-        },
+        "data": {"data": "mock_data"},
     }
 
-    mock_vault_client_patch = patch("gestalt.vault.hvac.Client.read",
-                                    return_value=mock_response)
+    mock_vault_client_patch = patch(
+        "gestalt.vault.hvac.Client.read", return_value=mock_response
+    )
     with mock_vault_client_patch as mock_vault_client_read:
         mock_dynamic_token_queue = Mock()
         mock_kube_token_queue = Mock()
         with patch(
-                "gestalt.vault.Queue",
-                side_effect=[mock_dynamic_token_queue, mock_kube_token_queue],
+            "gestalt.vault.Queue",
+            side_effect=[mock_dynamic_token_queue, mock_kube_token_queue],
         ) as mock_queues:
             v = Vault(role=None, jwt=None)
             g = gestalt.Gestalt()

tests/test_vault.py

@@ -8,7 +8,7 @@ def test_get(mount_setup):
     mount_setup_path = "test-mount/data/test"
     key = "test_mount"
     filter_ = f".{key}"
-    expected = "test_mount_password"
+    expected = r"test_mount_password\\$"
     vault = Vault()
     result = vault.get(key=key, path=mount_setup_path, filter=filter_)
     assert result == expected