Merge pull request #10 from JMTamayo/fixture/v0.2.5

Fixture/v0.2.5

.gitignore

@@ -105,4 +105,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
-env.ps1
+
+#Idea
+*.idea

README.md

@@ -101,4 +101,8 @@ La opción `--help` imprime la ayuda de cada comando en la pantalla.
  Check latest published settlements in asic's website.
 ```
 
+### Para tener en cuenta
+
+Tener presente que no se está realizando la verificación de certificados del servidor de XM **neptuno.xm.com.co** al consultar las versiones de liquidación publicadas usando el comando **asic pubs**.
+
 ## Contribuir

src/asic/VERSION

@@ -1 +1 @@
-0.2.4
+0.2.5

src/asic/__init__.py

@@ -1,7 +1,4 @@
-from .config import (
-    load_asic_file_config,
-    load_asic_file_extension_map,
-)
+from .config import load_asic_file_config, load_asic_file_extension_map
 
 ASIC_FILE_CONFIG = load_asic_file_config()
 ASIC_FILE_EXTENSION_MAP = load_asic_file_extension_map()

src/asic/cli.py

@@ -13,11 +13,8 @@
 from asic import ASIC_FILE_CONFIG, ASIC_FILE_EXTENSION_MAP
 from asic.config import ASICFileVisibility
 from asic.files.definitions import SUPPORTED_FILE_CLASSES
-from asic.ftp import (
-    get_ftps,
-    grab_file,  # list_supported_files_in_location,
-    list_supported_files,
-)
+from asic.ftp import grab_file  # list_supported_files_in_location,
+from asic.ftp import get_ftps, list_supported_files
 from asic.publication import list_latest_published_versions
 
 logger = logging.getLogger("asic")
@@ -81,7 +78,7 @@ def main(
 def validate_month(month: str) -> str:
     for f in YEAR_MONTH_FORMATS:
         try:
-            _value = dt.datetime.strptime(month, f).date()
+            dt.datetime.strptime(month, f).date()
             break
         except ValueError:
             continue
@@ -332,8 +329,20 @@ def download(
             grab_file(ftps, remote.path, local)
 
         if is_preprocessing_required:
+            normalized_version = (
+                f.metadata.version
+                if f.metadata.version is not None
+                else f.metadata.extension
+            )
+            subpath_str = str(f.path)[1:].replace(
+                f"{f.year:04d}-{f.month:02d}",
+                f"{f.year:04d}-{f.month:02d}\\{normalized_version}",
+            )
+
+            preprocessed_path = destination.joinpath(subpath_str)
+
             preprocessed = f.preprocess(local)
-            write_to = local.with_suffix(".csv")
+            write_to = preprocessed_path.with_suffix(".csv")
             preprocessed.to_csv(
                 write_to,
             )

src/asic/config.py

@@ -24,7 +24,7 @@
 
 class ASICExtesionMap(BaseModel):
     asic_extension: Annotated[str, StringConstraints(pattern=r"^\.[a-zA-Z0-9]*$")]
-    normalized_version: Annotated[str, StringConstraints(pattern=r"^[0-9]{3}$")]
+    normalized_version: Annotated[str, StringConstraints(pattern=r"^[-]?[0-9]{3}$")]
     order: int
 
 
@@ -88,7 +88,7 @@ def pattern_to_template_replacement(match_object: re.Match) -> str:
 
 
 def pattern_to_template(patt: str) -> str:
-    matches = PATTERN_REGEX.findall(patt)
+    PATTERN_REGEX.findall(patt)
     template = PATTERN_REGEX.sub(pattern_to_template_replacement, patt)
     return template
 

src/asic/data/ASIC_FILE_CONFIG.jsonl

@@ -9,6 +9,7 @@
 {"code":"ldcbmr", "visibility": "public","name_pattern":"(?P<kind>ldcbmr)(?P<name_month>[0-9]{2}).(?P<ext_versioned>[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P<location_year>[0-9]{4})-(?P<location_month>[0-9]{2})/","description":""}
 {"code":"oefagnch", "visibility": "agent","name_pattern":"(?P<kind>oefagnch)(?P<name_month>[0-9]{2})(?P<name_day>[0-9]{2}).(?P<ext_versioned>[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/USUARIOSK/(?P<location_agent>[a-zA-Z]{4})/SIC/COMERCIA/(?P<location_year>[0-9]{4})-(?P<location_month>[0-9]{2})/","description":""}
 {"code":"pep", "visibility": "public","name_pattern":"(?P<kind>pep)(?P<name_month>[0-9]{2})(?P<name_day>[0-9]{2}).(?P<ext_versioned>[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P<location_year>[0-9]{4})-(?P<location_month>[0-9]{2})/","description":""}
+{"code":"PME", "visibility": "public","name_pattern":"(?P<kind>PME)(?P<ordinance>140)(?P<name_month>[0-9]{2}).(?P<ext_versioned>[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P<location_year>[0-9]{4})-(?P<location_month>[0-9]{2})/","description":""}
 {"code":"pubfc_falla_hurto", "visibility": "public", "name_pattern":"(?P<kind>PubFC_Falla-Hurto)(?P<name_year>[0-9]{4})-(?P<name_month>[0-9]{2})-(?P<name_day>[0-9]{2}).(?P<ext_excel>xlsx)", "location_pattern":"/informacion_xm/publicok/sic/comercia/Fronteras_en_proceso_de_registro/PUB_Falla_Hurto/","description":""}
 {"code":"pubfc", "visibility": "public", "name_pattern":"(?P<kind>PubFC)(?P<name_year>[0-9]{4}).*(?P<name_month>[0-9]{2}).*(?P<name_day>[0-9]{2}).(?P<ext_excel>xlsx)", "location_pattern":"/informacion_xm/publicok/sic/comercia/Fronteras_en_proceso_de_registro/PUB_Registros/","description":""}
 {"code":"sntie", "visibility": "public","name_pattern":"(?P<kind>sntie)(?P<name_month>[0-9]{2}).(?P<ext_versioned>[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P<location_year>[0-9]{4})-(?P<location_month>[0-9]{2})/","description":""}

src/asic/data/ASIC_FILE_EXTENSION_MAP.jsonl

@@ -1,3 +1,4 @@
+{"asic_extension": ".txa", "order":  -1, "normalized_version": "-001"}
 {"asic_extension": ".tx1", "order":  0, "normalized_version": "000"}
 {"asic_extension": ".tx2", "order":  1, "normalized_version": "001"}
 {"asic_extension": ".txr", "order":  2, "normalized_version": "002"}

src/asic/files/definitions/__init__.py

@@ -3,12 +3,18 @@
 from asic.files.definitions.adem import ADEM
 from asic.files.definitions.aenc import AENC
 from asic.files.definitions.balcttos import BALCTTOS
+from asic.files.definitions.pep import PEP
+from asic.files.definitions.pme import PME
+from asic.files.definitions.trsd import TRSD
 from asic.files.file import AsicFile, FileKind
 
 SUPPORTED_FILE_CLASSES: dict[FileKind, Type[AsicFile]] = {
     FileKind.ADEM: ADEM,
     FileKind.AENC: AENC,
     FileKind.BALCTTOS: BALCTTOS,
+    FileKind.PEP: PEP,
+    FileKind.PME: PME,
+    FileKind.TRSD: TRSD,
 }
 
 

src/asic/files/definitions/adem.py

@@ -1,5 +1,6 @@
 import logging
-import pathlib
+from io import BytesIO, StringIO
+from pathlib import Path, PureWindowsPath
 
 import numpy as np
 
@@ -59,34 +60,34 @@ class ADEM(AsicFile):
     _format = FORMAT
 
     @property
-    def path(self):
+    def path(self) -> PureWindowsPath:
         return self._path
 
     @property
-    def year(self):
+    def year(self) -> int:
         return self._year
 
     @property
-    def month(self):
+    def month(self) -> int:
         return self._month
 
     @property
-    def day(self):
+    def day(self) -> int | None:
         return self._day
 
     @property
-    def extension(self):
+    def extension(self) -> str:
         return self._extension
 
     @property
-    def version(self):
+    def version(self) -> str | None:
         return self._version
 
     @property
-    def agent(self):
+    def agent(self) -> str | None:
         return self._agent
 
-    def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame:
+    def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame:
         """
         ADEM: es un archivo diario
         versiones: TX2, TXR, TXF
@@ -96,7 +97,7 @@ def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame:
         DMRE: demanda regulada
         PRRE: perdidas reguladas
         """
-        total = self.reader.read(filepath)
+        total = self.read(target)
         total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}"
 
         filter = np.full(total.index.shape, True)

src/asic/files/definitions/aenc.py

@@ -1,5 +1,6 @@
 import logging
-import pathlib
+from io import BytesIO, StringIO
+from pathlib import Path, PureWindowsPath
 
 # Third party imports
 import pandas as pd
@@ -65,40 +66,40 @@ class AENC(AsicFile):
     _format = FORMAT
 
     @property
-    def path(self):
+    def path(self) -> PureWindowsPath:
         return self._path
 
     @property
-    def year(self):
+    def year(self) -> int:
         return self._year
 
     @property
-    def month(self):
+    def month(self) -> int:
         return self._month
 
     @property
-    def day(self):
+    def day(self) -> int | None:
         return self._day
 
     @property
-    def extension(self):
+    def extension(self) -> str:
         return self._extension
 
     @property
-    def version(self):
+    def version(self) -> str | None:
         return self._version
 
     @property
-    def agent(self):
+    def agent(self) -> str | None:
         return self._agent
 
-    def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame:
+    def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame:
         """
         AENC: es un archivo diario
         versiones: TX2, TXR, TXF
         VALOR: energia calculada por ASIC para frontera en cada periodo
         """
-        total = self.reader.read(filepath)
+        total = self.read(target)
         total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}"
         total["AGENTE"] = self.agent
 

src/asic/files/definitions/balcttos.py

@@ -1,5 +1,6 @@
 import logging
-import pathlib
+from io import BytesIO, StringIO
+from pathlib import Path, PureWindowsPath
 
 # Third party imports
 import pandas as pd
@@ -58,34 +59,34 @@ class BALCTTOS(AsicFile):
     _format = FORMAT
 
     @property
-    def path(self):
+    def path(self) -> PureWindowsPath:
         return self._path
 
     @property
-    def year(self):
+    def year(self) -> int:
         return self._year
 
     @property
-    def month(self):
+    def month(self) -> int:
         return self._month
 
     @property
-    def day(self):
+    def day(self) -> int | None:
         return self._day
 
     @property
-    def extension(self):
+    def extension(self) -> str:
         return self._extension
 
     @property
-    def version(self):
+    def version(self) -> str | None:
         return self._version
 
     @property
-    def agent(self):
+    def agent(self) -> str | None:
         return self._agent
 
-    def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame:
+    def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame:
         """
         balcttos: se publica un archivo por día.
         versiones: TX2, TXR y TXF
@@ -95,7 +96,7 @@ def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame:
         AGENTE:
         EPSC: Código SIC del agente comercializador
         """
-        total = self.reader.read(filepath)
+        total = self.read(target)
         total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}"
 
         total["FECHA"] = pd.to_datetime(

src/asic/files/definitions/pep.py

@@ -0,0 +1,74 @@
+import logging
+from io import BytesIO, StringIO
+from pathlib import Path, PureWindowsPath
+
+# Third party imports
+import pandas as pd
+
+# Local application imports
+from asic.files.file import AsicFile, FileKind, VisibilityEnum
+
+logger = logging.getLogger(__name__)
+
+FORMAT = {
+    "type": "csv",
+    "sep": ";",
+    "encoding": "cp1252",
+    "dt_fields": {},
+    "dtype": {"AGENTE": str, "VALOR PE": float},
+}
+
+
+class PEP(AsicFile):
+    kind = FileKind.PEP
+    visibility = VisibilityEnum.PUBLIC
+    name_pattern = "(?P<kind>pep)(?P<name_month>[0-9]{2})(?P<name_day>[0-9]{2}).(?P<ext_versioned>[a-zA-Z0-9]+)"
+    location_pattern = "/informacion_xm/publicok/sic/comercia/(?P<location_year>[0-9]{4})-(?P<location_month>[0-9]{2})/"
+    description = ""  # TODO
+
+    _format = FORMAT
+
+    @property
+    def path(self) -> PureWindowsPath:
+        return self._path
+
+    @property
+    def year(self) -> int:
+        return self._year
+
+    @property
+    def month(self) -> int:
+        return self._month
+
+    @property
+    def day(self) -> int | None:
+        return self._day
+
+    @property
+    def extension(self) -> str:
+        return self._extension
+
+    @property
+    def version(self) -> str | None:
+        return self._version
+
+    @property
+    def agent(self) -> str | None:
+        return self._agent
+
+    def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame:
+        """
+        pep: se publica un archivo por día.
+        versiones: TX1, TX2, TXR y TXF
+        AGENTE:
+          SISTEMA: el precio de escasez ponderado del sistema
+        """
+        total = self.read(target)
+        total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}"
+
+        total["FECHA"] = pd.to_datetime(
+            total["FECHA"],
+            format="%Y-%m-%d",
+        )
+        return_cols = ["FECHA", "AGENTE", "VALOR PE"]
+        return total[return_cols]