diff --git a/.gitignore b/.gitignore index b7b7958..f52d3aa 100644 --- a/.gitignore +++ b/.gitignore @@ -105,4 +105,6 @@ venv.bak/ # mypy .mypy_cache/ -env.ps1 \ No newline at end of file + +#Idea +*.idea \ No newline at end of file diff --git a/README.md b/README.md index 309e999..3e9236f 100644 --- a/README.md +++ b/README.md @@ -101,4 +101,8 @@ La opción `--help` imprime la ayuda de cada comando en la pantalla. Check latest published settlements in asic's website. ``` +### Para tener en cuenta + +Tener presente que no se está realizando la verificación de certificados del servidor de XM **neptuno.xm.com.co** al consultar las versiones de liquidación publicadas usando el comando **asic pubs**. + ## Contribuir diff --git a/src/asic/VERSION b/src/asic/VERSION index abd4105..3a4036f 100644 --- a/src/asic/VERSION +++ b/src/asic/VERSION @@ -1 +1 @@ -0.2.4 +0.2.5 diff --git a/src/asic/__init__.py b/src/asic/__init__.py index b08f4bf..b14a73c 100644 --- a/src/asic/__init__.py +++ b/src/asic/__init__.py @@ -1,7 +1,4 @@ -from .config import ( - load_asic_file_config, - load_asic_file_extension_map, -) +from .config import load_asic_file_config, load_asic_file_extension_map ASIC_FILE_CONFIG = load_asic_file_config() ASIC_FILE_EXTENSION_MAP = load_asic_file_extension_map() diff --git a/src/asic/cli.py b/src/asic/cli.py index 4902880..abf4c40 100644 --- a/src/asic/cli.py +++ b/src/asic/cli.py @@ -13,11 +13,8 @@ from asic import ASIC_FILE_CONFIG, ASIC_FILE_EXTENSION_MAP from asic.config import ASICFileVisibility from asic.files.definitions import SUPPORTED_FILE_CLASSES -from asic.ftp import ( - get_ftps, - grab_file, # list_supported_files_in_location, - list_supported_files, -) +from asic.ftp import grab_file # list_supported_files_in_location, +from asic.ftp import get_ftps, list_supported_files from asic.publication import list_latest_published_versions logger = logging.getLogger("asic") @@ -81,7 +78,7 @@ def main( def validate_month(month: str) -> str: for f in YEAR_MONTH_FORMATS: try: - _value = dt.datetime.strptime(month, f).date() + dt.datetime.strptime(month, f).date() break except ValueError: continue @@ -332,8 +329,20 @@ def download( grab_file(ftps, remote.path, local) if is_preprocessing_required: + normalized_version = ( + f.metadata.version + if f.metadata.version is not None + else f.metadata.extension + ) + subpath_str = str(f.path)[1:].replace( + f"{f.year:04d}-{f.month:02d}", + f"{f.year:04d}-{f.month:02d}\\{normalized_version}", + ) + + preprocessed_path = destination.joinpath(subpath_str) + preprocessed = f.preprocess(local) - write_to = local.with_suffix(".csv") + write_to = preprocessed_path.with_suffix(".csv") preprocessed.to_csv( write_to, ) diff --git a/src/asic/config.py b/src/asic/config.py index f0eaaca..3a4dfe3 100644 --- a/src/asic/config.py +++ b/src/asic/config.py @@ -24,7 +24,7 @@ class ASICExtesionMap(BaseModel): asic_extension: Annotated[str, StringConstraints(pattern=r"^\.[a-zA-Z0-9]*$")] - normalized_version: Annotated[str, StringConstraints(pattern=r"^[0-9]{3}$")] + normalized_version: Annotated[str, StringConstraints(pattern=r"^[-]?[0-9]{3}$")] order: int @@ -88,7 +88,7 @@ def pattern_to_template_replacement(match_object: re.Match) -> str: def pattern_to_template(patt: str) -> str: - matches = PATTERN_REGEX.findall(patt) + PATTERN_REGEX.findall(patt) template = PATTERN_REGEX.sub(pattern_to_template_replacement, patt) return template diff --git a/src/asic/data/ASIC_FILE_CONFIG.jsonl b/src/asic/data/ASIC_FILE_CONFIG.jsonl index b13ce62..54bd805 100644 --- a/src/asic/data/ASIC_FILE_CONFIG.jsonl +++ b/src/asic/data/ASIC_FILE_CONFIG.jsonl @@ -9,6 +9,7 @@ {"code":"ldcbmr", "visibility": "public","name_pattern":"(?Pldcbmr)(?P[0-9]{2}).(?P[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/","description":""} {"code":"oefagnch", "visibility": "agent","name_pattern":"(?Poefagnch)(?P[0-9]{2})(?P[0-9]{2}).(?P[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/USUARIOSK/(?P[a-zA-Z]{4})/SIC/COMERCIA/(?P[0-9]{4})-(?P[0-9]{2})/","description":""} {"code":"pep", "visibility": "public","name_pattern":"(?Ppep)(?P[0-9]{2})(?P[0-9]{2}).(?P[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/","description":""} +{"code":"PME", "visibility": "public","name_pattern":"(?PPME)(?P140)(?P[0-9]{2}).(?P[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/","description":""} {"code":"pubfc_falla_hurto", "visibility": "public", "name_pattern":"(?PPubFC_Falla-Hurto)(?P[0-9]{4})-(?P[0-9]{2})-(?P[0-9]{2}).(?Pxlsx)", "location_pattern":"/informacion_xm/publicok/sic/comercia/Fronteras_en_proceso_de_registro/PUB_Falla_Hurto/","description":""} {"code":"pubfc", "visibility": "public", "name_pattern":"(?PPubFC)(?P[0-9]{4}).*(?P[0-9]{2}).*(?P[0-9]{2}).(?Pxlsx)", "location_pattern":"/informacion_xm/publicok/sic/comercia/Fronteras_en_proceso_de_registro/PUB_Registros/","description":""} {"code":"sntie", "visibility": "public","name_pattern":"(?Psntie)(?P[0-9]{2}).(?P[a-zA-Z0-9]+)","location_pattern":"/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/","description":""} diff --git a/src/asic/data/ASIC_FILE_EXTENSION_MAP.jsonl b/src/asic/data/ASIC_FILE_EXTENSION_MAP.jsonl index 6ca3aa9..3697c95 100644 --- a/src/asic/data/ASIC_FILE_EXTENSION_MAP.jsonl +++ b/src/asic/data/ASIC_FILE_EXTENSION_MAP.jsonl @@ -1,3 +1,4 @@ +{"asic_extension": ".txa", "order": -1, "normalized_version": "-001"} {"asic_extension": ".tx1", "order": 0, "normalized_version": "000"} {"asic_extension": ".tx2", "order": 1, "normalized_version": "001"} {"asic_extension": ".txr", "order": 2, "normalized_version": "002"} diff --git a/src/asic/files/definitions/__init__.py b/src/asic/files/definitions/__init__.py index dc38557..2cef2ca 100644 --- a/src/asic/files/definitions/__init__.py +++ b/src/asic/files/definitions/__init__.py @@ -3,12 +3,18 @@ from asic.files.definitions.adem import ADEM from asic.files.definitions.aenc import AENC from asic.files.definitions.balcttos import BALCTTOS +from asic.files.definitions.pep import PEP +from asic.files.definitions.pme import PME +from asic.files.definitions.trsd import TRSD from asic.files.file import AsicFile, FileKind SUPPORTED_FILE_CLASSES: dict[FileKind, Type[AsicFile]] = { FileKind.ADEM: ADEM, FileKind.AENC: AENC, FileKind.BALCTTOS: BALCTTOS, + FileKind.PEP: PEP, + FileKind.PME: PME, + FileKind.TRSD: TRSD, } diff --git a/src/asic/files/definitions/adem.py b/src/asic/files/definitions/adem.py index b389a3d..01c6f56 100644 --- a/src/asic/files/definitions/adem.py +++ b/src/asic/files/definitions/adem.py @@ -1,5 +1,6 @@ import logging -import pathlib +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath import numpy as np @@ -59,34 +60,34 @@ class ADEM(AsicFile): _format = FORMAT @property - def path(self): + def path(self) -> PureWindowsPath: return self._path @property - def year(self): + def year(self) -> int: return self._year @property - def month(self): + def month(self) -> int: return self._month @property - def day(self): + def day(self) -> int | None: return self._day @property - def extension(self): + def extension(self) -> str: return self._extension @property - def version(self): + def version(self) -> str | None: return self._version @property - def agent(self): + def agent(self) -> str | None: return self._agent - def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame: + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: """ ADEM: es un archivo diario versiones: TX2, TXR, TXF @@ -96,7 +97,7 @@ def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame: DMRE: demanda regulada PRRE: perdidas reguladas """ - total = self.reader.read(filepath) + total = self.read(target) total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}" filter = np.full(total.index.shape, True) diff --git a/src/asic/files/definitions/aenc.py b/src/asic/files/definitions/aenc.py index 149add4..ea78ac9 100644 --- a/src/asic/files/definitions/aenc.py +++ b/src/asic/files/definitions/aenc.py @@ -1,5 +1,6 @@ import logging -import pathlib +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath # Third party imports import pandas as pd @@ -65,40 +66,40 @@ class AENC(AsicFile): _format = FORMAT @property - def path(self): + def path(self) -> PureWindowsPath: return self._path @property - def year(self): + def year(self) -> int: return self._year @property - def month(self): + def month(self) -> int: return self._month @property - def day(self): + def day(self) -> int | None: return self._day @property - def extension(self): + def extension(self) -> str: return self._extension @property - def version(self): + def version(self) -> str | None: return self._version @property - def agent(self): + def agent(self) -> str | None: return self._agent - def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame: + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: """ AENC: es un archivo diario versiones: TX2, TXR, TXF VALOR: energia calculada por ASIC para frontera en cada periodo """ - total = self.reader.read(filepath) + total = self.read(target) total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}" total["AGENTE"] = self.agent diff --git a/src/asic/files/definitions/balcttos.py b/src/asic/files/definitions/balcttos.py index 185788f..a09c1cc 100644 --- a/src/asic/files/definitions/balcttos.py +++ b/src/asic/files/definitions/balcttos.py @@ -1,5 +1,6 @@ import logging -import pathlib +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath # Third party imports import pandas as pd @@ -58,34 +59,34 @@ class BALCTTOS(AsicFile): _format = FORMAT @property - def path(self): + def path(self) -> PureWindowsPath: return self._path @property - def year(self): + def year(self) -> int: return self._year @property - def month(self): + def month(self) -> int: return self._month @property - def day(self): + def day(self) -> int | None: return self._day @property - def extension(self): + def extension(self) -> str: return self._extension @property - def version(self): + def version(self) -> str | None: return self._version @property - def agent(self): + def agent(self) -> str | None: return self._agent - def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame: + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: """ balcttos: se publica un archivo por día. versiones: TX2, TXR y TXF @@ -95,7 +96,7 @@ def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame: AGENTE: EPSC: Código SIC del agente comercializador """ - total = self.reader.read(filepath) + total = self.read(target) total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}" total["FECHA"] = pd.to_datetime( diff --git a/src/asic/files/definitions/pep.py b/src/asic/files/definitions/pep.py new file mode 100644 index 0000000..485aa35 --- /dev/null +++ b/src/asic/files/definitions/pep.py @@ -0,0 +1,74 @@ +import logging +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath + +# Third party imports +import pandas as pd + +# Local application imports +from asic.files.file import AsicFile, FileKind, VisibilityEnum + +logger = logging.getLogger(__name__) + +FORMAT = { + "type": "csv", + "sep": ";", + "encoding": "cp1252", + "dt_fields": {}, + "dtype": {"AGENTE": str, "VALOR PE": float}, +} + + +class PEP(AsicFile): + kind = FileKind.PEP + visibility = VisibilityEnum.PUBLIC + name_pattern = "(?Ppep)(?P[0-9]{2})(?P[0-9]{2}).(?P[a-zA-Z0-9]+)" + location_pattern = "/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/" + description = "" # TODO + + _format = FORMAT + + @property + def path(self) -> PureWindowsPath: + return self._path + + @property + def year(self) -> int: + return self._year + + @property + def month(self) -> int: + return self._month + + @property + def day(self) -> int | None: + return self._day + + @property + def extension(self) -> str: + return self._extension + + @property + def version(self) -> str | None: + return self._version + + @property + def agent(self) -> str | None: + return self._agent + + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: + """ + pep: se publica un archivo por día. + versiones: TX1, TX2, TXR y TXF + AGENTE: + SISTEMA: el precio de escasez ponderado del sistema + """ + total = self.read(target) + total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}" + + total["FECHA"] = pd.to_datetime( + total["FECHA"], + format="%Y-%m-%d", + ) + return_cols = ["FECHA", "AGENTE", "VALOR PE"] + return total[return_cols] diff --git a/src/asic/files/definitions/pme.py b/src/asic/files/definitions/pme.py new file mode 100644 index 0000000..1f1b9ff --- /dev/null +++ b/src/asic/files/definitions/pme.py @@ -0,0 +1,70 @@ +import logging +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath + +# Third party imports +import pandas as pd + +# Local application imports +from asic.files.file import AsicFile, FileKind, VisibilityEnum + +logger = logging.getLogger(__name__) + +FORMAT = { + "type": "csv", + "sep": ";", + "encoding": "cp1252", + "dt_fields": {}, + "dtype": {"CONCEPTO": str, "DESCRIPCION": str, "VALOR": float}, +} + + +class PME(AsicFile): + kind = FileKind.PME + visibility = VisibilityEnum.PUBLIC + name_pattern = "(?PPME)(?P140)(?P[0-9]{2}).(?P[a-zA-Z0-9]+)" + location_pattern = "/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/" + description = "" # TODO + + _format = FORMAT + + @property + def path(self) -> PureWindowsPath: + return self._path + + @property + def year(self) -> int: + return self._year + + @property + def month(self) -> int: + return self._month + + @property + def day(self) -> int | None: + return self._day + + @property + def extension(self) -> str: + return self._extension + + @property + def version(self) -> str | None: + return self._version + + @property + def agent(self) -> str | None: + return self._agent + + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: + """ + PME: se publica un archivo mensual. + versiones: TXA + AGENTE: + SISTEMA: el precio de escacez de activación + """ + total = self.read(target) + total["MES"] = f"{self.year:04d}-{self.month:02d}" + + return_cols = ["MES", "CONCEPTO", "DESCRIPCION", "VALOR"] + return total[return_cols] diff --git a/src/asic/files/definitions/trsd.py b/src/asic/files/definitions/trsd.py new file mode 100644 index 0000000..ad69e77 --- /dev/null +++ b/src/asic/files/definitions/trsd.py @@ -0,0 +1,106 @@ +import logging +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath + +# Third party imports +import pandas as pd + +# Local application imports +from asic.files.file import AsicFile, FileKind, VisibilityEnum + +logger = logging.getLogger(__name__) + +FORMAT = { + "type": "csv", + "sep": ";", + "encoding": "cp1252", + "dt_fields": {}, + "dtype": { + "CODIGO": str, + "CONTENIDO": str, + "HORA 01": float, + "HORA 02": float, + "HORA 03": float, + "HORA 04": float, + "HORA 05": float, + "HORA 06": float, + "HORA 07": float, + "HORA 08": float, + "HORA 09": float, + "HORA 10": float, + "HORA 11": float, + "HORA 12": float, + "HORA 13": float, + "HORA 14": float, + "HORA 15": float, + "HORA 16": float, + "HORA 17": float, + "HORA 18": float, + "HORA 19": float, + "HORA 20": float, + "HORA 21": float, + "HORA 22": float, + "HORA 23": float, + "HORA 24": float, + }, +} + + +class TRSD(AsicFile): + kind = FileKind.TRSD + visibility = VisibilityEnum.PUBLIC + name_pattern = "(?Ptrsd)(?P[0-9]{2})(?P[0-9]{2}).(?P[a-zA-Z0-9]+)" + location_pattern = "/informacion_xm/publicok/sic/comercia/(?P[0-9]{4})-(?P[0-9]{2})/" + description = "" # TODO + + _format = FORMAT + + @property + def path(self) -> PureWindowsPath: + return self._path + + @property + def year(self) -> int: + return self._year + + @property + def month(self) -> int: + return self._month + + @property + def day(self) -> int | None: + return self._day + + @property + def extension(self) -> str: + return self._extension + + @property + def version(self) -> str | None: + return self._version + + @property + def agent(self) -> str | None: + return self._agent + + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: + """ + trsd: se publica un archivo por día. + versiones: TX1, TX2, TXR y TXF + PBNA: Precio de bolsa nacional + """ + total = self.read(target) + total["FECHA"] = f"{self.year:04d}-{self.month:02d}-{self.day:02d}" + total["FECHA"] = pd.to_datetime( + total["FECHA"], + format="%Y-%m-%d", + ) + total = total.set_index(["CODIGO", "CONTENIDO", "FECHA"]).stack().reset_index() + total = total.rename(columns={"level_3": "NOMBRE HORA", 0: "PRECIO"}) + total["HORA"] = (total["NOMBRE HORA"].str.slice(start=-2)).astype(int) - 1 + total["HORA"] = pd.to_timedelta(total["HORA"], unit="h") + total["FECHA_HORA"] = total["FECHA"] + total["HORA"] + total["HORA"] = total["FECHA_HORA"].dt.strftime("%H:%M:%S") + total["FECHA"] = total["FECHA"].dt.date + ret_cols = ["FECHA_HORA", "CODIGO", "CONTENIDO", "PRECIO"] + return total[ret_cols] diff --git a/src/asic/files/file.py b/src/asic/files/file.py index 692353b..938f59d 100644 --- a/src/asic/files/file.py +++ b/src/asic/files/file.py @@ -1,7 +1,8 @@ import enum -import pathlib import re from abc import ABC, abstractmethod +from io import BytesIO, StringIO +from pathlib import Path, PureWindowsPath import pandas as pd import pydantic @@ -60,19 +61,19 @@ class FileKind(str, enum.Enum): ADEM = "adem" AENC = "aenc" BALCTTOS = "balcttos" + PEP = "pep" + PME = "PME" + TRSD = "trsd" # TGRL = "tgrl" # TRSM = "trsm" # LDCBMR = "ldcbmr" - # PEP = "pep" - # TRSD = "trsd" # SNTIE = "sntie" # DSPCTTOS = "dspcttos" # PUBFC = "pubfc" # PUBFC_FALLA_HURTO = "pubfc_falla-hurto" # TFROC = "tfroc" # CLIQ = "cliq" - # DESBM = "desbm" # DESBMEX = "desbmex" # OEFAGNCH = "oefagnch" @@ -85,12 +86,12 @@ class AsicFileMetadataInput(pydantic.BaseModel): month: int day: int | None = None extension: str - version: str | None = pydantic.Field(None, pattern=r"^[0-9]{3}$") + version: str | None = pydantic.Field(None, pattern=r"^[-]?[0-9]{3}$") agent: str | None = pydantic.Field(None, pattern=r"^[a-z]{4}$") class AsicFileMetadata(AsicFileMetadataInput): - remote_path: pathlib.PureWindowsPath + remote_path: PureWindowsPath kind: FileKind @@ -108,7 +109,7 @@ class AsicFile(ABC): @property @abstractmethod - def path(self) -> pathlib.PureWindowsPath: + def path(self) -> PureWindowsPath: pass @property @@ -142,7 +143,7 @@ def agent(self) -> str | None: pass @abstractmethod - def preprocess(self, filepath: pathlib.Path) -> pd.DataFrame: + def preprocess(self, target: Path | BytesIO | StringIO) -> pd.DataFrame: pass @property @@ -152,27 +153,27 @@ def _format(self) -> dict: def __init__( self, - path: pathlib.PureWindowsPath, + path: PureWindowsPath, year: int, month: int, + extension: str, day: int | None = None, - extension: str | None = None, version: str | None = None, agent: str | None = None, ) -> None: - self._path: pathlib.PureWindowsPath = path + self._path: PureWindowsPath = path self._year: int = year self._month: int = month self._day: int | None = day - self._extension: str | None = extension + self._extension: str = extension self._version: str | None = version self._agent: str | None = agent self.reader = FileReader(self._format) self.name_template = pattern_to_template(self.name_pattern) self.location_template = pattern_to_template(self.location_pattern) - def read(self, local_path: pathlib.Path) -> pd.DataFrame: - return self.reader.read(local_path) + def read(self, target: str | Path | StringIO | BytesIO) -> pd.DataFrame: + return self.reader.read(target) @property def metadata(self) -> AsicFileMetadata: @@ -188,14 +189,14 @@ def metadata(self) -> AsicFileMetadata: ) @classmethod - def from_remote_path(cls, remote_path: pathlib.PureWindowsPath) -> Self: + def from_remote_path(cls, remote_path: PureWindowsPath) -> Self: path_metadata = cls.extract_metadata_from_remote_path(remote_path) file = cls(path=remote_path, **path_metadata.model_dump()) return file @classmethod def extract_metadata_from_remote_path( - cls, file_path: pathlib.PureWindowsPath + cls, file_path: PureWindowsPath ) -> AsicFileMetadataInput: file_path_as_posix = file_path.as_posix() path_pattern = cls.location_pattern + cls.name_pattern diff --git a/src/asic/publication.py b/src/asic/publication.py index d4ca388..89bfac0 100644 --- a/src/asic/publication.py +++ b/src/asic/publication.py @@ -27,7 +27,7 @@ class ASICVersionPublication(pydantic.BaseModel): "Chrome/80.0.3987.149 Safari/537.36" ) ASIC_MONTHLY_VERSION_PUBLICATION_SERVICE: dict[str, typing.Any] = { - "url": "http://sv01.xm.com.co/gmem/Admon_Mcdo/Liquidacion/versionesliq.htm", + "url": "https://neptuno.xm.com.co/Admon_Mcdo/Liquidacion/versionesliq.htm", "table-index": 1, "encoding": "cp1252", "month_settlement_format": r"%b %Y", @@ -70,7 +70,7 @@ def get_monthly_pubs_table() -> pd.DataFrame: table_index: int = ASIC_MONTHLY_VERSION_PUBLICATION_SERVICE["table-index"] logger.debug(f"Getting content from '{url}'") - res = requests.get(url, headers=headers) + res = requests.get(url, headers=headers, verify=False) res.raise_for_status() html_text = typing.cast(str, res.content) diff --git a/src/asic/reader.py b/src/asic/reader.py index 3e50e2d..45ecef4 100644 --- a/src/asic/reader.py +++ b/src/asic/reader.py @@ -1,8 +1,7 @@ # Standard library imports import logging -from io import StringIO +from io import BytesIO, StringIO from pathlib import Path -from typing import Union # Third party imports import pandas as pd @@ -21,7 +20,7 @@ def __init__(self, file_def: dict) -> None: logger.debug(f"Creating FileReader with file definition:\n{file_def}") self.file_def = file_def - def read(self, target: Union[str, Path, StringIO]) -> pd.DataFrame: + def read(self, target: str | Path | StringIO | BytesIO) -> pd.DataFrame: """Reads DataFrame from target.""" _def = self.file_def.copy() file_type = _def.pop("type", None) @@ -33,7 +32,6 @@ def read(self, target: Union[str, Path, StringIO]) -> pd.DataFrame: res = pd.read_csv(target, **_def) elif file_type in ["xls", "xlsx"]: - _encoding = _def.pop("encoding") if "sheet_name" in _def: res = pd.read_excel(str(target), **_def) else: diff --git a/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/PME14001.txa b/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/PME14001.txa new file mode 100644 index 0000000..226a95a --- /dev/null +++ b/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/PME14001.txa @@ -0,0 +1,60 @@ +CONCEPTO;DESCRIPCION;VALOR +ACPM;Heat Rate Reportado Por La Planta TDR1 ;10.4674 +ACPM;Heat Rate Reportado Por La Planta TEC1 ;7.1852 +ACPM;Heat Rate Reportado Por La Planta TRN1 ;8.45 +ACPM;Heat Rate Reportado Por La Planta TSR1 ;7.1417 +ACPM;Heat Rate Reportado Por La Planta TVL1 ;6.5573 +CARB;Heat Rate Reportado Por La Planta GE32 ;9 +CARB;Heat Rate Reportado Por La Planta GEC3 ;10.15 +CARB;Heat Rate Reportado Por La Planta PPA1 ;12.4335 +CARB;Heat Rate Reportado Por La Planta PPA2 ;10.5569 +CARB;Heat Rate Reportado Por La Planta PPA3 ;10.4642 +CARB;Heat Rate Reportado Por La Planta PPA4 ;8.6485 +CARB;Heat Rate Reportado Por La Planta TGJ1 ;10 +CARB;Heat Rate Reportado Por La Planta TGJ2 ;9.8 +CARB;Heat Rate Reportado Por La Planta TSJ1 ;9.2374 +CARB;Heat Rate Reportado Por La Planta TSJ2 ;10.1502 +CARB;Heat Rate Reportado Por La Planta ZPA2 ;11.4401 +CARB;Heat Rate Reportado Por La Planta ZPA3 ;11.7024 +CARB;Heat Rate Reportado Por La Planta ZPA4 ;12.9999 +CARB;Heat Rate Reportado Por La Planta ZPA5 ;12.6678 +CRUD;Heat Rate Reportado Por La Planta JAG1 ;8.0091 +CRUD;Heat Rate Reportado Por La Planta RUB1 ;8.0091 +FOIL;Heat Rate Reportado Por La Planta CTG1 ;13.665 +FOIL;Heat Rate Reportado Por La Planta CTG2 ;11.5167 +FOIL;Heat Rate Reportado Por La Planta CTG3 ;13.8215 +GANI;Heat Rate Reportado Por La Planta TBQ3 ;10.4846 +GANI;Heat Rate Reportado Por La Planta TBQ4 ;11.6696 +GANI;Heat Rate Reportado Por La Planta TBST ;7.1898 +GANI;Heat Rate Reportado Por La Planta TCD1 ;10.2925 +GANI;Heat Rate Reportado Por La Planta TCD2 ;10.0884 +GANI;Heat Rate Reportado Por La Planta TCDT ;6.685 +GANI;Heat Rate Reportado Por La Planta TFL1 ;7.267 +GANI;Heat Rate Reportado Por La Planta TFL4 ;7.0066 +GASN;Heat Rate Reportado Por La Planta 3ENA ;7.5166 +GASN;Heat Rate Reportado Por La Planta MRL1 ;9.0003 +GASN;Heat Rate Reportado Por La Planta PRG1 ;8.1666 +GASN;Heat Rate Reportado Por La Planta TRM1 ;7.6331 +GASN;Heat Rate Reportado Por La Planta TSR1 ;6.7852 +GASN;Heat Rate Reportado Por La Planta TYP1 ;13.5516 +GASN;Heat Rate Reportado Por La Planta TYP2 ;12.0234 +GASN;Heat Rate Reportado Por La Planta TYP3 ;8.4932 +GASN;Heat Rate Reportado Por La Planta TYP4 ;8.3937 +GASN;Heat Rate Reportado Por La Planta TYP5 ;8.0585 +GSLP;Heat Rate Reportado Por La Planta TCBE ;8.42 +MDMA;Heat Rate Reportado Por La Planta TRN1 ;8.45 +PE;Precio Escasez ;778.9951119672 +PEA;PRECIO ESCASEZ DE ACTIVACIÓN ;1144.04343630868563442014268458332081211 +PME;PRECIO MARGINAL DE ESCASEZ ;1144.04343630868563442014268458332081211 +SISTEMA;Costo De Operacion Y Mantenimiento CARB ;32.3404903661 +SISTEMA;Costo De Operacion Y Mantenimiento GASN ;15.7732519268 +SISTEMA;Costo De Operacion Y Mantenimiento OTRO ;24.0588429672 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] ACPM ;143684.030498323 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] CARB ;15360.5684598475985294117647058823529412 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] CRUD ;52300.541045 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] FOIL ;75739 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] GANI ;68439.8971085714285714285714285714285714 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] GASN ;44722.0125687006 +SISTEMA;Costo Referencia Por Tipo Combustible [$/Mbtu] GSLP ;.00002 +SISTEMA;Otros Costos Variables Hidráulicos ;90.7265974049352148201426845833208121139 +SISTEMA;Otros Costos Variables Termicos ;87.5860974049352148201426845833208121139 diff --git a/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/pep1001.tx1 b/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/pep1001.tx1 index a5ca9c2..96fc957 100644 --- a/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/pep1001.tx1 +++ b/tests/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/pep1001.tx1 @@ -1,2 +1,24 @@ -AGENTE;VALOR PE -SISTEMA;0 +AGENTE;VALOR PE +CHVG;1144.0434363 +EMUG;1144.0434363 +ENDG;1096.1854645 +EPMG;1068.1930556 +EPSG;1139.5240051 +GECG;1144.0434363 +HDPG;860.7375742 +HIMG;1144.0434363 +ISGG;984.1067685 +PRIG;1144.0434363 +SISTEMA;1092.3653461 +SLMG;1144.0434363 +SOCG;1144.0434363 +TBSG;1144.0434363 +TCBG;1144.0434363 +TCIG;1144.0434363 +TEMG;1144.0434363 +TERG;801.5377949 +TMFG;1144.0434363 +TMNG;813.7457986 +TMVG;1144.0434363 +TRMG;1144.0434363 +TYPG;1144.0434363 diff --git a/tests/conftest.py b/tests/conftest.py index 968b6ef..dd993bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,4 +50,37 @@ "version": "001", "agent": "xxxc", }, + "pep": { + "path": "/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/pep1001.tx1", + "kind": "pep", + "visibility": "public", + "year": 2023, + "month": 10, + "day": 1, + "extension": ".tx1", + "version": "000", + "agent": None, + }, + "PME": { + "path": "/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/PME14001.txa", + "kind": "PME", + "visibility": "public", + "year": 2023, + "month": 10, + "day": None, + "extension": ".txa", + "version": "-001", + "agent": None, + }, + "trsd": { + "path": "/INFORMACION_XM/PUBLICOK/SIC/COMERCIA/2023-10/trsd1001.tx2", + "kind": "trsd", + "visibility": "public", + "year": 2023, + "month": 10, + "day": 1, + "extension": ".tx2", + "version": "001", + "agent": None, + }, } diff --git a/tests/test_pep.py b/tests/test_pep.py new file mode 100644 index 0000000..e005073 --- /dev/null +++ b/tests/test_pep.py @@ -0,0 +1,40 @@ +import pathlib + +from asic.files.definitions.pep import PEP +from pytest import fixture + +from .conftest import ALL_FILES, TESTFILES + + +@fixture +def pep_remote_path(): + pep_path = ALL_FILES["pep"]["path"] + path = pathlib.PureWindowsPath(pep_path) + return path + + +@fixture +def pep_file(pep_remote_path): + path = pep_remote_path + file = PEP.from_remote_path(path) + return file + + +@fixture +def local_pep_file(pep_file: PEP, datafiles: pathlib.Path) -> pathlib.Path: + relative_path = pep_file.path.relative_to(pep_file.path.anchor) + local_file = datafiles / relative_path + assert local_file.is_file() + return local_file + + +@TESTFILES +def test_pep_read(pep_file: PEP, local_pep_file): + data = pep_file.read(local_pep_file) + assert len(data) == 23 + + +@TESTFILES +def test_pep_preprocess(pep_file: PEP, local_pep_file): + long_data = pep_file.preprocess(local_pep_file) + assert len(long_data) == 23 diff --git a/tests/test_pme.py b/tests/test_pme.py new file mode 100644 index 0000000..96e3ce6 --- /dev/null +++ b/tests/test_pme.py @@ -0,0 +1,40 @@ +import pathlib + +from asic.files.definitions.pme import PME +from pytest import fixture + +from .conftest import ALL_FILES, TESTFILES + + +@fixture +def PME_remote_path(): + PME_path = ALL_FILES["PME"]["path"] + path = pathlib.PureWindowsPath(PME_path) + return path + + +@fixture +def PME_file(PME_remote_path): + path = PME_remote_path + file = PME.from_remote_path(path) + return file + + +@fixture +def local_PME_file(PME_file: PME, datafiles: pathlib.Path) -> pathlib.Path: + relative_path = PME_file.path.relative_to(PME_file.path.anchor) + local_file = datafiles / relative_path + assert local_file.is_file() + return local_file + + +@TESTFILES +def test_PME_read(PME_file: PME, local_PME_file): + data = PME_file.read(local_PME_file) + assert len(data) == 59 + + +@TESTFILES +def test_PME_preprocess(PME_file: PME, local_PME_file): + long_data = PME_file.preprocess(local_PME_file) + assert len(long_data) == 59 diff --git a/tests/test_trsd.py b/tests/test_trsd.py new file mode 100644 index 0000000..3e08b0a --- /dev/null +++ b/tests/test_trsd.py @@ -0,0 +1,40 @@ +import pathlib + +from asic.files.definitions.trsd import TRSD +from pytest import fixture + +from .conftest import ALL_FILES, TESTFILES + + +@fixture +def trsd_remote_path(): + trsd_path = ALL_FILES["trsd"]["path"] + path = pathlib.PureWindowsPath(trsd_path) + return path + + +@fixture +def trsd_file(trsd_remote_path): + path = trsd_remote_path + file = TRSD.from_remote_path(path) + return file + + +@fixture +def local_trsd_file(trsd_file: TRSD, datafiles: pathlib.Path) -> pathlib.Path: + relative_path = trsd_file.path.relative_to(trsd_file.path.anchor) + local_file = datafiles / relative_path + assert local_file.is_file() + return local_file + + +@TESTFILES +def test_trsd_read(trsd_file: TRSD, local_trsd_file): + data = trsd_file.read(local_trsd_file) + assert len(data) == 33 + + +@TESTFILES +def test_trsd_preprocess(trsd_file: TRSD, local_trsd_file): + long_data = trsd_file.preprocess(local_trsd_file) + assert len(long_data) == 720