From b6d022509bbc0b29d71d9ce7a3b7241ec33a1f6d Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Wed, 10 Apr 2024 16:13:40 +0200 Subject: [PATCH 01/18] modify converter and tests according to issues (#57) and (#60) correct errors after testing correct suffix modify tests according to converter modifications --- eye2bids/edf2bids.py | 269 ++++++++++++++++++++++++----------------- tests/test_edf2bids.py | 78 +++++++++--- 2 files changed, 222 insertions(+), 125 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 9be7b85..edaacfc 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -115,23 +115,29 @@ def _convert_edf_to_asc_samples(input_file: str | Path) -> Path: subprocess.run(["edf2asc", "-y", "-s", input_file, "-o", samples_asc_file]) return Path(samples_asc_file).with_suffix(".asc") +def _2eyesmode(df: pd.DataFrame) -> bool: + eye = df[df[2] == "RECCFG"].iloc[0:1, 5:6].to_string(header=False, index=False) + if eye == "LR": + two_eyes = True + else: + two_eyes = False + return two_eyes def _calibrations(df: pd.DataFrame) -> pd.DataFrame: return df[df[3] == "CALIBRATION"] - def _extract_CalibrationType(df: pd.DataFrame) -> list[int]: return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False) def _extract_CalibrationCount(df: pd.DataFrame) -> int: - if _extract_RecordedEye(df) == "Both": + if _2eyesmode(df) == True: return len(_calibrations(df)) // 2 return len(_calibrations(df)) def _get_calibration_positions(df: pd.DataFrame) -> list[int]: - if _extract_RecordedEye(df) == "Both": + if _2eyesmode(df) == True: return ( np.array(df[df[2] == "VALIDATE"][8].str.split(",", expand=True)) .astype(int) @@ -198,30 +204,12 @@ def _has_validation(df: pd.DataFrame) -> bool: def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]: if not _has_validation(df): return [] - if _extract_RecordedEye(df) == "Both" and _extract_CalibrationCount(df) > 1: - return ( - np.array_split( - np.array(_validations(df)[[11]]), - _extract_CalibrationCount(df), - ) - .astype(float) - .tolist() - ) return np.array(_validations(df)[[11]]).astype(float).tolist() def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]: if not _has_validation(df): return [] - if _extract_RecordedEye(df) == "Both" and _extract_CalibrationCount(df) > 1: - return ( - np.array_split( - np.array(_validations(df)[[9]]), - _extract_CalibrationCount(df), - ) - .astype(float) - .tolist() - ) return np.array(_validations(df)[[9]]).astype(float).tolist() @@ -256,8 +244,8 @@ def _extract_RecordedEye(df: pd.DataFrame) -> str: elif eye == "R": return "Right" elif eye == "LR": - return "Both" - return "" + return ["Left", "Right"] + return "" def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]: @@ -331,36 +319,6 @@ def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame: df_ms = _load_asc_file_as_df(events_asc_file) return pd.DataFrame(df_ms.iloc[0:, 2:]) - -def _samples_to_data_frame(samples_asc_file: str | Path) -> pd.DataFrame: - column_names = [ - "eye_timestamp", - "eye1_x_coordinate", - "eye1_y_coordinate", - "eye1_pupil_size", - "eye2_x_coordinate", - "eye2_y_coordinate", - "eye2_pupil_size", - ] - - data: dict[str, list[str]] = {name: [] for name in column_names} - - with open(samples_asc_file) as file: - for line in file: - columns = line.strip().split("\t") - for i in range(len(column_names)): - if i < len(columns): - data[column_names[i]].append(columns[i]) - - data = { - key: value - for key, value in data.items() - if any(val not in ("", "...") for val in value) - } - - return pd.DataFrame(data) - - def edf2bids( input_file: str | Path | None = None, metadata_file: str | Path | None = None, @@ -394,44 +352,108 @@ def edf2bids( with open(metadata_file) as f: metadata = yaml.load(f, Loader=SafeLoader) - # events.json Metadata - eyetrack_json = { - "Manufacturer": "SR-Research", - "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), - "EyeCameraSettings": metadata.get("EyeCameraSettings"), - "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), - "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), - "GazeMappingSettings": metadata.get("GazeMappingSettings"), - "RawDataFilters": metadata.get("RawDataFilters"), - "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), - "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), - "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), - "SoftwareVersion": metadata.get("SoftwareVersion"), - "DeviceSerialNumber": _extract_DeviceSerialNumber(events), - "EyeTrackingMethod": _extract_EyeTrackingMethod(events), - "ManufacturersModelName": _extract_ManufacturersModelName(events), - "AverageCalibrationError": _extract_AverageCalibrationError(df_ms), - "MaximalCalibrationError": _extract_MaximalCalibrationError(df_ms), - "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), - "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced), - "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), - "CalibrationType": _extract_CalibrationType(df_ms_reduced), - "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), - "RecordedEye": _extract_RecordedEye(df_ms_reduced), - "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), - "StartTime": _extract_StartTime(events), - "StopTime": _extract_StopTime(events), - } - output_filename = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_eyetrack", extension="json" - ) - with open(output_filename, "w") as outfile: - json.dump(eyetrack_json, outfile, indent=4) - e2b_log.info(f"file generated: {output_filename}") - # events.json Metadata + # eye-physio.json Metadata + base_json = { + "Columns": [ "x_coordinate", "y_coordinate", "pupil_size", "timestamp"], + "x_coordinate": { + "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u." + }, + "y_coordinate": { + "Description": "Gaze position y-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u." + }, + "pupil_size": { + "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", + "Units": "a.u." + }, + "timestamp": { + "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." + }, + "Manufacturer": "SR-Research", + "ManufacturersModelName": _extract_ManufacturersModelName(events), + "DeviceSerialNumber": _extract_DeviceSerialNumber(events), + "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), + "SoftwareVersion": metadata.get("SoftwareVersion"), + "EyeCameraSettings": metadata.get("EyeCameraSettings"), + "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), + "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), + "GazeMappingSettings": metadata.get("GazeMappingSettings"), + "RawDataFilters": metadata.get("RawDataFilters"), + "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), + "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), + "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), + "EyeTrackingMethod": _extract_EyeTrackingMethod(events), + "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), + "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), + "StartTime": _extract_StartTime(events), + "StopTime": _extract_StopTime(events), + "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), + "CalibrationType": _extract_CalibrationType(df_ms_reduced), + "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), + "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced) + } + + if _2eyesmode(df_ms_reduced) == True: + metadata_eye1 = { + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[0]) + } + + metadata_eye2 = { + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[1::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[1::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[1]) + } + else: + metadata_eye1 = { + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) + } + + json_eye1 = base_json | metadata_eye1 + if _2eyesmode(df_ms_reduced) == True: + json_eye2 = base_json | metadata_eye2 + + # to json + + output_filename_eye1 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="json" + ) + with open(output_filename_eye1, "w") as outfile: + json.dump(json_eye1, outfile, indent=4) + + e2b_log.info(f"file generated: {output_filename_eye1}") + + if _2eyesmode(df_ms_reduced) == True: + output_filename_eye2 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="json" + ) + with open(output_filename_eye2, "w") as outfile: + json.dump(json_eye2, outfile, indent=4) + + e2b_log.info(f"file generated: {output_filename_eye2}") + + # physioevents.json Metadata + events_json = { + "Columns": ["onset", "duration", "trial_type", "blink", "message"], + "Description": "Messages logged by the measurement device", + "ForeignIndexColumn": "timestamp", + "blink": { + "Description": "One indicates if the eye was closed, zero if open." + }, + "message": { + "Description": "String messages logged by the eye-tracker." + }, + "trial_type": { + "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." + }, + "TaskName": _extract_TaskName(events), "InstitutionAddress": metadata.get("InstitutionAddress"), "InstitutionName": metadata.get("InstitutionName"), "StimulusPresentation": { @@ -439,41 +461,68 @@ def edf2bids( "ScreenRefreshRate": metadata.get("ScreenRefreshRate"), "ScreenSize": metadata.get("ScreenSize"), "ScreenResolution": _extract_ScreenResolution(df_ms_reduced), - }, - "TaskName": _extract_TaskName(events), + } } - output_filename = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_events", extension="json" - ) - with open(output_filename, "w") as outfile: + + output_filename_eye1 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", extension="json" + ) + with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) - e2b_log.info(f"file generated: {output_filename}") - # Samples to eyetrack.tsv + e2b_log.info(f"file generated: {output_filename_eye1}") + + if _2eyesmode(df_ms_reduced) == True: + + output_filename_eye2 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physioevents", extension="json" + ) + with open(output_filename_eye2, "w") as outfile: + json.dump(events_json, outfile, indent=4) + + e2b_log.info(f"file generated: {output_filename_eye2}") + + # Samples to dataframe + samples_asc_file = _convert_edf_to_asc_samples(input_file) if not samples_asc_file.exists(): e2b_log.error( "The following .edf input file could not be converted to .asc:" f"{input_file}" ) - eyetrack_tsv = _samples_to_data_frame(samples_asc_file) - # strip blankspcace and convert empty cells to nan - eyetrack_tsv = eyetrack_tsv.applymap(lambda x: x.strip() if isinstance(x, str) else x) - eyetrack_tsv = eyetrack_tsv.replace(".", np.nan, regex=False) - - output_filename = generate_output_filename( - output_dir=output_dir, - input_file=input_file, - suffix="_eyetrack", - extension="tsv.gz", - ) - content = eyetrack_tsv.to_csv(sep="\t", index=False, na_rep="n/a") - with gzip.open(output_filename, "wb") as f: + samples = pd.read_csv(samples_asc_file, sep="\t", header=None) + samples_eye1 = pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]).map(lambda x: x.strip() if isinstance(x, str) else x).replace(".", np.nan, regex=False) + + if _2eyesmode(df_ms_reduced) == True: + samples_eye2 = pd.DataFrame(samples.iloc[:, [4, 5, 6, 0]]) + + # Samples to eye_physio.tsv.gz + + output_filename_eye1 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="tsv.gz" + ) + content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_filename_eye1, "wb") as f: f.write(content.encode()) - e2b_log.info(f"file generated: {output_filename}") + e2b_log.info(f"file generated: {output_filename_eye1}") + + if _2eyesmode(df_ms_reduced) == True: + + output_filename_eye2 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="tsv.gz" + ) + content = samples_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_filename_eye2, "wb") as f: + f.write(content.encode()) + + e2b_log.info(f"file generated: {output_filename_eye2}") + + + # Messages and events to physioevents.tsv.gz - tbc + def generate_output_filename( diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 55fc9f1..9ada11f 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -52,21 +52,21 @@ def test_edf_end_to_end(metadata_file, eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_events.json" + expected_events_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" assert expected_events_sidecar.exists() with open(expected_events_sidecar) as f: events = json.load(f) assert events["StimulusPresentation"]["ScreenResolution"] == [1919, 1079] - expected_data_sidecar = output_dir / f"{input_file.stem}_eyetrack.json" + expected_data_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physio.json" assert expected_data_sidecar.exists() with open(expected_data_sidecar) as f: eyetrack = json.load(f) assert eyetrack["SamplingFrequency"] == 500 assert eyetrack["RecordedEye"] == "Right" - expected_events_sidecar = output_dir / f"{input_file.stem}_eyetrack.tsv.gz" - assert expected_events_sidecar.exists() + expected_events_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + assert expected_events_tsv.exists() @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") @@ -83,15 +83,63 @@ def test_edf_nan_in_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_eyetrack.tsv.gz" - df = pd.read_csv(expected_events_sidecar, sep="\t") - count = sum(i == "." for i in df["eye1_x_coordinate"]) + expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + df = pd.read_csv(expected_eyetrack_tsv, sep="\t") + count = sum(i == "." for i in df[0]) assert count == 0 +@pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") +def test_2files_eye1(eyelink_test_data_dir): + """Check that for datafile with 2eyes 2 eye1 file is created and check input. + + function _2eyesmode + """ + input_dir = eyelink_test_data_dir / "2eyes" + input_file = edf_test_files(input_dir=input_dir)[0] + + output_dir = data_dir() / "output" + output_dir.mkdir(exist_ok=True) + + edf2bids( + input_file=input_file, + output_dir=output_dir, + ) + + expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physio.json" + assert expected_eyetrack_sidecar.exists() + with open(expected_eyetrack_sidecar) as f: + eyetrack = json.load(f) + assert eyetrack["AverageCalibrationError"] == [[0.29]] + assert eyetrack["RecordedEye"] == "Left" + +@pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") +def test_2files_eye2(eyelink_test_data_dir): + """Check that for datafile with 2eyes 2 eye2 file is created and check input. + + function _2eyesmode + """ + input_dir = eyelink_test_data_dir / "2eyes" + input_file = edf_test_files(input_dir=input_dir)[0] + + output_dir = data_dir() / "output" + output_dir.mkdir(exist_ok=True) + + edf2bids( + input_file=input_file, + output_dir=output_dir, + ) + + expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye2_physio.json" + assert expected_eyetrack_sidecar.exists() + with open(expected_eyetrack_sidecar) as f: + eyetrack = json.load(f) + assert eyetrack["AverageCalibrationError"] == [[0.35]] + assert eyetrack["RecordedEye"] == "Right" + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_number_columns_2eyes_tsv(eyelink_test_data_dir): - """Check that values for both eyes were extracted by number of columns. + """Check that values for only one eye were extracted in eye1-physio.tsv.gz by number of columns. function _samples_to_data_frame """ @@ -106,10 +154,10 @@ def test_number_columns_2eyes_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_eyetrack.tsv.gz" - df = pd.read_csv(expected_events_sidecar, sep="\t") + expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + df = pd.read_csv(expected_eyetrack_tsv, sep="\t") number_columns = len(df.columns) - assert number_columns == 7 + assert number_columns == 4 @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") @@ -130,8 +178,8 @@ def test_number_columns_1eye_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}.tsv.gz" - df = pd.read_csv(expected_events_sidecar, sep="\t") + expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + df = pd.read_csv(expected_eyetrack_tsv, sep="\t") number_columns = len(df.columns) assert number_columns == 4 @@ -360,8 +408,8 @@ def test_extract_DeviceSerialNumber(folder, expected, eyelink_test_data_dir): ("pitracker", "Right"), ("rest", "Left"), ("satf", "Right"), - ("vergence", "Both"), - ("2eyes", "Both"), + ("vergence", ["Left", "Right"]), + ("2eyes", ["Left", "Right"]), ], ) def test_extract_RecordedEye(folder, expected, eyelink_test_data_dir): From c462c0a5167da0164512ec86bf1974b6fa49a279 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 Apr 2024 10:36:52 +0200 Subject: [PATCH 02/18] tmp --- eye2bids/2eyes.ipynb | 664 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 664 insertions(+) create mode 100644 eye2bids/2eyes.ipynb diff --git a/eye2bids/2eyes.ipynb b/eye2bids/2eyes.ipynb new file mode 100644 index 0000000..71eabc5 --- /dev/null +++ b/eye2bids/2eyes.ipynb @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "import gzip\n", + "import json\n", + "import subprocess\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import yaml\n", + "from rich.prompt import Prompt\n", + "from yaml.loader import SafeLoader" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "events_asc_file = \"/Users/julia/Desktop/NOWA.nosync/eyelink/2eyes/sub-99_task-FreeView_run-01_eyeData_events.asc\"" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def _load_asc_file(events_asc_file: str | Path) -> list[str]:\n", + " with open(events_asc_file) as f:\n", + " return f.readlines()\n", + "\n", + "\n", + "def _load_asc_file_as_df(events_asc_file: str | Path) -> pd.DataFrame:\n", + " # dataframe for events, all\n", + " events = _load_asc_file(events_asc_file)\n", + " return pd.DataFrame([ms.split() for ms in events if ms.startswith(\"MSG\")])\n", + "\n", + "\n", + "def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame:\n", + " # reduced dataframe without MSG and sample columns\n", + " df_ms = _load_asc_file_as_df(events_asc_file)\n", + " return pd.DataFrame(df_ms.iloc[0:, 2:])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "events = _load_asc_file(events_asc_file)\n", + "df_ms = _load_asc_file_as_df(events_asc_file)\n", + "df_ms_reduced = _load_asc_file_as_reduced_df(events_asc_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def _calibrations(df: pd.DataFrame) -> pd.DataFrame:\n", + " return df[df[3] == \"CALIBRATION\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationType(df: pd.DataFrame) -> list[int]:\n", + " return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationCount(df: pd.DataFrame) -> int:\n", + " if _extract_RecordedEye(df) == \"Both\":\n", + " return len(_calibrations(df)) // 2\n", + " return len(_calibrations(df))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def _get_calibration_positions(df: pd.DataFrame) -> list[int]:\n", + " if _extract_RecordedEye(df) == \"Both\":\n", + " return (\n", + " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", + " .astype(int)\n", + " .tolist()\n", + " )[::2]\n", + " return (\n", + " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", + " .astype(int)\n", + " .tolist()\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationPosition(df: pd.DataFrame) -> list[list[int]]:\n", + " cal_pos = _get_calibration_positions(df)\n", + " cal_num = len(cal_pos) // _extract_CalibrationCount(df)\n", + "\n", + " CalibrationPosition: list[list[int]] = []\n", + "\n", + " if len(cal_pos) == 0:\n", + " return CalibrationPosition\n", + "\n", + " CalibrationPosition.extend(\n", + " cal_pos[i : i + cal_num] for i in range(0, len(cal_pos), cal_num)\n", + " )\n", + " return CalibrationPosition" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationUnit(df: pd.DataFrame) -> str:\n", + " if len(_get_calibration_positions(df)) == 0:\n", + " return \"\"\n", + "\n", + " cal_unit = (\n", + " (df[df[2] == \"VALIDATE\"][[13]])\n", + " .iloc[0:1, 0:1]\n", + " .to_string(header=False, index=False)\n", + " )\n", + " if cal_unit == \"pix.\":\n", + " return \"pixel\"\n", + " elif cal_unit in [\"cm\", \"mm\"]:\n", + " return cal_unit\n", + " return \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_EyeTrackingMethod(events: list[str]) -> str:\n", + " return (\n", + " pd.DataFrame(\n", + " \" \".join([tm for tm in events if tm.startswith(\">>>>>>>\")])\n", + " .replace(\")\", \",\")\n", + " .split(\",\")\n", + " )\n", + " .iloc[1:2]\n", + " .to_string(header=False, index=False)\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_DeviceSerialNumber(events: list[str]) -> str:\n", + " return (\n", + " \" \".join([sl for sl in events if sl.startswith(\"** SERIAL NUMBER:\")])\n", + " .replace(\"** SERIAL NUMBER: \", \"\")\n", + " .replace(\"\\n\", \"\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_PupilFitMethod(df: pd.DataFrame) -> str:\n", + " return (df[df[2] == \"ELCL_PROC\"]).iloc[0:1, 1:2].to_string(header=False, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_SamplingFrequency(df: pd.DataFrame) -> int:\n", + " return int(df[df[2] == \"RECCFG\"].iloc[0:1, 2:3].to_string(header=False, index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def _validations(df: pd.DataFrame) -> pd.DataFrame:\n", + " return df[df[3] == \"VALIDATION\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def _has_validation(df: pd.DataFrame) -> bool:\n", + " return not _validations(df).empty" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_ManufacturersModelName(events: list[str]) -> str:\n", + " return (\n", + " \" \".join([ml for ml in events if ml.startswith(\"** EYELINK\")])\n", + " .replace(\"** \", \"\")\n", + " .replace(\"\\n\", \"\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]:\n", + " list_res = (\n", + " (df[df[2] == \"GAZE_COORDS\"])\n", + " .iloc[0:1, 3:5]\n", + " .to_string(header=False, index=False)\n", + " .replace(\".00\", \"\")\n", + " .split(\" \")\n", + " )\n", + " return [eval(i) for i in list_res]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_TaskName(events: list[str]) -> str:\n", + " return (\n", + " \" \".join([ts for ts in events if ts.startswith(\"** RECORDED BY\")])\n", + " .replace(\"** RECORDED BY \", \"\")\n", + " .replace(\"\\n\", \"\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_StartTime(events: list[str]) -> int:\n", + " StartTime = (\n", + " np.array(pd.DataFrame([st.split() for st in events if st.startswith(\"START\")])[1])\n", + " .astype(int)\n", + " .tolist()\n", + " )\n", + " if len(StartTime) > 1:\n", + " e2b_log.info(\n", + " \"\"\"Your input file contains multiple start times.\\n\n", + " As this is not seen as good practice in eyetracking experiments, \\n\n", + " only the first start time will be kept for the metadata file. \\n\n", + " Please consider changing your code accordingly\n", + " for future eyetracking experiments.\\n\"\"\"\n", + " )\n", + " return StartTime[0]\n", + " return StartTime" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_StopTime(events: list[str]) -> int:\n", + " StopTime = (\n", + " np.array(pd.DataFrame([so.split() for so in events if so.startswith(\"END\")])[1])\n", + " .astype(int)\n", + " .tolist()\n", + " )\n", + " if len(StopTime) > 1:\n", + " e2b_log.info(\n", + " \"\"\"Your input file contains multiple stop times.\\n\n", + " As this is not seen as good practice in eyetracking experiments, \\n\n", + " only the last stop time will be kept for the metadata file. \\n\n", + " Please consider changing your code accordingly\n", + " for future eyetracking experiments.\\n\"\"\"\n", + " )\n", + " return StopTime[-1]\n", + " return StopTime" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", + " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", + " if eye == \"L\":\n", + " return \"Left\"\n", + " elif eye == \"R\":\n", + " return \"Right\"\n", + " elif eye == \"LR\":\n", + " return [\"Left\", \"Right\"]\n", + " return \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CLG-BAF38 P-CR EYELINK II CL v5.12 May 12 2017 pixel HV13 CENTROID 1000 [767979] [819652]\n" + ] + } + ], + "source": [ + "DeviceSerialNumber = _extract_DeviceSerialNumber(events)\n", + "EyeTrackingMethod = _extract_EyeTrackingMethod(events)\n", + "ManufacturersModelName = _extract_ManufacturersModelName(events)\n", + "CalibrationUnit = _extract_CalibrationUnit(df_ms_reduced)\n", + "CalibrationType = _extract_CalibrationType(df_ms_reduced)\n", + "PupilFitMethod = _extract_PupilFitMethod(df_ms_reduced)\n", + "SamplingFrequency = _extract_SamplingFrequency(df_ms_reduced)\n", + "StartTime = _extract_StartTime(events)\n", + "StopTime = _extract_StopTime(events)\n", + "\n", + "print(DeviceSerialNumber, EyeTrackingMethod, ManufacturersModelName, CalibrationUnit, CalibrationType, PupilFitMethod, SamplingFrequency, StartTime, StopTime)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "def _2eyesmode(df: pd.DataFrame) -> bool:\n", + " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", + " if eye == \"LR\":\n", + " two_eyes = True\n", + " return two_eyes" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_2eyesmode(df_ms_reduced)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", + " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", + " if eye == \"L\":\n", + " return eye1\n", + " elif eye == \"R\":\n", + " return \"Right\"\n", + " elif eye == \"LR\":\n", + " return [\"Left\", \"Right\"]\n", + " return \"\" " + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]:\n", + " if not _has_validation(df):\n", + " return []\n", + " if _extract_CalibrationCount(df) > 1:\n", + " return np.array(_validations(df)[[9]]).astype(float).tolist()\n", + " return np.array(_validations(df)[[9]]).astype(float).tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]:\n", + " if not _has_validation(df):\n", + " return []\n", + " return np.array(_validations(df)[[11]]).astype(float).tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [], + "source": [ + "if _2eyesmode(df_ms_reduced) == True:\n", + " json_eye1 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", + " }\n", + " json_eye2 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", + " }\n", + "else: json_eye1 = {\n", + " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", + " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", + " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", + " }\n" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'RecordedEye': 'Left',\n", + " 'AverageCalibrationError': [[0.29]],\n", + " 'MaximalCalibrationError': [[0.62]]}" + ] + }, + "execution_count": 98, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_eye1" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'RecordedEye': 'Right',\n", + " 'AverageCalibrationError': [[0.35]],\n", + " 'MaximalCalibrationError': [[1.21]]}" + ] + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_eye2" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": {}, + "outputs": [], + "source": [ + "base_json = {\n", + " \"Manufacturer\": \"SR-Research\",\n", + " \"DeviceSerialNumber\": _extract_DeviceSerialNumber(events),\n", + " \"EyeTrackingMethod\": _extract_EyeTrackingMethod(events),\n", + " \"ManufacturersModelName\": _extract_ManufacturersModelName(events),\n", + " \"CalibrationUnit\": _extract_CalibrationUnit(df_ms_reduced),\n", + " \"CalibrationType\": _extract_CalibrationType(df_ms_reduced),\n", + " \"PupilFitMethod\": _extract_PupilFitMethod(df_ms_reduced),\n", + " \"SamplingFrequency\": _extract_SamplingFrequency(df_ms_reduced),\n", + " \"StartTime\": _extract_StartTime(events),\n", + " \"StopTime\": _extract_StopTime(events),\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [], + "source": [ + "if _2eyesmode(df_ms_reduced) == True:\n", + " metadata_eye1 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", + " }\n", + " metadata_eye2 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", + " }\n", + "else: \n", + " metadata_eye1 = {\n", + " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", + " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", + " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Manufacturer': 'SR-Research',\n", + " 'DeviceSerialNumber': 'CLG-BAF38',\n", + " 'EyeTrackingMethod': 'P-CR',\n", + " 'ManufacturersModelName': 'EYELINK II CL v5.12 May 12 2017',\n", + " 'CalibrationUnit': 'pixel',\n", + " 'CalibrationType': 'HV13',\n", + " 'PupilFitMethod': 'CENTROID',\n", + " 'SamplingFrequency': 1000,\n", + " 'StartTime': [767979],\n", + " 'StopTime': [819652],\n", + " 'RecordedEye': 'Left',\n", + " 'AverageCalibrationError': [[0.29]],\n", + " 'MaximalCalibrationError': [[0.62]]}" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_eye1 = base_json | metadata_eye1\n", + "json_eye1" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_output_filename(\n", + " output_dir: Path, input_file: Path, suffix: str, extension: str\n", + ") -> Path:\n", + " \"\"\"Generate output filename.\"\"\"\n", + " filename = Path(input_file).stem\n", + " if filename.endswith(suffix):\n", + " suffix = \"\"\n", + " return output_dir / f\"{filename}{suffix}.{extension}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "metadata": {}, + "outputs": [], + "source": [ + "if _2eyesmode == True:\n", + " output_filename_eye1 = generate_output_filename(\n", + " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", + " )\n", + " with open(output_filename_eye1, \"w\") as outfile:\n", + " json.dump(json_eye1, outfile, indent=4)\n", + "\n", + " output_filename_eye2 = generate_output_filename(\n", + " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye2_physio\", extension=\"json\"\n", + " )\n", + " with open(output_filename_eye2, \"w\") as outfile:\n", + " json.dump(json_eye2, outfile, indent=4)\n", + " \n", + " #e2b_log.info(f\"files generated: {output_filename_eye1} and {output_filename_eye2}\")\n", + "\n", + "else:\n", + " output_filename_eye1 = generate_output_filename(\n", + " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", + " )\n", + " with open(output_filename_eye1, \"w\") as outfile:\n", + " json.dump(json_eye1, outfile, indent=4)\n", + " \n", + " #e2b_log.info(f\"file generated: {output_filename_eye1}\")\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 3ef81562df8d7da73edcdebe4f16b667e95dc569 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 08:32:01 +0200 Subject: [PATCH 03/18] [pre-commit.ci] pre-commit autoupdate (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e202ac..be642ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-ast - id: check-case-conflict From fe7df431a1fe6ca3a109113003e204880dfeb372 Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Wed, 10 Apr 2024 17:56:57 +0200 Subject: [PATCH 04/18] ignore test ipynb --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1aa6ebe..e58cc53 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ eye2bids/_version.py *events.json *eyetrack.json *eyetrack.tsv +2eyes.ipynb tmp # General From 9244429e01dea1755a32368ccbdc683caaab487b Mon Sep 17 00:00:00 2001 From: Julia-Katharina Pfarr <111446107+julia-pfarr@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:58:04 +0200 Subject: [PATCH 05/18] Delete eye2bids/2eyes.ipynb --- eye2bids/2eyes.ipynb | 664 ------------------------------------------- 1 file changed, 664 deletions(-) delete mode 100644 eye2bids/2eyes.ipynb diff --git a/eye2bids/2eyes.ipynb b/eye2bids/2eyes.ipynb deleted file mode 100644 index 71eabc5..0000000 --- a/eye2bids/2eyes.ipynb +++ /dev/null @@ -1,664 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "\n", - "import gzip\n", - "import json\n", - "import subprocess\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import yaml\n", - "from rich.prompt import Prompt\n", - "from yaml.loader import SafeLoader" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "events_asc_file = \"/Users/julia/Desktop/NOWA.nosync/eyelink/2eyes/sub-99_task-FreeView_run-01_eyeData_events.asc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def _load_asc_file(events_asc_file: str | Path) -> list[str]:\n", - " with open(events_asc_file) as f:\n", - " return f.readlines()\n", - "\n", - "\n", - "def _load_asc_file_as_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # dataframe for events, all\n", - " events = _load_asc_file(events_asc_file)\n", - " return pd.DataFrame([ms.split() for ms in events if ms.startswith(\"MSG\")])\n", - "\n", - "\n", - "def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # reduced dataframe without MSG and sample columns\n", - " df_ms = _load_asc_file_as_df(events_asc_file)\n", - " return pd.DataFrame(df_ms.iloc[0:, 2:])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "events = _load_asc_file(events_asc_file)\n", - "df_ms = _load_asc_file_as_df(events_asc_file)\n", - "df_ms_reduced = _load_asc_file_as_reduced_df(events_asc_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def _calibrations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"CALIBRATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationType(df: pd.DataFrame) -> list[int]:\n", - " return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationCount(df: pd.DataFrame) -> int:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return len(_calibrations(df)) // 2\n", - " return len(_calibrations(df))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def _get_calibration_positions(df: pd.DataFrame) -> list[int]:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )[::2]\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationPosition(df: pd.DataFrame) -> list[list[int]]:\n", - " cal_pos = _get_calibration_positions(df)\n", - " cal_num = len(cal_pos) // _extract_CalibrationCount(df)\n", - "\n", - " CalibrationPosition: list[list[int]] = []\n", - "\n", - " if len(cal_pos) == 0:\n", - " return CalibrationPosition\n", - "\n", - " CalibrationPosition.extend(\n", - " cal_pos[i : i + cal_num] for i in range(0, len(cal_pos), cal_num)\n", - " )\n", - " return CalibrationPosition" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationUnit(df: pd.DataFrame) -> str:\n", - " if len(_get_calibration_positions(df)) == 0:\n", - " return \"\"\n", - "\n", - " cal_unit = (\n", - " (df[df[2] == \"VALIDATE\"][[13]])\n", - " .iloc[0:1, 0:1]\n", - " .to_string(header=False, index=False)\n", - " )\n", - " if cal_unit == \"pix.\":\n", - " return \"pixel\"\n", - " elif cal_unit in [\"cm\", \"mm\"]:\n", - " return cal_unit\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_EyeTrackingMethod(events: list[str]) -> str:\n", - " return (\n", - " pd.DataFrame(\n", - " \" \".join([tm for tm in events if tm.startswith(\">>>>>>>\")])\n", - " .replace(\")\", \",\")\n", - " .split(\",\")\n", - " )\n", - " .iloc[1:2]\n", - " .to_string(header=False, index=False)\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_DeviceSerialNumber(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([sl for sl in events if sl.startswith(\"** SERIAL NUMBER:\")])\n", - " .replace(\"** SERIAL NUMBER: \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_PupilFitMethod(df: pd.DataFrame) -> str:\n", - " return (df[df[2] == \"ELCL_PROC\"]).iloc[0:1, 1:2].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_SamplingFrequency(df: pd.DataFrame) -> int:\n", - " return int(df[df[2] == \"RECCFG\"].iloc[0:1, 2:3].to_string(header=False, index=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def _validations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"VALIDATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "def _has_validation(df: pd.DataFrame) -> bool:\n", - " return not _validations(df).empty" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ManufacturersModelName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ml for ml in events if ml.startswith(\"** EYELINK\")])\n", - " .replace(\"** \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]:\n", - " list_res = (\n", - " (df[df[2] == \"GAZE_COORDS\"])\n", - " .iloc[0:1, 3:5]\n", - " .to_string(header=False, index=False)\n", - " .replace(\".00\", \"\")\n", - " .split(\" \")\n", - " )\n", - " return [eval(i) for i in list_res]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_TaskName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ts for ts in events if ts.startswith(\"** RECORDED BY\")])\n", - " .replace(\"** RECORDED BY \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StartTime(events: list[str]) -> int:\n", - " StartTime = (\n", - " np.array(pd.DataFrame([st.split() for st in events if st.startswith(\"START\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StartTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple start times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the first start time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StartTime[0]\n", - " return StartTime" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StopTime(events: list[str]) -> int:\n", - " StopTime = (\n", - " np.array(pd.DataFrame([so.split() for so in events if so.startswith(\"END\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StopTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple stop times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the last stop time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StopTime[-1]\n", - " return StopTime" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return \"Left\"\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CLG-BAF38 P-CR EYELINK II CL v5.12 May 12 2017 pixel HV13 CENTROID 1000 [767979] [819652]\n" - ] - } - ], - "source": [ - "DeviceSerialNumber = _extract_DeviceSerialNumber(events)\n", - "EyeTrackingMethod = _extract_EyeTrackingMethod(events)\n", - "ManufacturersModelName = _extract_ManufacturersModelName(events)\n", - "CalibrationUnit = _extract_CalibrationUnit(df_ms_reduced)\n", - "CalibrationType = _extract_CalibrationType(df_ms_reduced)\n", - "PupilFitMethod = _extract_PupilFitMethod(df_ms_reduced)\n", - "SamplingFrequency = _extract_SamplingFrequency(df_ms_reduced)\n", - "StartTime = _extract_StartTime(events)\n", - "StopTime = _extract_StopTime(events)\n", - "\n", - "print(DeviceSerialNumber, EyeTrackingMethod, ManufacturersModelName, CalibrationUnit, CalibrationType, PupilFitMethod, SamplingFrequency, StartTime, StopTime)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "def _2eyesmode(df: pd.DataFrame) -> bool:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"LR\":\n", - " two_eyes = True\n", - " return two_eyes" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "_2eyesmode(df_ms_reduced)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return eye1\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\" " - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " if _extract_CalibrationCount(df) > 1:\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " return np.array(_validations(df)[[11]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " json_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " json_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: json_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }\n" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Right',\n", - " 'AverageCalibrationError': [[0.35]],\n", - " 'MaximalCalibrationError': [[1.21]]}" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye2" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "base_json = {\n", - " \"Manufacturer\": \"SR-Research\",\n", - " \"DeviceSerialNumber\": _extract_DeviceSerialNumber(events),\n", - " \"EyeTrackingMethod\": _extract_EyeTrackingMethod(events),\n", - " \"ManufacturersModelName\": _extract_ManufacturersModelName(events),\n", - " \"CalibrationUnit\": _extract_CalibrationUnit(df_ms_reduced),\n", - " \"CalibrationType\": _extract_CalibrationType(df_ms_reduced),\n", - " \"PupilFitMethod\": _extract_PupilFitMethod(df_ms_reduced),\n", - " \"SamplingFrequency\": _extract_SamplingFrequency(df_ms_reduced),\n", - " \"StartTime\": _extract_StartTime(events),\n", - " \"StopTime\": _extract_StopTime(events),\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " metadata_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " metadata_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: \n", - " metadata_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Manufacturer': 'SR-Research',\n", - " 'DeviceSerialNumber': 'CLG-BAF38',\n", - " 'EyeTrackingMethod': 'P-CR',\n", - " 'ManufacturersModelName': 'EYELINK II CL v5.12 May 12 2017',\n", - " 'CalibrationUnit': 'pixel',\n", - " 'CalibrationType': 'HV13',\n", - " 'PupilFitMethod': 'CENTROID',\n", - " 'SamplingFrequency': 1000,\n", - " 'StartTime': [767979],\n", - " 'StopTime': [819652],\n", - " 'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1 = base_json | metadata_eye1\n", - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_output_filename(\n", - " output_dir: Path, input_file: Path, suffix: str, extension: str\n", - ") -> Path:\n", - " \"\"\"Generate output filename.\"\"\"\n", - " filename = Path(input_file).stem\n", - " if filename.endswith(suffix):\n", - " suffix = \"\"\n", - " return output_dir / f\"{filename}{suffix}.{extension}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode == True:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - "\n", - " output_filename_eye2 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye2_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye2, \"w\") as outfile:\n", - " json.dump(json_eye2, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"files generated: {output_filename_eye1} and {output_filename_eye2}\")\n", - "\n", - "else:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"file generated: {output_filename_eye1}\")\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From d5bef7fe3f719deb0a66371d04f4dea1624c31e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:04:06 +0000 Subject: [PATCH 06/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- eye2bids/edf2bids.py | 190 ++++++++++++++++++++++------------------- tests/test_edf2bids.py | 14 ++- 2 files changed, 115 insertions(+), 89 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 39dddbf..97a6b96 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -115,6 +115,7 @@ def _convert_edf_to_asc_samples(input_file: str | Path) -> Path: subprocess.run(["edf2asc", "-y", "-s", input_file, "-o", samples_asc_file]) return Path(samples_asc_file).with_suffix(".asc") + def _2eyesmode(df: pd.DataFrame) -> bool: eye = df[df[2] == "RECCFG"].iloc[0:1, 5:6].to_string(header=False, index=False) if eye == "LR": @@ -123,9 +124,11 @@ def _2eyesmode(df: pd.DataFrame) -> bool: two_eyes = False return two_eyes + def _calibrations(df: pd.DataFrame) -> pd.DataFrame: return df[df[3] == "CALIBRATION"] + def _extract_CalibrationType(df: pd.DataFrame) -> list[int]: return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False) @@ -245,7 +248,7 @@ def _extract_RecordedEye(df: pd.DataFrame) -> str: return "Right" elif eye == "LR": return ["Left", "Right"] - return "" + return "" def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]: @@ -319,6 +322,7 @@ def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame: df_ms = _load_asc_file_as_df(events_asc_file) return pd.DataFrame(df_ms.iloc[0:, 2:]) + def edf2bids( input_file: str | Path | None = None, metadata_file: str | Path | None = None, @@ -352,89 +356,94 @@ def edf2bids( with open(metadata_file) as f: metadata = yaml.load(f, Loader=SafeLoader) - # eye-physio.json Metadata base_json = { - "Columns": [ "x_coordinate", "y_coordinate", "pupil_size", "timestamp"], - "x_coordinate": { - "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", - "Units": "a.u." - }, - "y_coordinate": { - "Description": "Gaze position y-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", - "Units": "a.u." - }, - "pupil_size": { - "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", - "Units": "a.u." - }, - "timestamp": { - "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." - }, - "Manufacturer": "SR-Research", - "ManufacturersModelName": _extract_ManufacturersModelName(events), - "DeviceSerialNumber": _extract_DeviceSerialNumber(events), - "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), - "SoftwareVersion": metadata.get("SoftwareVersion"), - "EyeCameraSettings": metadata.get("EyeCameraSettings"), - "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), - "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), - "GazeMappingSettings": metadata.get("GazeMappingSettings"), - "RawDataFilters": metadata.get("RawDataFilters"), - "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), - "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), - "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), - "EyeTrackingMethod": _extract_EyeTrackingMethod(events), - "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), - "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), - "StartTime": _extract_StartTime(events), - "StopTime": _extract_StopTime(events), - "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), - "CalibrationType": _extract_CalibrationType(df_ms_reduced), - "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), - "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced) - } - + "Columns": ["x_coordinate", "y_coordinate", "pupil_size", "timestamp"], + "x_coordinate": { + "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u.", + }, + "y_coordinate": { + "Description": "Gaze position y-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u.", + }, + "pupil_size": { + "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", + "Units": "a.u.", + }, + "timestamp": { + "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." + }, + "Manufacturer": "SR-Research", + "ManufacturersModelName": _extract_ManufacturersModelName(events), + "DeviceSerialNumber": _extract_DeviceSerialNumber(events), + "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), + "SoftwareVersion": metadata.get("SoftwareVersion"), + "EyeCameraSettings": metadata.get("EyeCameraSettings"), + "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), + "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), + "GazeMappingSettings": metadata.get("GazeMappingSettings"), + "RawDataFilters": metadata.get("RawDataFilters"), + "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), + "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), + "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), + "EyeTrackingMethod": _extract_EyeTrackingMethod(events), + "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), + "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), + "StartTime": _extract_StartTime(events), + "StopTime": _extract_StopTime(events), + "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), + "CalibrationType": _extract_CalibrationType(df_ms_reduced), + "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), + "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced), + } + if _2eyesmode(df_ms_reduced) == True: metadata_eye1 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[0]) - } - + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[0]), + } + metadata_eye2 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[1::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[1::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[1]) + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[1::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[1::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[1]), } else: metadata_eye1 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) - } - + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)), + } + json_eye1 = base_json | metadata_eye1 if _2eyesmode(df_ms_reduced) == True: json_eye2 = base_json | metadata_eye2 # to json - + output_filename_eye1 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physio", + extension="json", + ) with open(output_filename_eye1, "w") as outfile: - json.dump(json_eye1, outfile, indent=4) + json.dump(json_eye1, outfile, indent=4) e2b_log.info(f"file generated: {output_filename_eye1}") if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physio", + extension="json", + ) with open(output_filename_eye2, "w") as outfile: json.dump(json_eye2, outfile, indent=4) - + e2b_log.info(f"file generated: {output_filename_eye2}") # physioevents.json Metadata @@ -443,16 +452,12 @@ def edf2bids( "Columns": ["onset", "duration", "trial_type", "blink", "message"], "Description": "Messages logged by the measurement device", "ForeignIndexColumn": "timestamp", - "blink": { - "Description": "One indicates if the eye was closed, zero if open." - }, - "message": { - "Description": "String messages logged by the eye-tracker." - }, + "blink": {"Description": "One indicates if the eye was closed, zero if open."}, + "message": {"Description": "String messages logged by the eye-tracker."}, "trial_type": { - "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." + "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." }, - "TaskName": _extract_TaskName(events), + "TaskName": _extract_TaskName(events), "InstitutionAddress": metadata.get("InstitutionAddress"), "InstitutionName": metadata.get("InstitutionName"), "StimulusPresentation": { @@ -460,13 +465,15 @@ def edf2bids( "ScreenRefreshRate": metadata.get("ScreenRefreshRate"), "ScreenSize": metadata.get("ScreenSize"), "ScreenResolution": _extract_ScreenResolution(df_ms_reduced), - } + }, } - output_filename_eye1 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physioevents", + extension="json", + ) with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) @@ -475,11 +482,14 @@ def edf2bids( if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physioevents", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physioevents", + extension="json", + ) with open(output_filename_eye2, "w") as outfile: json.dump(events_json, outfile, indent=4) - + e2b_log.info(f"file generated: {output_filename_eye2}") # Samples to dataframe @@ -492,16 +502,23 @@ def edf2bids( ) samples = pd.read_csv(samples_asc_file, sep="\t", header=None) - samples_eye1 = pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]).map(lambda x: x.strip() if isinstance(x, str) else x).replace(".", np.nan, regex=False) - + samples_eye1 = ( + pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]) + .map(lambda x: x.strip() if isinstance(x, str) else x) + .replace(".", np.nan, regex=False) + ) + if _2eyesmode(df_ms_reduced) == True: samples_eye2 = pd.DataFrame(samples.iloc[:, [4, 5, 6, 0]]) # Samples to eye_physio.tsv.gz output_filename_eye1 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="tsv.gz" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physio", + extension="tsv.gz", + ) content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) with gzip.open(output_filename_eye1, "wb") as f: f.write(content.encode()) @@ -511,19 +528,20 @@ def edf2bids( if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="tsv.gz" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physio", + extension="tsv.gz", + ) content = samples_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) with gzip.open(output_filename_eye2, "wb") as f: f.write(content.encode()) - - e2b_log.info(f"file generated: {output_filename_eye2}") + e2b_log.info(f"file generated: {output_filename_eye2}") # Messages and events to physioevents.tsv.gz - tbc - def generate_output_filename( output_dir: Path, input_file: Path, suffix: str, extension: str ) -> Path: diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 9ada11f..e3b57a8 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -52,7 +52,9 @@ def test_edf_end_to_end(metadata_file, eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" + expected_events_sidecar = ( + output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" + ) assert expected_events_sidecar.exists() with open(expected_events_sidecar) as f: events = json.load(f) @@ -88,6 +90,7 @@ def test_edf_nan_in_tsv(eyelink_test_data_dir): count = sum(i == "." for i in df[0]) assert count == 0 + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_2files_eye1(eyelink_test_data_dir): """Check that for datafile with 2eyes 2 eye1 file is created and check input. @@ -105,13 +108,16 @@ def test_2files_eye1(eyelink_test_data_dir): output_dir=output_dir, ) - expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physio.json" + expected_eyetrack_sidecar = ( + output_dir / f"{input_file.stem}_recording-eye1_physio.json" + ) assert expected_eyetrack_sidecar.exists() with open(expected_eyetrack_sidecar) as f: eyetrack = json.load(f) assert eyetrack["AverageCalibrationError"] == [[0.29]] assert eyetrack["RecordedEye"] == "Left" + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_2files_eye2(eyelink_test_data_dir): """Check that for datafile with 2eyes 2 eye2 file is created and check input. @@ -129,7 +135,9 @@ def test_2files_eye2(eyelink_test_data_dir): output_dir=output_dir, ) - expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye2_physio.json" + expected_eyetrack_sidecar = ( + output_dir / f"{input_file.stem}_recording-eye2_physio.json" + ) assert expected_eyetrack_sidecar.exists() with open(expected_eyetrack_sidecar) as f: eyetrack = json.load(f) From b5f971a7d02df1c9cb35595593fe8549e28cbe6f Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Thu, 11 Apr 2024 11:00:58 +0200 Subject: [PATCH 07/18] change how json dicts are combined because tests were failing --- eye2bids/2eyes.ipynb | 664 ------------------------------------------- eye2bids/edf2bids.py | 4 +- 2 files changed, 2 insertions(+), 666 deletions(-) delete mode 100644 eye2bids/2eyes.ipynb diff --git a/eye2bids/2eyes.ipynb b/eye2bids/2eyes.ipynb deleted file mode 100644 index 71eabc5..0000000 --- a/eye2bids/2eyes.ipynb +++ /dev/null @@ -1,664 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "\n", - "import gzip\n", - "import json\n", - "import subprocess\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import yaml\n", - "from rich.prompt import Prompt\n", - "from yaml.loader import SafeLoader" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "events_asc_file = \"/Users/julia/Desktop/NOWA.nosync/eyelink/2eyes/sub-99_task-FreeView_run-01_eyeData_events.asc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def _load_asc_file(events_asc_file: str | Path) -> list[str]:\n", - " with open(events_asc_file) as f:\n", - " return f.readlines()\n", - "\n", - "\n", - "def _load_asc_file_as_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # dataframe for events, all\n", - " events = _load_asc_file(events_asc_file)\n", - " return pd.DataFrame([ms.split() for ms in events if ms.startswith(\"MSG\")])\n", - "\n", - "\n", - "def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # reduced dataframe without MSG and sample columns\n", - " df_ms = _load_asc_file_as_df(events_asc_file)\n", - " return pd.DataFrame(df_ms.iloc[0:, 2:])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "events = _load_asc_file(events_asc_file)\n", - "df_ms = _load_asc_file_as_df(events_asc_file)\n", - "df_ms_reduced = _load_asc_file_as_reduced_df(events_asc_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def _calibrations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"CALIBRATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationType(df: pd.DataFrame) -> list[int]:\n", - " return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationCount(df: pd.DataFrame) -> int:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return len(_calibrations(df)) // 2\n", - " return len(_calibrations(df))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def _get_calibration_positions(df: pd.DataFrame) -> list[int]:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )[::2]\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationPosition(df: pd.DataFrame) -> list[list[int]]:\n", - " cal_pos = _get_calibration_positions(df)\n", - " cal_num = len(cal_pos) // _extract_CalibrationCount(df)\n", - "\n", - " CalibrationPosition: list[list[int]] = []\n", - "\n", - " if len(cal_pos) == 0:\n", - " return CalibrationPosition\n", - "\n", - " CalibrationPosition.extend(\n", - " cal_pos[i : i + cal_num] for i in range(0, len(cal_pos), cal_num)\n", - " )\n", - " return CalibrationPosition" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationUnit(df: pd.DataFrame) -> str:\n", - " if len(_get_calibration_positions(df)) == 0:\n", - " return \"\"\n", - "\n", - " cal_unit = (\n", - " (df[df[2] == \"VALIDATE\"][[13]])\n", - " .iloc[0:1, 0:1]\n", - " .to_string(header=False, index=False)\n", - " )\n", - " if cal_unit == \"pix.\":\n", - " return \"pixel\"\n", - " elif cal_unit in [\"cm\", \"mm\"]:\n", - " return cal_unit\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_EyeTrackingMethod(events: list[str]) -> str:\n", - " return (\n", - " pd.DataFrame(\n", - " \" \".join([tm for tm in events if tm.startswith(\">>>>>>>\")])\n", - " .replace(\")\", \",\")\n", - " .split(\",\")\n", - " )\n", - " .iloc[1:2]\n", - " .to_string(header=False, index=False)\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_DeviceSerialNumber(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([sl for sl in events if sl.startswith(\"** SERIAL NUMBER:\")])\n", - " .replace(\"** SERIAL NUMBER: \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_PupilFitMethod(df: pd.DataFrame) -> str:\n", - " return (df[df[2] == \"ELCL_PROC\"]).iloc[0:1, 1:2].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_SamplingFrequency(df: pd.DataFrame) -> int:\n", - " return int(df[df[2] == \"RECCFG\"].iloc[0:1, 2:3].to_string(header=False, index=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def _validations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"VALIDATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "def _has_validation(df: pd.DataFrame) -> bool:\n", - " return not _validations(df).empty" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ManufacturersModelName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ml for ml in events if ml.startswith(\"** EYELINK\")])\n", - " .replace(\"** \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]:\n", - " list_res = (\n", - " (df[df[2] == \"GAZE_COORDS\"])\n", - " .iloc[0:1, 3:5]\n", - " .to_string(header=False, index=False)\n", - " .replace(\".00\", \"\")\n", - " .split(\" \")\n", - " )\n", - " return [eval(i) for i in list_res]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_TaskName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ts for ts in events if ts.startswith(\"** RECORDED BY\")])\n", - " .replace(\"** RECORDED BY \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StartTime(events: list[str]) -> int:\n", - " StartTime = (\n", - " np.array(pd.DataFrame([st.split() for st in events if st.startswith(\"START\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StartTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple start times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the first start time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StartTime[0]\n", - " return StartTime" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StopTime(events: list[str]) -> int:\n", - " StopTime = (\n", - " np.array(pd.DataFrame([so.split() for so in events if so.startswith(\"END\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StopTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple stop times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the last stop time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StopTime[-1]\n", - " return StopTime" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return \"Left\"\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CLG-BAF38 P-CR EYELINK II CL v5.12 May 12 2017 pixel HV13 CENTROID 1000 [767979] [819652]\n" - ] - } - ], - "source": [ - "DeviceSerialNumber = _extract_DeviceSerialNumber(events)\n", - "EyeTrackingMethod = _extract_EyeTrackingMethod(events)\n", - "ManufacturersModelName = _extract_ManufacturersModelName(events)\n", - "CalibrationUnit = _extract_CalibrationUnit(df_ms_reduced)\n", - "CalibrationType = _extract_CalibrationType(df_ms_reduced)\n", - "PupilFitMethod = _extract_PupilFitMethod(df_ms_reduced)\n", - "SamplingFrequency = _extract_SamplingFrequency(df_ms_reduced)\n", - "StartTime = _extract_StartTime(events)\n", - "StopTime = _extract_StopTime(events)\n", - "\n", - "print(DeviceSerialNumber, EyeTrackingMethod, ManufacturersModelName, CalibrationUnit, CalibrationType, PupilFitMethod, SamplingFrequency, StartTime, StopTime)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "def _2eyesmode(df: pd.DataFrame) -> bool:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"LR\":\n", - " two_eyes = True\n", - " return two_eyes" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "_2eyesmode(df_ms_reduced)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return eye1\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\" " - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " if _extract_CalibrationCount(df) > 1:\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " return np.array(_validations(df)[[11]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " json_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " json_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: json_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }\n" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Right',\n", - " 'AverageCalibrationError': [[0.35]],\n", - " 'MaximalCalibrationError': [[1.21]]}" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye2" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "base_json = {\n", - " \"Manufacturer\": \"SR-Research\",\n", - " \"DeviceSerialNumber\": _extract_DeviceSerialNumber(events),\n", - " \"EyeTrackingMethod\": _extract_EyeTrackingMethod(events),\n", - " \"ManufacturersModelName\": _extract_ManufacturersModelName(events),\n", - " \"CalibrationUnit\": _extract_CalibrationUnit(df_ms_reduced),\n", - " \"CalibrationType\": _extract_CalibrationType(df_ms_reduced),\n", - " \"PupilFitMethod\": _extract_PupilFitMethod(df_ms_reduced),\n", - " \"SamplingFrequency\": _extract_SamplingFrequency(df_ms_reduced),\n", - " \"StartTime\": _extract_StartTime(events),\n", - " \"StopTime\": _extract_StopTime(events),\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " metadata_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " metadata_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: \n", - " metadata_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Manufacturer': 'SR-Research',\n", - " 'DeviceSerialNumber': 'CLG-BAF38',\n", - " 'EyeTrackingMethod': 'P-CR',\n", - " 'ManufacturersModelName': 'EYELINK II CL v5.12 May 12 2017',\n", - " 'CalibrationUnit': 'pixel',\n", - " 'CalibrationType': 'HV13',\n", - " 'PupilFitMethod': 'CENTROID',\n", - " 'SamplingFrequency': 1000,\n", - " 'StartTime': [767979],\n", - " 'StopTime': [819652],\n", - " 'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1 = base_json | metadata_eye1\n", - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_output_filename(\n", - " output_dir: Path, input_file: Path, suffix: str, extension: str\n", - ") -> Path:\n", - " \"\"\"Generate output filename.\"\"\"\n", - " filename = Path(input_file).stem\n", - " if filename.endswith(suffix):\n", - " suffix = \"\"\n", - " return output_dir / f\"{filename}{suffix}.{extension}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode == True:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - "\n", - " output_filename_eye2 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye2_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye2, \"w\") as outfile:\n", - " json.dump(json_eye2, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"files generated: {output_filename_eye1} and {output_filename_eye2}\")\n", - "\n", - "else:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"file generated: {output_filename_eye1}\")\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 39dddbf..2e3e0e1 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -414,9 +414,9 @@ def edf2bids( "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) } - json_eye1 = base_json | metadata_eye1 + json_eye1 = dict(base_json, **metadata_eye1) if _2eyesmode(df_ms_reduced) == True: - json_eye2 = base_json | metadata_eye2 + json_eye2 = dict(base_json, **metadata_eye2) # to json From 56cca43f8ae9852442dde4ce099a11e32bff31d1 Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Thu, 11 Apr 2024 12:43:58 +0200 Subject: [PATCH 08/18] ignore *ipynb --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e58cc53..6f0dbbd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ eye2bids/_version.py *events.json *eyetrack.json *eyetrack.tsv -2eyes.ipynb +*.ipynb tmp # General From f79d237c242df7b73418695230eae6747609117e Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Thu, 18 Apr 2024 15:14:54 +0200 Subject: [PATCH 09/18] add PhysioType metadata --- eye2bids/edf2bids.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 6b968df..0284eb9 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -358,6 +358,7 @@ def edf2bids( # eye-physio.json Metadata base_json = { + "PhysioType": "eyetrack", "Columns": ["x_coordinate", "y_coordinate", "pupil_size", "timestamp"], "x_coordinate": { "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", @@ -378,7 +379,7 @@ def edf2bids( "ManufacturersModelName": _extract_ManufacturersModelName(events), "DeviceSerialNumber": _extract_DeviceSerialNumber(events), "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), - "SoftwareVersion": metadata.get("SoftwareVersion"), + "SoftwareVersions": metadata.get("SoftwareVersion"), "EyeCameraSettings": metadata.get("EyeCameraSettings"), "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), From e0dd90e8dc9e860c3c496a50fb394c41e6b22267 Mon Sep 17 00:00:00 2001 From: julia-pfarr Date: Mon, 13 May 2024 16:36:47 +0200 Subject: [PATCH 10/18] resolve merge residuals --- eye2bids/edf2bids.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index f5fd486..b93304c 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -134,14 +134,12 @@ def _extract_CalibrationType(df: pd.DataFrame) -> list[int]: def _extract_CalibrationCount(df: pd.DataFrame) -> int: - if _2eyesmode(df) == True: if _2eyesmode(df) == True: return len(_calibrations(df)) // 2 return len(_calibrations(df)) def _get_calibration_positions(df: pd.DataFrame) -> list[int]: - if _2eyesmode(df) == True: if _2eyesmode(df) == True: return ( np.array(df[df[2] == "VALIDATE"][8].str.split(",", expand=True)) @@ -491,14 +489,15 @@ def edf2bids( output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", - extension="json", + extension="json" + ) output_filename_eye1 = generate_output_filename( output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", - extension="json", - ) - with open(output_filename_eye1, "w") as outfile: + extension="json" + ) + with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) @@ -562,8 +561,7 @@ def edf2bids( suffix="_recording-eye1_physio", extension="tsv.gz", ) - content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) - with gzip.open(output_filename_eye1, "wb") as f: + content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) with gzip.open(output_filename_eye1, "wb") as f: f.write(content.encode()) From 3490d8df0748b9c1e55347d30f63e89062dc632f Mon Sep 17 00:00:00 2001 From: julia-pfarr Date: Wed, 5 Jun 2024 16:11:14 +0200 Subject: [PATCH 11/18] first try for physioevents.tsv.gz --- eye2bids/edf2bids.py | 152 ++++++++++++++++++++++++++++--------------- 1 file changed, 101 insertions(+), 51 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index b93304c..1cfd752 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -15,6 +15,7 @@ from eye2bids._parser import global_parser from eye2bids.logger import eye2bids_logger +import re e2b_log = eye2bids_logger() @@ -248,7 +249,6 @@ def _extract_RecordedEye(df: pd.DataFrame) -> str: return "Right" elif eye == "LR": return ["Left", "Right"] - return ["Left", "Right"] return "" @@ -323,6 +323,67 @@ def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame: df_ms = _load_asc_file_as_df(events_asc_file) return pd.DataFrame(df_ms.iloc[0:, 2:]) +def _df_events_after_start(events: list) -> pd.DataFrame: + + start_index = next(i for i, line in enumerate(events) if re.match(r'START\s+.*', line)) + end_index = next(i for i in range(len(events) - 1, -1, -1) if re.match(r'END\s+.*', events[i])) + + if end_index > start_index: + data_lines = events[start_index + 1:end_index] + return pd.DataFrame([line.strip().split('\t') for line in data_lines]) + else: + return print("No 'END' found after the selected 'START'.") + +def _df_physioevents(events_after_start: pd.DataFrame) -> pd.DataFrame: + events_after_start['Event_Letters'] = events_after_start[0].str.extractall(r"([A-Za-z]+)").groupby(level=0).agg(''.join) + events_after_start['Event_Numbers'] = events_after_start[0].str.extract(r'(\d+)') + events_after_start[['msg_timestamp', 'message']] = events_after_start[1].str.split(n=1, expand=True) + events_after_start['message'] = events_after_start['message'].astype(str) + + msg_mask = events_after_start['Event_Letters'] == 'MSG' + events_after_start.loc[msg_mask, 'Event_Numbers'] = events_after_start.loc[msg_mask, 'msg_timestamp'] + physioevents_reordered = (pd.concat([ + events_after_start['Event_Numbers'], + events_after_start[2], events_after_start['Event_Letters'], + events_after_start['message'] + ], axis=1, ignore_index=True) + .replace({None: np.nan, 'None': np.nan}) + .rename(columns={0: 'timestamp',1: 'duration', 2: 'trial_type', 3: 'message'}) + ) + return physioevents_reordered + +def _physioevents_eye1 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: + physioevents_eye1_list = ["MSG", "EFIXL", "ESACCL", "EBLINKL"] + physioevents_eye1 = physioevents_reordered[physioevents_reordered['trial_type'].isin(physioevents_eye1_list)] + physioevents_eye1 = physioevents_eye1.replace({"EFIXL": "fixation", "ESACCL": "saccade", "MSG": np.nan, None: np.nan}) + physioevents_eye1['blink'] = 0 + + for i in range(1, len(physioevents_eye1)): + if physioevents_eye1.iloc[i]['trial_type'] == 'saccade': + if physioevents_eye1.iloc[i-1]['trial_type'] == 'EBLINKL': + physioevents_eye1.iloc[i, physioevents_eye1.columns.get_loc('blink')] = 1 + + physioevents_eye1.loc[physioevents_eye1['trial_type'].isna(), 'blink'] = np.nan + physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('Int64') + physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] + return physioevents_eye1 + +def _physioevents_eye2 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: + physioevents_eye2_list = ["MSG", "EFIXR", "ESACCR", "EBLINKR"] + physioevents_eye2 = physioevents_reordered[physioevents_reordered['trial_type'].isin(physioevents_eye2_list)] + physioevents_eye2 = physioevents_eye2.replace({"EFIXR": "fixation", "ESACCR": "saccade", "MSG": np.nan, None: np.nan}) + physioevents_eye2['blink'] = 0 + + for i in range(1, len(physioevents_eye2)): + if physioevents_eye2.iloc[i]['trial_type'] == 'saccade': + if physioevents_eye2.iloc[i-1]['trial_type'] == 'EBLINKR': + physioevents_eye2.iloc[i, physioevents_eye2.columns.get_loc('blink')] = 1 + + physioevents_eye2.loc[physioevents_eye2['trial_type'].isna(), 'blink'] = np.nan + physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('Int64') + physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] + return physioevents_eye2 + def edf2bids( input_file: str | Path | None = None, @@ -447,25 +508,10 @@ def edf2bids( json.dump(json_eye2, outfile, indent=4) e2b_log.info(f"file generated: {output_filename_eye2}") - with open(output_filename_eye2, "w") as outfile: - json.dump(json_eye2, outfile, indent=4) - - e2b_log.info(f"file generated: {output_filename_eye2}") - - # physioevents.json Metadata # physioevents.json Metadata events_json = { - "Columns": ["onset", "duration", "trial_type", "blink", "message"], - "Description": "Messages logged by the measurement device", - "ForeignIndexColumn": "timestamp", - "blink": {"Description": "One indicates if the eye was closed, zero if open."}, - "message": {"Description": "String messages logged by the eye-tracker."}, - "trial_type": { - "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." - }, - "TaskName": _extract_TaskName(events), "Columns": ["onset", "duration", "trial_type", "blink", "message"], "Description": "Messages logged by the measurement device", "ForeignIndexColumn": "timestamp", @@ -491,35 +537,11 @@ def edf2bids( suffix="_recording-eye1_physioevents", extension="json" ) - output_filename_eye1 = generate_output_filename( - output_dir=output_dir, - input_file=input_file, - suffix="_recording-eye1_physioevents", - extension="json" - ) - with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) e2b_log.info(f"file generated: {output_filename_eye1}") - e2b_log.info(f"file generated: {output_filename_eye1}") - - if _2eyesmode(df_ms_reduced) == True: - - output_filename_eye2 = generate_output_filename( - output_dir=output_dir, - input_file=input_file, - suffix="_recording-eye2_physioevents", - extension="json", - ) - with open(output_filename_eye2, "w") as outfile: - json.dump(events_json, outfile, indent=4) - - e2b_log.info(f"file generated: {output_filename_eye2}") - - # Samples to dataframe - if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( @@ -558,10 +580,8 @@ def edf2bids( output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", - suffix="_recording-eye1_physio", extension="tsv.gz", ) - content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) with gzip.open(output_filename_eye1, "wb") as f: f.write(content.encode()) @@ -582,25 +602,55 @@ def edf2bids( e2b_log.info(f"file generated: {output_filename_eye2}") - # Messages and events to physioevents.tsv.gz - tbc - e2b_log.info(f"file generated: {output_filename_eye1}") + # Messages and events to dataframes - if _2eyesmode(df_ms_reduced) == True: + events_after_start = _df_events_after_start(events) + physioevents_reordered = _df_physioevents(events_after_start) + physioevents_eye1 = _physioevents_eye1(physioevents_reordered) + physioevents_eye2 = _physioevents_eye2(physioevents_reordered) - output_filename_eye2 = generate_output_filename( + # Messages and events to physioevents.tsv.gz + + if _2eyesmode(df_ms_reduced) == False: + output_eventsfilename_eye1 = generate_output_filename( output_dir=output_dir, input_file=input_file, - suffix="_recording-eye2_physio", + suffix="_recording-eye1_physioevents", extension="tsv.gz", ) - content = samples_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) - with gzip.open(output_filename_eye2, "wb") as f: + if _extract_RecordedEye(df_ms_reduced) == 'Left': + content = physioevents_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + elif _extract_RecordedEye(df_ms_reduced) == 'Right': + content = physioevents_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_eventsfilename_eye1, "wb") as f: f.write(content.encode()) - e2b_log.info(f"file generated: {output_filename_eye2}") + e2b_log.info(f"file generated: {output_eventsfilename_eye1}") + + else: + output_eventsfilename_eye1 = generate_output_filename( + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physioevents", + extension="tsv.gz", + ) + content = physioevents_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_eventsfilename_eye1, "wb") as f: + f.write(content.encode()) + + e2b_log.info(f"file generated: {output_eventsfilename_eye1}") - # Messages and events to physioevents.tsv.gz - tbc + output_eventsfilename_eye2 = generate_output_filename( + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physioevents", + extension="tsv.gz", + ) + content = physioevents_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_eventsfilename_eye2, "wb") as f: + f.write(content.encode()) + e2b_log.info(f"file generated: {output_eventsfilename_eye2}") def generate_output_filename( output_dir: Path, input_file: Path, suffix: str, extension: str From baae3c017768aaa15963aad85f723a4bb605bbe6 Mon Sep 17 00:00:00 2001 From: julia-pfarr Date: Wed, 5 Jun 2024 17:30:02 +0200 Subject: [PATCH 12/18] save pre-final physioevents.tsv.gz code --- eye2bids/edf2bids.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 1cfd752..c66be9f 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -356,32 +356,44 @@ def _physioevents_eye1 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye1_list = ["MSG", "EFIXL", "ESACCL", "EBLINKL"] physioevents_eye1 = physioevents_reordered[physioevents_reordered['trial_type'].isin(physioevents_eye1_list)] physioevents_eye1 = physioevents_eye1.replace({"EFIXL": "fixation", "ESACCL": "saccade", "MSG": np.nan, None: np.nan}) + physioevents_eye1['blink'] = 0 + last_non_na_trial_type = None - for i in range(1, len(physioevents_eye1)): - if physioevents_eye1.iloc[i]['trial_type'] == 'saccade': - if physioevents_eye1.iloc[i-1]['trial_type'] == 'EBLINKL': + for i in range(len(physioevents_eye1)): + current_trial_type = physioevents_eye1.iloc[i]['trial_type'] + if pd.notna(current_trial_type): + if current_trial_type == 'saccade' and last_non_na_trial_type == 'EBLINKL': physioevents_eye1.iloc[i, physioevents_eye1.columns.get_loc('blink')] = 1 + last_non_na_trial_type = current_trial_type physioevents_eye1.loc[physioevents_eye1['trial_type'].isna(), 'blink'] = np.nan physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('Int64') physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] + + physioevents_eye1 = physioevents_eye1[['timestamp', 'duration', 'trial_type','blink', 'message']] return physioevents_eye1 def _physioevents_eye2 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye2_list = ["MSG", "EFIXR", "ESACCR", "EBLINKR"] physioevents_eye2 = physioevents_reordered[physioevents_reordered['trial_type'].isin(physioevents_eye2_list)] physioevents_eye2 = physioevents_eye2.replace({"EFIXR": "fixation", "ESACCR": "saccade", "MSG": np.nan, None: np.nan}) + physioevents_eye2['blink'] = 0 + last_non_na_trial_type = None - for i in range(1, len(physioevents_eye2)): - if physioevents_eye2.iloc[i]['trial_type'] == 'saccade': - if physioevents_eye2.iloc[i-1]['trial_type'] == 'EBLINKR': + for i in range(len(physioevents_eye2)): + current_trial_type = physioevents_eye2.iloc[i]['trial_type'] + if pd.notna(current_trial_type): + if current_trial_type == 'saccade' and last_non_na_trial_type == 'EBLINKR': physioevents_eye2.iloc[i, physioevents_eye2.columns.get_loc('blink')] = 1 + last_non_na_trial_type = current_trial_type physioevents_eye2.loc[physioevents_eye2['trial_type'].isna(), 'blink'] = np.nan physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('Int64') physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] + + physioevents_eye2 = physioevents_eye2[['timestamp', 'duration', 'trial_type','blink', 'message']] return physioevents_eye2 From 0277a79f4573ded931174812b2add1277d236556 Mon Sep 17 00:00:00 2001 From: julia-pfarr Date: Fri, 14 Jun 2024 15:33:00 +0200 Subject: [PATCH 13/18] add tests for physioevents.tsv --- tests/test_edf2bids.py | 112 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 1a69daa..81e6660 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -4,6 +4,7 @@ from pathlib import Path import pandas as pd +import numpy as np import pytest from eye2bids.edf2bids import ( @@ -24,6 +25,10 @@ _load_asc_file, _load_asc_file_as_df, _load_asc_file_as_reduced_df, + _df_events_after_start, + _df_physioevents, + _physioevents_eye1, + _physioevents_eye2, edf2bids, ) @@ -67,11 +72,65 @@ def test_edf_end_to_end(metadata_file, eyelink_test_data_dir): assert eyetrack["SamplingFrequency"] == 500 assert eyetrack["RecordedEye"] == "Right" - expected_events_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - assert expected_events_tsv.exists() - expected_events_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + expected_data_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + assert expected_data_tsv.exists() + + expected_events_tsv = output_dir / f"{input_file.stem}_recording-eye1_physioevents.tsv.gz" assert expected_events_tsv.exists() +@pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") +@pytest.mark.parametrize("metadata_file", [data_dir() / "metadata.yml", None]) +def test_edf_end_to_end_2eyes(metadata_file, eyelink_test_data_dir): + input_dir = eyelink_test_data_dir / "2eyes" + input_file = edf_test_files(input_dir=input_dir)[0] + + output_dir = data_dir() / "output" + output_dir.mkdir(exist_ok=True) + + edf2bids( + input_file=input_file, + metadata_file=metadata_file, + output_dir=output_dir, + ) + + expected_events_sidecar_eye1 = ( + output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" + ) + assert expected_events_sidecar_eye1.exists() + with open(expected_events_sidecar_eye1) as f: + events = json.load(f) + assert events["StimulusPresentation"]["ScreenResolution"] == [1919, 1079] + + expected_data_sidecar_eye1 = output_dir / f"{input_file.stem}_recording-eye1_physio.json" + assert expected_data_sidecar_eye1.exists() + with open(expected_data_sidecar_eye1) as f: + eyetrack = json.load(f) + assert eyetrack["SamplingFrequency"] == 1000 + assert eyetrack["RecordedEye"] == "Left" + + expected_data_tsv_eye1 = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + assert expected_data_tsv_eye1.exists() + + expected_events_tsv_eye1 = output_dir / f"{input_file.stem}_recording-eye1_physioevents.tsv.gz" + assert expected_events_tsv_eye1.exists() + + expected_events_sidecar_eye2 = ( + output_dir / f"{input_file.stem}_recording-eye2_physioevents.json" + ) + assert expected_events_sidecar_eye2.exists() + + expected_data_sidecar_eye2 = output_dir / f"{input_file.stem}_recording-eye2_physio.json" + assert expected_data_sidecar_eye2.exists() + with open(expected_data_sidecar_eye2) as f: + eyetrack = json.load(f) + assert eyetrack["RecordedEye"] == "Right" + + expected_data_tsv_eye2 = output_dir / f"{input_file.stem}_recording-eye2_physio.tsv.gz" + assert expected_data_tsv_eye2.exists() + + expected_events_tsv_eye2 = output_dir / f"{input_file.stem}_recording-eye2_physioevents.tsv.gz" + assert expected_events_tsv_eye2.exists() + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_edf_nan_in_tsv(eyelink_test_data_dir): @@ -204,7 +263,6 @@ def test_2files_eye2(eyelink_test_data_dir): @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_number_columns_2eyes_tsv(eyelink_test_data_dir): """Check that values for only one eye were extracted in eye1-physio.tsv.gz by number of columns. - """Check that values for only one eye were extracted in eye1-physio.tsv.gz by number of columns. function _samples_to_data_frame """ @@ -223,7 +281,6 @@ def test_number_columns_2eyes_tsv(eyelink_test_data_dir): df = pd.read_csv(expected_eyetrack_tsv, sep="\t") number_columns = len(df.columns) assert number_columns == 4 - assert number_columns == 4 @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") @@ -548,3 +605,48 @@ def test_extract_AverageCalibrationError(folder, expected, eyelink_test_data_dir asc_file = asc_test_files(input_dir=input_dir, suffix="*_events")[0] df_ms = _load_asc_file_as_df(asc_file) assert _extract_AverageCalibrationError(df_ms) == expected + + +@pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") +def test_number_columns_physioevents_tsv(eyelink_test_data_dir): + """Check right number of columns in physioevents.tsv.gz. + + function _df_physioevents, _physioevents_eye1, _physioevents_eye2 + """ + input_dir = eyelink_test_data_dir / "2eyes" + print(edf_test_files(input_dir=input_dir)) + input_file = edf_test_files(input_dir=input_dir)[0] + + output_dir = data_dir() / "output" + output_dir.mkdir(exist_ok=True) + + edf2bids( + input_file=input_file, + output_dir=output_dir, + ) + + expected_physioevents_tsv = output_dir / f"{input_file.stem}_recording-eye2_physioevents.tsv.gz" + df = pd.read_csv(expected_physioevents_tsv, sep="\t") + number_columns = len(df.columns) + assert number_columns == 5 + +@pytest.mark.parametrize( + "folder, expected", + [ + ("emg", [2116084, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 R"]), + ("lt", [3852521, np.nan, np.nan, np.nan, "EYE_USED 0 LEFT"]), + ("pitracker", [248610, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 R"]), + ("rest", [9199380, np.nan, np.nan, np.nan, "RECORD_START"]), + ("satf", [3098683, 20, "saccade", 1, np.nan]), + ("vergence", [819655, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 LR"]), + ("2eyes",[767985, 169, "fixation", 0, np.nan]) + ], +) +def test_physioevents_value(folder, expected, eyelink_test_data_dir): + input_dir = eyelink_test_data_dir / folder + asc_file = asc_test_files(input_dir=input_dir, suffix="*_events")[0] + events = _load_asc_file(asc_file) + events_after_start = _df_events_after_start(events) + physioevents_reordered = _df_physioevents(events_after_start) + physioevents_eye1 = _physioevents_eye1(physioevents_reordered) + assert physioevents_eye1.iloc[0].tolist() == expected \ No newline at end of file From af2b24d3c78c82d0a41ba10534e9be867dd947c7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:36:15 +0000 Subject: [PATCH 14/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- eye2bids/edf2bids.py | 161 ++++++++++++++++++++++++++--------------- tests/test_edf2bids.py | 44 +++++++---- 2 files changed, 132 insertions(+), 73 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index c66be9f..c781043 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -4,6 +4,7 @@ import gzip import json +import re import subprocess from pathlib import Path @@ -15,7 +16,6 @@ from eye2bids._parser import global_parser from eye2bids.logger import eye2bids_logger -import re e2b_log = eye2bids_logger() @@ -323,77 +323,109 @@ def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame: df_ms = _load_asc_file_as_df(events_asc_file) return pd.DataFrame(df_ms.iloc[0:, 2:]) + def _df_events_after_start(events: list) -> pd.DataFrame: - start_index = next(i for i, line in enumerate(events) if re.match(r'START\s+.*', line)) - end_index = next(i for i in range(len(events) - 1, -1, -1) if re.match(r'END\s+.*', events[i])) + start_index = next( + i for i, line in enumerate(events) if re.match(r"START\s+.*", line) + ) + end_index = next( + i for i in range(len(events) - 1, -1, -1) if re.match(r"END\s+.*", events[i]) + ) if end_index > start_index: - data_lines = events[start_index + 1:end_index] - return pd.DataFrame([line.strip().split('\t') for line in data_lines]) + data_lines = events[start_index + 1 : end_index] + return pd.DataFrame([line.strip().split("\t") for line in data_lines]) else: return print("No 'END' found after the selected 'START'.") -def _df_physioevents(events_after_start: pd.DataFrame) -> pd.DataFrame: - events_after_start['Event_Letters'] = events_after_start[0].str.extractall(r"([A-Za-z]+)").groupby(level=0).agg(''.join) - events_after_start['Event_Numbers'] = events_after_start[0].str.extract(r'(\d+)') - events_after_start[['msg_timestamp', 'message']] = events_after_start[1].str.split(n=1, expand=True) - events_after_start['message'] = events_after_start['message'].astype(str) - - msg_mask = events_after_start['Event_Letters'] == 'MSG' - events_after_start.loc[msg_mask, 'Event_Numbers'] = events_after_start.loc[msg_mask, 'msg_timestamp'] - physioevents_reordered = (pd.concat([ - events_after_start['Event_Numbers'], - events_after_start[2], events_after_start['Event_Letters'], - events_after_start['message'] - ], axis=1, ignore_index=True) - .replace({None: np.nan, 'None': np.nan}) - .rename(columns={0: 'timestamp',1: 'duration', 2: 'trial_type', 3: 'message'}) + +def _df_physioevents(events_after_start: pd.DataFrame) -> pd.DataFrame: + events_after_start["Event_Letters"] = ( + events_after_start[0].str.extractall(r"([A-Za-z]+)").groupby(level=0).agg("".join) + ) + events_after_start["Event_Numbers"] = events_after_start[0].str.extract(r"(\d+)") + events_after_start[["msg_timestamp", "message"]] = events_after_start[1].str.split( + n=1, expand=True + ) + events_after_start["message"] = events_after_start["message"].astype(str) + + msg_mask = events_after_start["Event_Letters"] == "MSG" + events_after_start.loc[msg_mask, "Event_Numbers"] = events_after_start.loc[ + msg_mask, "msg_timestamp" + ] + physioevents_reordered = ( + pd.concat( + [ + events_after_start["Event_Numbers"], + events_after_start[2], + events_after_start["Event_Letters"], + events_after_start["message"], + ], + axis=1, + ignore_index=True, + ) + .replace({None: np.nan, "None": np.nan}) + .rename(columns={0: "timestamp", 1: "duration", 2: "trial_type", 3: "message"}) ) return physioevents_reordered -def _physioevents_eye1 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: + +def _physioevents_eye1(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye1_list = ["MSG", "EFIXL", "ESACCL", "EBLINKL"] - physioevents_eye1 = physioevents_reordered[physioevents_reordered['trial_type'].isin(physioevents_eye1_list)] - physioevents_eye1 = physioevents_eye1.replace({"EFIXL": "fixation", "ESACCL": "saccade", "MSG": np.nan, None: np.nan}) - - physioevents_eye1['blink'] = 0 + physioevents_eye1 = physioevents_reordered[ + physioevents_reordered["trial_type"].isin(physioevents_eye1_list) + ] + physioevents_eye1 = physioevents_eye1.replace( + {"EFIXL": "fixation", "ESACCL": "saccade", "MSG": np.nan, None: np.nan} + ) + + physioevents_eye1["blink"] = 0 last_non_na_trial_type = None for i in range(len(physioevents_eye1)): - current_trial_type = physioevents_eye1.iloc[i]['trial_type'] - if pd.notna(current_trial_type): - if current_trial_type == 'saccade' and last_non_na_trial_type == 'EBLINKL': - physioevents_eye1.iloc[i, physioevents_eye1.columns.get_loc('blink')] = 1 + current_trial_type = physioevents_eye1.iloc[i]["trial_type"] + if pd.notna(current_trial_type): + if current_trial_type == "saccade" and last_non_na_trial_type == "EBLINKL": + physioevents_eye1.iloc[i, physioevents_eye1.columns.get_loc("blink")] = 1 last_non_na_trial_type = current_trial_type - physioevents_eye1.loc[physioevents_eye1['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('Int64') - physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] + physioevents_eye1.loc[physioevents_eye1["trial_type"].isna(), "blink"] = np.nan + physioevents_eye1["blink"] = physioevents_eye1["blink"].astype("Int64") + physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != "EBLINKL"] - physioevents_eye1 = physioevents_eye1[['timestamp', 'duration', 'trial_type','blink', 'message']] + physioevents_eye1 = physioevents_eye1[ + ["timestamp", "duration", "trial_type", "blink", "message"] + ] return physioevents_eye1 -def _physioevents_eye2 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: + +def _physioevents_eye2(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye2_list = ["MSG", "EFIXR", "ESACCR", "EBLINKR"] - physioevents_eye2 = physioevents_reordered[physioevents_reordered['trial_type'].isin(physioevents_eye2_list)] - physioevents_eye2 = physioevents_eye2.replace({"EFIXR": "fixation", "ESACCR": "saccade", "MSG": np.nan, None: np.nan}) + physioevents_eye2 = physioevents_reordered[ + physioevents_reordered["trial_type"].isin(physioevents_eye2_list) + ] + physioevents_eye2 = physioevents_eye2.replace( + {"EFIXR": "fixation", "ESACCR": "saccade", "MSG": np.nan, None: np.nan} + ) - physioevents_eye2['blink'] = 0 + physioevents_eye2["blink"] = 0 last_non_na_trial_type = None for i in range(len(physioevents_eye2)): - current_trial_type = physioevents_eye2.iloc[i]['trial_type'] - if pd.notna(current_trial_type): - if current_trial_type == 'saccade' and last_non_na_trial_type == 'EBLINKR': - physioevents_eye2.iloc[i, physioevents_eye2.columns.get_loc('blink')] = 1 + current_trial_type = physioevents_eye2.iloc[i]["trial_type"] + if pd.notna(current_trial_type): + if current_trial_type == "saccade" and last_non_na_trial_type == "EBLINKR": + physioevents_eye2.iloc[i, physioevents_eye2.columns.get_loc("blink")] = 1 last_non_na_trial_type = current_trial_type - physioevents_eye2.loc[physioevents_eye2['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('Int64') - physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] + physioevents_eye2.loc[physioevents_eye2["trial_type"].isna(), "blink"] = np.nan + physioevents_eye2["blink"] = physioevents_eye2["blink"].astype("Int64") + physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != "EBLINKR"] - physioevents_eye2 = physioevents_eye2[['timestamp', 'duration', 'trial_type','blink', 'message']] + physioevents_eye2 = physioevents_eye2[ + ["timestamp", "duration", "trial_type", "blink", "message"] + ] return physioevents_eye2 @@ -487,12 +519,12 @@ def edf2bids( } else: metadata_eye1 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) - } - - json_eye1 = dict(base_json, **metadata_eye1) + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)), + } + + json_eye1 = dict(base_json, **metadata_eye1) if _2eyesmode(df_ms_reduced) == True: json_eye2 = dict(base_json, **metadata_eye2) @@ -547,8 +579,8 @@ def edf2bids( output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", - extension="json" - ) + extension="json", + ) with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) @@ -630,15 +662,19 @@ def edf2bids( suffix="_recording-eye1_physioevents", extension="tsv.gz", ) - if _extract_RecordedEye(df_ms_reduced) == 'Left': - content = physioevents_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) - elif _extract_RecordedEye(df_ms_reduced) == 'Right': - content = physioevents_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + if _extract_RecordedEye(df_ms_reduced) == "Left": + content = physioevents_eye1.to_csv( + sep="\t", index=False, na_rep="n/a", header=None + ) + elif _extract_RecordedEye(df_ms_reduced) == "Right": + content = physioevents_eye2.to_csv( + sep="\t", index=False, na_rep="n/a", header=None + ) with gzip.open(output_eventsfilename_eye1, "wb") as f: f.write(content.encode()) e2b_log.info(f"file generated: {output_eventsfilename_eye1}") - + else: output_eventsfilename_eye1 = generate_output_filename( output_dir=output_dir, @@ -646,7 +682,9 @@ def edf2bids( suffix="_recording-eye1_physioevents", extension="tsv.gz", ) - content = physioevents_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + content = physioevents_eye1.to_csv( + sep="\t", index=False, na_rep="n/a", header=None + ) with gzip.open(output_eventsfilename_eye1, "wb") as f: f.write(content.encode()) @@ -658,12 +696,15 @@ def edf2bids( suffix="_recording-eye2_physioevents", extension="tsv.gz", ) - content = physioevents_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + content = physioevents_eye2.to_csv( + sep="\t", index=False, na_rep="n/a", header=None + ) with gzip.open(output_eventsfilename_eye2, "wb") as f: f.write(content.encode()) e2b_log.info(f"file generated: {output_eventsfilename_eye2}") + def generate_output_filename( output_dir: Path, input_file: Path, suffix: str, extension: str ) -> Path: diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 81e6660..0b9f50e 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -3,13 +3,15 @@ import json from pathlib import Path -import pandas as pd import numpy as np +import pandas as pd import pytest from eye2bids.edf2bids import ( _check_edf2asc_present, _convert_edf_to_asc_events, + _df_events_after_start, + _df_physioevents, _extract_AverageCalibrationError, _extract_CalibrationPosition, _extract_CalibrationType, @@ -25,8 +27,6 @@ _load_asc_file, _load_asc_file_as_df, _load_asc_file_as_reduced_df, - _df_events_after_start, - _df_physioevents, _physioevents_eye1, _physioevents_eye2, edf2bids, @@ -75,9 +75,12 @@ def test_edf_end_to_end(metadata_file, eyelink_test_data_dir): expected_data_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" assert expected_data_tsv.exists() - expected_events_tsv = output_dir / f"{input_file.stem}_recording-eye1_physioevents.tsv.gz" + expected_events_tsv = ( + output_dir / f"{input_file.stem}_recording-eye1_physioevents.tsv.gz" + ) assert expected_events_tsv.exists() + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") @pytest.mark.parametrize("metadata_file", [data_dir() / "metadata.yml", None]) def test_edf_end_to_end_2eyes(metadata_file, eyelink_test_data_dir): @@ -101,17 +104,23 @@ def test_edf_end_to_end_2eyes(metadata_file, eyelink_test_data_dir): events = json.load(f) assert events["StimulusPresentation"]["ScreenResolution"] == [1919, 1079] - expected_data_sidecar_eye1 = output_dir / f"{input_file.stem}_recording-eye1_physio.json" + expected_data_sidecar_eye1 = ( + output_dir / f"{input_file.stem}_recording-eye1_physio.json" + ) assert expected_data_sidecar_eye1.exists() with open(expected_data_sidecar_eye1) as f: eyetrack = json.load(f) assert eyetrack["SamplingFrequency"] == 1000 assert eyetrack["RecordedEye"] == "Left" - expected_data_tsv_eye1 = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + expected_data_tsv_eye1 = ( + output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + ) assert expected_data_tsv_eye1.exists() - expected_events_tsv_eye1 = output_dir / f"{input_file.stem}_recording-eye1_physioevents.tsv.gz" + expected_events_tsv_eye1 = ( + output_dir / f"{input_file.stem}_recording-eye1_physioevents.tsv.gz" + ) assert expected_events_tsv_eye1.exists() expected_events_sidecar_eye2 = ( @@ -119,16 +128,22 @@ def test_edf_end_to_end_2eyes(metadata_file, eyelink_test_data_dir): ) assert expected_events_sidecar_eye2.exists() - expected_data_sidecar_eye2 = output_dir / f"{input_file.stem}_recording-eye2_physio.json" + expected_data_sidecar_eye2 = ( + output_dir / f"{input_file.stem}_recording-eye2_physio.json" + ) assert expected_data_sidecar_eye2.exists() with open(expected_data_sidecar_eye2) as f: eyetrack = json.load(f) assert eyetrack["RecordedEye"] == "Right" - expected_data_tsv_eye2 = output_dir / f"{input_file.stem}_recording-eye2_physio.tsv.gz" + expected_data_tsv_eye2 = ( + output_dir / f"{input_file.stem}_recording-eye2_physio.tsv.gz" + ) assert expected_data_tsv_eye2.exists() - expected_events_tsv_eye2 = output_dir / f"{input_file.stem}_recording-eye2_physioevents.tsv.gz" + expected_events_tsv_eye2 = ( + output_dir / f"{input_file.stem}_recording-eye2_physioevents.tsv.gz" + ) assert expected_events_tsv_eye2.exists() @@ -625,11 +640,14 @@ def test_number_columns_physioevents_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_physioevents_tsv = output_dir / f"{input_file.stem}_recording-eye2_physioevents.tsv.gz" + expected_physioevents_tsv = ( + output_dir / f"{input_file.stem}_recording-eye2_physioevents.tsv.gz" + ) df = pd.read_csv(expected_physioevents_tsv, sep="\t") number_columns = len(df.columns) assert number_columns == 5 + @pytest.mark.parametrize( "folder, expected", [ @@ -639,7 +657,7 @@ def test_number_columns_physioevents_tsv(eyelink_test_data_dir): ("rest", [9199380, np.nan, np.nan, np.nan, "RECORD_START"]), ("satf", [3098683, 20, "saccade", 1, np.nan]), ("vergence", [819655, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 LR"]), - ("2eyes",[767985, 169, "fixation", 0, np.nan]) + ("2eyes", [767985, 169, "fixation", 0, np.nan]), ], ) def test_physioevents_value(folder, expected, eyelink_test_data_dir): @@ -649,4 +667,4 @@ def test_physioevents_value(folder, expected, eyelink_test_data_dir): events_after_start = _df_events_after_start(events) physioevents_reordered = _df_physioevents(events_after_start) physioevents_eye1 = _physioevents_eye1(physioevents_reordered) - assert physioevents_eye1.iloc[0].tolist() == expected \ No newline at end of file + assert physioevents_eye1.iloc[0].tolist() == expected From 3e35540dd0445eb4468bfdd355c57bb17a8e18bb Mon Sep 17 00:00:00 2001 From: julia-pfarr Date: Fri, 14 Jun 2024 16:50:08 +0200 Subject: [PATCH 15/18] fix error in tests --- eye2bids/edf2bids.py | 6 ++++++ tests/test_edf2bids.py | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index c66be9f..a7ab8c1 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -371,6 +371,9 @@ def _physioevents_eye1 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('Int64') physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] + physioevents_eye1['timestamp']= physioevents_eye1['timestamp'].astype('Int64') + physioevents_eye1['duration']= physioevents_eye1['duration'].astype('Int64') + physioevents_eye1 = physioevents_eye1[['timestamp', 'duration', 'trial_type','blink', 'message']] return physioevents_eye1 @@ -393,6 +396,9 @@ def _physioevents_eye2 (physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('Int64') physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] + physioevents_eye2['timestamp']= physioevents_eye2['timestamp'].astype('Int64') + physioevents_eye2['duration']= physioevents_eye2['duration'].astype('Int64') + physioevents_eye2 = physioevents_eye2[['timestamp', 'duration', 'trial_type','blink', 'message']] return physioevents_eye2 diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 81e6660..862f5e6 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -633,12 +633,12 @@ def test_number_columns_physioevents_tsv(eyelink_test_data_dir): @pytest.mark.parametrize( "folder, expected", [ - ("emg", [2116084, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 R"]), - ("lt", [3852521, np.nan, np.nan, np.nan, "EYE_USED 0 LEFT"]), - ("pitracker", [248610, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 R"]), - ("rest", [9199380, np.nan, np.nan, np.nan, "RECORD_START"]), - ("satf", [3098683, 20, "saccade", 1, np.nan]), - ("vergence", [819655, np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 LR"]), + ("emg", [2116084, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 R"]), + ("lt", [3852521, np.nan, np.nan, pd.NA, "EYE_USED 0 LEFT"]), + ("pitracker", [248610, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 R"]), + ("rest", [9199380, pd.NA, np.nan, pd.NA, "RECORD_START"]), + ("satf", [3209032, pd.NA, np.nan, pd.NA, "Calibration ended"]), + ("vergence", [819655, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 LR"]), ("2eyes",[767985, 169, "fixation", 0, np.nan]) ], ) From 06659fed59d4769a137b132ac976105fcbf9b267 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:52:32 +0000 Subject: [PATCH 16/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- eye2bids/edf2bids.py | 20 ++++++++++---------- tests/test_edf2bids.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index f041745..e274026 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -390,12 +390,12 @@ def _physioevents_eye1(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye1.iloc[i, physioevents_eye1.columns.get_loc("blink")] = 1 last_non_na_trial_type = current_trial_type - physioevents_eye1.loc[physioevents_eye1['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('Int64') - physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] + physioevents_eye1.loc[physioevents_eye1["trial_type"].isna(), "blink"] = np.nan + physioevents_eye1["blink"] = physioevents_eye1["blink"].astype("Int64") + physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != "EBLINKL"] - physioevents_eye1['timestamp']= physioevents_eye1['timestamp'].astype('Int64') - physioevents_eye1['duration']= physioevents_eye1['duration'].astype('Int64') + physioevents_eye1["timestamp"] = physioevents_eye1["timestamp"].astype("Int64") + physioevents_eye1["duration"] = physioevents_eye1["duration"].astype("Int64") physioevents_eye1 = physioevents_eye1[ ["timestamp", "duration", "trial_type", "blink", "message"] @@ -422,12 +422,12 @@ def _physioevents_eye2(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye2.iloc[i, physioevents_eye2.columns.get_loc("blink")] = 1 last_non_na_trial_type = current_trial_type - physioevents_eye2.loc[physioevents_eye2['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('Int64') - physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] + physioevents_eye2.loc[physioevents_eye2["trial_type"].isna(), "blink"] = np.nan + physioevents_eye2["blink"] = physioevents_eye2["blink"].astype("Int64") + physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != "EBLINKR"] - physioevents_eye2['timestamp']= physioevents_eye2['timestamp'].astype('Int64') - physioevents_eye2['duration']= physioevents_eye2['duration'].astype('Int64') + physioevents_eye2["timestamp"] = physioevents_eye2["timestamp"].astype("Int64") + physioevents_eye2["duration"] = physioevents_eye2["duration"].astype("Int64") physioevents_eye2 = physioevents_eye2[ ["timestamp", "duration", "trial_type", "blink", "message"] diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 2dc067a..efcb7bb 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -657,7 +657,7 @@ def test_number_columns_physioevents_tsv(eyelink_test_data_dir): ("rest", [9199380, pd.NA, np.nan, pd.NA, "RECORD_START"]), ("satf", [3209032, pd.NA, np.nan, pd.NA, "Calibration ended"]), ("vergence", [819655, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 LR"]), - ("2eyes",[767985, 169, "fixation", 0, np.nan]) + ("2eyes", [767985, 169, "fixation", 0, np.nan]), ], ) def test_physioevents_value(folder, expected, eyelink_test_data_dir): From 3e06d52c6e9ac459fed56d0eeca8a2efe5fb97c3 Mon Sep 17 00:00:00 2001 From: julia-pfarr Date: Fri, 14 Jun 2024 17:43:08 +0200 Subject: [PATCH 17/18] try fix test --- eye2bids/edf2bids.py | 10 ++-------- tests/test_edf2bids.py | 14 +++++++------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index f041745..fff60e0 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -391,12 +391,9 @@ def _physioevents_eye1(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: last_non_na_trial_type = current_trial_type physioevents_eye1.loc[physioevents_eye1['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('Int64') + physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('object') physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] - physioevents_eye1['timestamp']= physioevents_eye1['timestamp'].astype('Int64') - physioevents_eye1['duration']= physioevents_eye1['duration'].astype('Int64') - physioevents_eye1 = physioevents_eye1[ ["timestamp", "duration", "trial_type", "blink", "message"] ] @@ -423,12 +420,9 @@ def _physioevents_eye2(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: last_non_na_trial_type = current_trial_type physioevents_eye2.loc[physioevents_eye2['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('Int64') + physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('object') physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] - physioevents_eye2['timestamp']= physioevents_eye2['timestamp'].astype('Int64') - physioevents_eye2['duration']= physioevents_eye2['duration'].astype('Int64') - physioevents_eye2 = physioevents_eye2[ ["timestamp", "duration", "trial_type", "blink", "message"] ] diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 2dc067a..7872dbd 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -651,13 +651,13 @@ def test_number_columns_physioevents_tsv(eyelink_test_data_dir): @pytest.mark.parametrize( "folder, expected", [ - ("emg", [2116084, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 R"]), - ("lt", [3852521, np.nan, np.nan, pd.NA, "EYE_USED 0 LEFT"]), - ("pitracker", [248610, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 R"]), - ("rest", [9199380, pd.NA, np.nan, pd.NA, "RECORD_START"]), - ("satf", [3209032, pd.NA, np.nan, pd.NA, "Calibration ended"]), - ("vergence", [819655, pd.NA, np.nan, pd.NA, "!MODE RECORD CR 1000 2 1 LR"]), - ("2eyes",[767985, 169, "fixation", 0, np.nan]) + ("emg", ["2116084", np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 R"]), + ("lt", ["3852521", np.nan, np.nan, np.nan, "EYE_USED 0 LEFT"]), + ("pitracker", ["248610", np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 R"]), + ("rest", ["9199380", np.nan, np.nan, np.nan, "RECORD_START"]), + ("satf", ["3209032", np.nan, np.nan, np.nan, "Calibration ended"]), + ("vergence", ["819655", np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 LR"]), + ("2eyes",["767985", "169", "fixation", "0", np.nan]) ], ) def test_physioevents_value(folder, expected, eyelink_test_data_dir): From d3a6468504397e68ed3defa52819f9873ee526c7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:44:57 +0000 Subject: [PATCH 18/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- eye2bids/edf2bids.py | 12 ++++++------ tests/test_edf2bids.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index fff60e0..7bb2a1b 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -390,9 +390,9 @@ def _physioevents_eye1(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye1.iloc[i, physioevents_eye1.columns.get_loc("blink")] = 1 last_non_na_trial_type = current_trial_type - physioevents_eye1.loc[physioevents_eye1['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye1['blink'] = physioevents_eye1['blink'].astype('object') - physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != 'EBLINKL'] + physioevents_eye1.loc[physioevents_eye1["trial_type"].isna(), "blink"] = np.nan + physioevents_eye1["blink"] = physioevents_eye1["blink"].astype("object") + physioevents_eye1 = physioevents_eye1[physioevents_eye1.trial_type != "EBLINKL"] physioevents_eye1 = physioevents_eye1[ ["timestamp", "duration", "trial_type", "blink", "message"] @@ -419,9 +419,9 @@ def _physioevents_eye2(physioevents_reordered: pd.DataFrame) -> pd.DataFrame: physioevents_eye2.iloc[i, physioevents_eye2.columns.get_loc("blink")] = 1 last_non_na_trial_type = current_trial_type - physioevents_eye2.loc[physioevents_eye2['trial_type'].isna(), 'blink'] = np.nan - physioevents_eye2['blink'] = physioevents_eye2['blink'].astype('object') - physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != 'EBLINKR'] + physioevents_eye2.loc[physioevents_eye2["trial_type"].isna(), "blink"] = np.nan + physioevents_eye2["blink"] = physioevents_eye2["blink"].astype("object") + physioevents_eye2 = physioevents_eye2[physioevents_eye2.trial_type != "EBLINKR"] physioevents_eye2 = physioevents_eye2[ ["timestamp", "duration", "trial_type", "blink", "message"] diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 7872dbd..3121f7f 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -657,7 +657,7 @@ def test_number_columns_physioevents_tsv(eyelink_test_data_dir): ("rest", ["9199380", np.nan, np.nan, np.nan, "RECORD_START"]), ("satf", ["3209032", np.nan, np.nan, np.nan, "Calibration ended"]), ("vergence", ["819655", np.nan, np.nan, np.nan, "!MODE RECORD CR 1000 2 1 LR"]), - ("2eyes",["767985", "169", "fixation", "0", np.nan]) + ("2eyes", ["767985", "169", "fixation", "0", np.nan]), ], ) def test_physioevents_value(folder, expected, eyelink_test_data_dir):