From b6d022509bbc0b29d71d9ce7a3b7241ec33a1f6d Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Wed, 10 Apr 2024 16:13:40 +0200 Subject: [PATCH 01/13] modify converter and tests according to issues (#57) and (#60) correct errors after testing correct suffix modify tests according to converter modifications --- eye2bids/edf2bids.py | 269 ++++++++++++++++++++++++----------------- tests/test_edf2bids.py | 78 +++++++++--- 2 files changed, 222 insertions(+), 125 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 9be7b85..edaacfc 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -115,23 +115,29 @@ def _convert_edf_to_asc_samples(input_file: str | Path) -> Path: subprocess.run(["edf2asc", "-y", "-s", input_file, "-o", samples_asc_file]) return Path(samples_asc_file).with_suffix(".asc") +def _2eyesmode(df: pd.DataFrame) -> bool: + eye = df[df[2] == "RECCFG"].iloc[0:1, 5:6].to_string(header=False, index=False) + if eye == "LR": + two_eyes = True + else: + two_eyes = False + return two_eyes def _calibrations(df: pd.DataFrame) -> pd.DataFrame: return df[df[3] == "CALIBRATION"] - def _extract_CalibrationType(df: pd.DataFrame) -> list[int]: return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False) def _extract_CalibrationCount(df: pd.DataFrame) -> int: - if _extract_RecordedEye(df) == "Both": + if _2eyesmode(df) == True: return len(_calibrations(df)) // 2 return len(_calibrations(df)) def _get_calibration_positions(df: pd.DataFrame) -> list[int]: - if _extract_RecordedEye(df) == "Both": + if _2eyesmode(df) == True: return ( np.array(df[df[2] == "VALIDATE"][8].str.split(",", expand=True)) .astype(int) @@ -198,30 +204,12 @@ def _has_validation(df: pd.DataFrame) -> bool: def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]: if not _has_validation(df): return [] - if _extract_RecordedEye(df) == "Both" and _extract_CalibrationCount(df) > 1: - return ( - np.array_split( - np.array(_validations(df)[[11]]), - _extract_CalibrationCount(df), - ) - .astype(float) - .tolist() - ) return np.array(_validations(df)[[11]]).astype(float).tolist() def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]: if not _has_validation(df): return [] - if _extract_RecordedEye(df) == "Both" and _extract_CalibrationCount(df) > 1: - return ( - np.array_split( - np.array(_validations(df)[[9]]), - _extract_CalibrationCount(df), - ) - .astype(float) - .tolist() - ) return np.array(_validations(df)[[9]]).astype(float).tolist() @@ -256,8 +244,8 @@ def _extract_RecordedEye(df: pd.DataFrame) -> str: elif eye == "R": return "Right" elif eye == "LR": - return "Both" - return "" + return ["Left", "Right"] + return "" def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]: @@ -331,36 +319,6 @@ def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame: df_ms = _load_asc_file_as_df(events_asc_file) return pd.DataFrame(df_ms.iloc[0:, 2:]) - -def _samples_to_data_frame(samples_asc_file: str | Path) -> pd.DataFrame: - column_names = [ - "eye_timestamp", - "eye1_x_coordinate", - "eye1_y_coordinate", - "eye1_pupil_size", - "eye2_x_coordinate", - "eye2_y_coordinate", - "eye2_pupil_size", - ] - - data: dict[str, list[str]] = {name: [] for name in column_names} - - with open(samples_asc_file) as file: - for line in file: - columns = line.strip().split("\t") - for i in range(len(column_names)): - if i < len(columns): - data[column_names[i]].append(columns[i]) - - data = { - key: value - for key, value in data.items() - if any(val not in ("", "...") for val in value) - } - - return pd.DataFrame(data) - - def edf2bids( input_file: str | Path | None = None, metadata_file: str | Path | None = None, @@ -394,44 +352,108 @@ def edf2bids( with open(metadata_file) as f: metadata = yaml.load(f, Loader=SafeLoader) - # events.json Metadata - eyetrack_json = { - "Manufacturer": "SR-Research", - "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), - "EyeCameraSettings": metadata.get("EyeCameraSettings"), - "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), - "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), - "GazeMappingSettings": metadata.get("GazeMappingSettings"), - "RawDataFilters": metadata.get("RawDataFilters"), - "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), - "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), - "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), - "SoftwareVersion": metadata.get("SoftwareVersion"), - "DeviceSerialNumber": _extract_DeviceSerialNumber(events), - "EyeTrackingMethod": _extract_EyeTrackingMethod(events), - "ManufacturersModelName": _extract_ManufacturersModelName(events), - "AverageCalibrationError": _extract_AverageCalibrationError(df_ms), - "MaximalCalibrationError": _extract_MaximalCalibrationError(df_ms), - "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), - "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced), - "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), - "CalibrationType": _extract_CalibrationType(df_ms_reduced), - "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), - "RecordedEye": _extract_RecordedEye(df_ms_reduced), - "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), - "StartTime": _extract_StartTime(events), - "StopTime": _extract_StopTime(events), - } - output_filename = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_eyetrack", extension="json" - ) - with open(output_filename, "w") as outfile: - json.dump(eyetrack_json, outfile, indent=4) - e2b_log.info(f"file generated: {output_filename}") - # events.json Metadata + # eye-physio.json Metadata + base_json = { + "Columns": [ "x_coordinate", "y_coordinate", "pupil_size", "timestamp"], + "x_coordinate": { + "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u." + }, + "y_coordinate": { + "Description": "Gaze position y-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u." + }, + "pupil_size": { + "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", + "Units": "a.u." + }, + "timestamp": { + "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." + }, + "Manufacturer": "SR-Research", + "ManufacturersModelName": _extract_ManufacturersModelName(events), + "DeviceSerialNumber": _extract_DeviceSerialNumber(events), + "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), + "SoftwareVersion": metadata.get("SoftwareVersion"), + "EyeCameraSettings": metadata.get("EyeCameraSettings"), + "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), + "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), + "GazeMappingSettings": metadata.get("GazeMappingSettings"), + "RawDataFilters": metadata.get("RawDataFilters"), + "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), + "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), + "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), + "EyeTrackingMethod": _extract_EyeTrackingMethod(events), + "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), + "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), + "StartTime": _extract_StartTime(events), + "StopTime": _extract_StopTime(events), + "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), + "CalibrationType": _extract_CalibrationType(df_ms_reduced), + "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), + "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced) + } + + if _2eyesmode(df_ms_reduced) == True: + metadata_eye1 = { + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[0]) + } + + metadata_eye2 = { + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[1::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[1::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[1]) + } + else: + metadata_eye1 = { + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) + } + + json_eye1 = base_json | metadata_eye1 + if _2eyesmode(df_ms_reduced) == True: + json_eye2 = base_json | metadata_eye2 + + # to json + + output_filename_eye1 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="json" + ) + with open(output_filename_eye1, "w") as outfile: + json.dump(json_eye1, outfile, indent=4) + + e2b_log.info(f"file generated: {output_filename_eye1}") + + if _2eyesmode(df_ms_reduced) == True: + output_filename_eye2 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="json" + ) + with open(output_filename_eye2, "w") as outfile: + json.dump(json_eye2, outfile, indent=4) + + e2b_log.info(f"file generated: {output_filename_eye2}") + + # physioevents.json Metadata + events_json = { + "Columns": ["onset", "duration", "trial_type", "blink", "message"], + "Description": "Messages logged by the measurement device", + "ForeignIndexColumn": "timestamp", + "blink": { + "Description": "One indicates if the eye was closed, zero if open." + }, + "message": { + "Description": "String messages logged by the eye-tracker." + }, + "trial_type": { + "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." + }, + "TaskName": _extract_TaskName(events), "InstitutionAddress": metadata.get("InstitutionAddress"), "InstitutionName": metadata.get("InstitutionName"), "StimulusPresentation": { @@ -439,41 +461,68 @@ def edf2bids( "ScreenRefreshRate": metadata.get("ScreenRefreshRate"), "ScreenSize": metadata.get("ScreenSize"), "ScreenResolution": _extract_ScreenResolution(df_ms_reduced), - }, - "TaskName": _extract_TaskName(events), + } } - output_filename = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_events", extension="json" - ) - with open(output_filename, "w") as outfile: + + output_filename_eye1 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", extension="json" + ) + with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) - e2b_log.info(f"file generated: {output_filename}") - # Samples to eyetrack.tsv + e2b_log.info(f"file generated: {output_filename_eye1}") + + if _2eyesmode(df_ms_reduced) == True: + + output_filename_eye2 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physioevents", extension="json" + ) + with open(output_filename_eye2, "w") as outfile: + json.dump(events_json, outfile, indent=4) + + e2b_log.info(f"file generated: {output_filename_eye2}") + + # Samples to dataframe + samples_asc_file = _convert_edf_to_asc_samples(input_file) if not samples_asc_file.exists(): e2b_log.error( "The following .edf input file could not be converted to .asc:" f"{input_file}" ) - eyetrack_tsv = _samples_to_data_frame(samples_asc_file) - # strip blankspcace and convert empty cells to nan - eyetrack_tsv = eyetrack_tsv.applymap(lambda x: x.strip() if isinstance(x, str) else x) - eyetrack_tsv = eyetrack_tsv.replace(".", np.nan, regex=False) - - output_filename = generate_output_filename( - output_dir=output_dir, - input_file=input_file, - suffix="_eyetrack", - extension="tsv.gz", - ) - content = eyetrack_tsv.to_csv(sep="\t", index=False, na_rep="n/a") - with gzip.open(output_filename, "wb") as f: + samples = pd.read_csv(samples_asc_file, sep="\t", header=None) + samples_eye1 = pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]).map(lambda x: x.strip() if isinstance(x, str) else x).replace(".", np.nan, regex=False) + + if _2eyesmode(df_ms_reduced) == True: + samples_eye2 = pd.DataFrame(samples.iloc[:, [4, 5, 6, 0]]) + + # Samples to eye_physio.tsv.gz + + output_filename_eye1 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="tsv.gz" + ) + content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_filename_eye1, "wb") as f: f.write(content.encode()) - e2b_log.info(f"file generated: {output_filename}") + e2b_log.info(f"file generated: {output_filename_eye1}") + + if _2eyesmode(df_ms_reduced) == True: + + output_filename_eye2 = generate_output_filename( + output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="tsv.gz" + ) + content = samples_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) + with gzip.open(output_filename_eye2, "wb") as f: + f.write(content.encode()) + + e2b_log.info(f"file generated: {output_filename_eye2}") + + + # Messages and events to physioevents.tsv.gz - tbc + def generate_output_filename( diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 55fc9f1..9ada11f 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -52,21 +52,21 @@ def test_edf_end_to_end(metadata_file, eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_events.json" + expected_events_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" assert expected_events_sidecar.exists() with open(expected_events_sidecar) as f: events = json.load(f) assert events["StimulusPresentation"]["ScreenResolution"] == [1919, 1079] - expected_data_sidecar = output_dir / f"{input_file.stem}_eyetrack.json" + expected_data_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physio.json" assert expected_data_sidecar.exists() with open(expected_data_sidecar) as f: eyetrack = json.load(f) assert eyetrack["SamplingFrequency"] == 500 assert eyetrack["RecordedEye"] == "Right" - expected_events_sidecar = output_dir / f"{input_file.stem}_eyetrack.tsv.gz" - assert expected_events_sidecar.exists() + expected_events_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + assert expected_events_tsv.exists() @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") @@ -83,15 +83,63 @@ def test_edf_nan_in_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_eyetrack.tsv.gz" - df = pd.read_csv(expected_events_sidecar, sep="\t") - count = sum(i == "." for i in df["eye1_x_coordinate"]) + expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + df = pd.read_csv(expected_eyetrack_tsv, sep="\t") + count = sum(i == "." for i in df[0]) assert count == 0 +@pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") +def test_2files_eye1(eyelink_test_data_dir): + """Check that for datafile with 2eyes 2 eye1 file is created and check input. + + function _2eyesmode + """ + input_dir = eyelink_test_data_dir / "2eyes" + input_file = edf_test_files(input_dir=input_dir)[0] + + output_dir = data_dir() / "output" + output_dir.mkdir(exist_ok=True) + + edf2bids( + input_file=input_file, + output_dir=output_dir, + ) + + expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physio.json" + assert expected_eyetrack_sidecar.exists() + with open(expected_eyetrack_sidecar) as f: + eyetrack = json.load(f) + assert eyetrack["AverageCalibrationError"] == [[0.29]] + assert eyetrack["RecordedEye"] == "Left" + +@pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") +def test_2files_eye2(eyelink_test_data_dir): + """Check that for datafile with 2eyes 2 eye2 file is created and check input. + + function _2eyesmode + """ + input_dir = eyelink_test_data_dir / "2eyes" + input_file = edf_test_files(input_dir=input_dir)[0] + + output_dir = data_dir() / "output" + output_dir.mkdir(exist_ok=True) + + edf2bids( + input_file=input_file, + output_dir=output_dir, + ) + + expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye2_physio.json" + assert expected_eyetrack_sidecar.exists() + with open(expected_eyetrack_sidecar) as f: + eyetrack = json.load(f) + assert eyetrack["AverageCalibrationError"] == [[0.35]] + assert eyetrack["RecordedEye"] == "Right" + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_number_columns_2eyes_tsv(eyelink_test_data_dir): - """Check that values for both eyes were extracted by number of columns. + """Check that values for only one eye were extracted in eye1-physio.tsv.gz by number of columns. function _samples_to_data_frame """ @@ -106,10 +154,10 @@ def test_number_columns_2eyes_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_eyetrack.tsv.gz" - df = pd.read_csv(expected_events_sidecar, sep="\t") + expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + df = pd.read_csv(expected_eyetrack_tsv, sep="\t") number_columns = len(df.columns) - assert number_columns == 7 + assert number_columns == 4 @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") @@ -130,8 +178,8 @@ def test_number_columns_1eye_tsv(eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}.tsv.gz" - df = pd.read_csv(expected_events_sidecar, sep="\t") + expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" + df = pd.read_csv(expected_eyetrack_tsv, sep="\t") number_columns = len(df.columns) assert number_columns == 4 @@ -360,8 +408,8 @@ def test_extract_DeviceSerialNumber(folder, expected, eyelink_test_data_dir): ("pitracker", "Right"), ("rest", "Left"), ("satf", "Right"), - ("vergence", "Both"), - ("2eyes", "Both"), + ("vergence", ["Left", "Right"]), + ("2eyes", ["Left", "Right"]), ], ) def test_extract_RecordedEye(folder, expected, eyelink_test_data_dir): From c462c0a5167da0164512ec86bf1974b6fa49a279 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 6 Apr 2024 10:36:52 +0200 Subject: [PATCH 02/13] tmp --- eye2bids/2eyes.ipynb | 664 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 664 insertions(+) create mode 100644 eye2bids/2eyes.ipynb diff --git a/eye2bids/2eyes.ipynb b/eye2bids/2eyes.ipynb new file mode 100644 index 0000000..71eabc5 --- /dev/null +++ b/eye2bids/2eyes.ipynb @@ -0,0 +1,664 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import annotations\n", + "\n", + "import gzip\n", + "import json\n", + "import subprocess\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import yaml\n", + "from rich.prompt import Prompt\n", + "from yaml.loader import SafeLoader" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "events_asc_file = \"/Users/julia/Desktop/NOWA.nosync/eyelink/2eyes/sub-99_task-FreeView_run-01_eyeData_events.asc\"" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def _load_asc_file(events_asc_file: str | Path) -> list[str]:\n", + " with open(events_asc_file) as f:\n", + " return f.readlines()\n", + "\n", + "\n", + "def _load_asc_file_as_df(events_asc_file: str | Path) -> pd.DataFrame:\n", + " # dataframe for events, all\n", + " events = _load_asc_file(events_asc_file)\n", + " return pd.DataFrame([ms.split() for ms in events if ms.startswith(\"MSG\")])\n", + "\n", + "\n", + "def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame:\n", + " # reduced dataframe without MSG and sample columns\n", + " df_ms = _load_asc_file_as_df(events_asc_file)\n", + " return pd.DataFrame(df_ms.iloc[0:, 2:])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "events = _load_asc_file(events_asc_file)\n", + "df_ms = _load_asc_file_as_df(events_asc_file)\n", + "df_ms_reduced = _load_asc_file_as_reduced_df(events_asc_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def _calibrations(df: pd.DataFrame) -> pd.DataFrame:\n", + " return df[df[3] == \"CALIBRATION\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationType(df: pd.DataFrame) -> list[int]:\n", + " return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationCount(df: pd.DataFrame) -> int:\n", + " if _extract_RecordedEye(df) == \"Both\":\n", + " return len(_calibrations(df)) // 2\n", + " return len(_calibrations(df))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def _get_calibration_positions(df: pd.DataFrame) -> list[int]:\n", + " if _extract_RecordedEye(df) == \"Both\":\n", + " return (\n", + " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", + " .astype(int)\n", + " .tolist()\n", + " )[::2]\n", + " return (\n", + " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", + " .astype(int)\n", + " .tolist()\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationPosition(df: pd.DataFrame) -> list[list[int]]:\n", + " cal_pos = _get_calibration_positions(df)\n", + " cal_num = len(cal_pos) // _extract_CalibrationCount(df)\n", + "\n", + " CalibrationPosition: list[list[int]] = []\n", + "\n", + " if len(cal_pos) == 0:\n", + " return CalibrationPosition\n", + "\n", + " CalibrationPosition.extend(\n", + " cal_pos[i : i + cal_num] for i in range(0, len(cal_pos), cal_num)\n", + " )\n", + " return CalibrationPosition" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_CalibrationUnit(df: pd.DataFrame) -> str:\n", + " if len(_get_calibration_positions(df)) == 0:\n", + " return \"\"\n", + "\n", + " cal_unit = (\n", + " (df[df[2] == \"VALIDATE\"][[13]])\n", + " .iloc[0:1, 0:1]\n", + " .to_string(header=False, index=False)\n", + " )\n", + " if cal_unit == \"pix.\":\n", + " return \"pixel\"\n", + " elif cal_unit in [\"cm\", \"mm\"]:\n", + " return cal_unit\n", + " return \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_EyeTrackingMethod(events: list[str]) -> str:\n", + " return (\n", + " pd.DataFrame(\n", + " \" \".join([tm for tm in events if tm.startswith(\">>>>>>>\")])\n", + " .replace(\")\", \",\")\n", + " .split(\",\")\n", + " )\n", + " .iloc[1:2]\n", + " .to_string(header=False, index=False)\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_DeviceSerialNumber(events: list[str]) -> str:\n", + " return (\n", + " \" \".join([sl for sl in events if sl.startswith(\"** SERIAL NUMBER:\")])\n", + " .replace(\"** SERIAL NUMBER: \", \"\")\n", + " .replace(\"\\n\", \"\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_PupilFitMethod(df: pd.DataFrame) -> str:\n", + " return (df[df[2] == \"ELCL_PROC\"]).iloc[0:1, 1:2].to_string(header=False, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_SamplingFrequency(df: pd.DataFrame) -> int:\n", + " return int(df[df[2] == \"RECCFG\"].iloc[0:1, 2:3].to_string(header=False, index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def _validations(df: pd.DataFrame) -> pd.DataFrame:\n", + " return df[df[3] == \"VALIDATION\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "def _has_validation(df: pd.DataFrame) -> bool:\n", + " return not _validations(df).empty" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_ManufacturersModelName(events: list[str]) -> str:\n", + " return (\n", + " \" \".join([ml for ml in events if ml.startswith(\"** EYELINK\")])\n", + " .replace(\"** \", \"\")\n", + " .replace(\"\\n\", \"\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]:\n", + " list_res = (\n", + " (df[df[2] == \"GAZE_COORDS\"])\n", + " .iloc[0:1, 3:5]\n", + " .to_string(header=False, index=False)\n", + " .replace(\".00\", \"\")\n", + " .split(\" \")\n", + " )\n", + " return [eval(i) for i in list_res]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_TaskName(events: list[str]) -> str:\n", + " return (\n", + " \" \".join([ts for ts in events if ts.startswith(\"** RECORDED BY\")])\n", + " .replace(\"** RECORDED BY \", \"\")\n", + " .replace(\"\\n\", \"\")\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_StartTime(events: list[str]) -> int:\n", + " StartTime = (\n", + " np.array(pd.DataFrame([st.split() for st in events if st.startswith(\"START\")])[1])\n", + " .astype(int)\n", + " .tolist()\n", + " )\n", + " if len(StartTime) > 1:\n", + " e2b_log.info(\n", + " \"\"\"Your input file contains multiple start times.\\n\n", + " As this is not seen as good practice in eyetracking experiments, \\n\n", + " only the first start time will be kept for the metadata file. \\n\n", + " Please consider changing your code accordingly\n", + " for future eyetracking experiments.\\n\"\"\"\n", + " )\n", + " return StartTime[0]\n", + " return StartTime" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_StopTime(events: list[str]) -> int:\n", + " StopTime = (\n", + " np.array(pd.DataFrame([so.split() for so in events if so.startswith(\"END\")])[1])\n", + " .astype(int)\n", + " .tolist()\n", + " )\n", + " if len(StopTime) > 1:\n", + " e2b_log.info(\n", + " \"\"\"Your input file contains multiple stop times.\\n\n", + " As this is not seen as good practice in eyetracking experiments, \\n\n", + " only the last stop time will be kept for the metadata file. \\n\n", + " Please consider changing your code accordingly\n", + " for future eyetracking experiments.\\n\"\"\"\n", + " )\n", + " return StopTime[-1]\n", + " return StopTime" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", + " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", + " if eye == \"L\":\n", + " return \"Left\"\n", + " elif eye == \"R\":\n", + " return \"Right\"\n", + " elif eye == \"LR\":\n", + " return [\"Left\", \"Right\"]\n", + " return \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CLG-BAF38 P-CR EYELINK II CL v5.12 May 12 2017 pixel HV13 CENTROID 1000 [767979] [819652]\n" + ] + } + ], + "source": [ + "DeviceSerialNumber = _extract_DeviceSerialNumber(events)\n", + "EyeTrackingMethod = _extract_EyeTrackingMethod(events)\n", + "ManufacturersModelName = _extract_ManufacturersModelName(events)\n", + "CalibrationUnit = _extract_CalibrationUnit(df_ms_reduced)\n", + "CalibrationType = _extract_CalibrationType(df_ms_reduced)\n", + "PupilFitMethod = _extract_PupilFitMethod(df_ms_reduced)\n", + "SamplingFrequency = _extract_SamplingFrequency(df_ms_reduced)\n", + "StartTime = _extract_StartTime(events)\n", + "StopTime = _extract_StopTime(events)\n", + "\n", + "print(DeviceSerialNumber, EyeTrackingMethod, ManufacturersModelName, CalibrationUnit, CalibrationType, PupilFitMethod, SamplingFrequency, StartTime, StopTime)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "def _2eyesmode(df: pd.DataFrame) -> bool:\n", + " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", + " if eye == \"LR\":\n", + " two_eyes = True\n", + " return two_eyes" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_2eyesmode(df_ms_reduced)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", + " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", + " if eye == \"L\":\n", + " return eye1\n", + " elif eye == \"R\":\n", + " return \"Right\"\n", + " elif eye == \"LR\":\n", + " return [\"Left\", \"Right\"]\n", + " return \"\" " + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]:\n", + " if not _has_validation(df):\n", + " return []\n", + " if _extract_CalibrationCount(df) > 1:\n", + " return np.array(_validations(df)[[9]]).astype(float).tolist()\n", + " return np.array(_validations(df)[[9]]).astype(float).tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": {}, + "outputs": [], + "source": [ + "def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]:\n", + " if not _has_validation(df):\n", + " return []\n", + " return np.array(_validations(df)[[11]]).astype(float).tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [], + "source": [ + "if _2eyesmode(df_ms_reduced) == True:\n", + " json_eye1 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", + " }\n", + " json_eye2 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", + " }\n", + "else: json_eye1 = {\n", + " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", + " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", + " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", + " }\n" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'RecordedEye': 'Left',\n", + " 'AverageCalibrationError': [[0.29]],\n", + " 'MaximalCalibrationError': [[0.62]]}" + ] + }, + "execution_count": 98, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_eye1" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'RecordedEye': 'Right',\n", + " 'AverageCalibrationError': [[0.35]],\n", + " 'MaximalCalibrationError': [[1.21]]}" + ] + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_eye2" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": {}, + "outputs": [], + "source": [ + "base_json = {\n", + " \"Manufacturer\": \"SR-Research\",\n", + " \"DeviceSerialNumber\": _extract_DeviceSerialNumber(events),\n", + " \"EyeTrackingMethod\": _extract_EyeTrackingMethod(events),\n", + " \"ManufacturersModelName\": _extract_ManufacturersModelName(events),\n", + " \"CalibrationUnit\": _extract_CalibrationUnit(df_ms_reduced),\n", + " \"CalibrationType\": _extract_CalibrationType(df_ms_reduced),\n", + " \"PupilFitMethod\": _extract_PupilFitMethod(df_ms_reduced),\n", + " \"SamplingFrequency\": _extract_SamplingFrequency(df_ms_reduced),\n", + " \"StartTime\": _extract_StartTime(events),\n", + " \"StopTime\": _extract_StopTime(events),\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [], + "source": [ + "if _2eyesmode(df_ms_reduced) == True:\n", + " metadata_eye1 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", + " }\n", + " metadata_eye2 = {\n", + " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", + " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", + " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", + " }\n", + "else: \n", + " metadata_eye1 = {\n", + " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", + " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", + " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Manufacturer': 'SR-Research',\n", + " 'DeviceSerialNumber': 'CLG-BAF38',\n", + " 'EyeTrackingMethod': 'P-CR',\n", + " 'ManufacturersModelName': 'EYELINK II CL v5.12 May 12 2017',\n", + " 'CalibrationUnit': 'pixel',\n", + " 'CalibrationType': 'HV13',\n", + " 'PupilFitMethod': 'CENTROID',\n", + " 'SamplingFrequency': 1000,\n", + " 'StartTime': [767979],\n", + " 'StopTime': [819652],\n", + " 'RecordedEye': 'Left',\n", + " 'AverageCalibrationError': [[0.29]],\n", + " 'MaximalCalibrationError': [[0.62]]}" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_eye1 = base_json | metadata_eye1\n", + "json_eye1" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_output_filename(\n", + " output_dir: Path, input_file: Path, suffix: str, extension: str\n", + ") -> Path:\n", + " \"\"\"Generate output filename.\"\"\"\n", + " filename = Path(input_file).stem\n", + " if filename.endswith(suffix):\n", + " suffix = \"\"\n", + " return output_dir / f\"{filename}{suffix}.{extension}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "metadata": {}, + "outputs": [], + "source": [ + "if _2eyesmode == True:\n", + " output_filename_eye1 = generate_output_filename(\n", + " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", + " )\n", + " with open(output_filename_eye1, \"w\") as outfile:\n", + " json.dump(json_eye1, outfile, indent=4)\n", + "\n", + " output_filename_eye2 = generate_output_filename(\n", + " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye2_physio\", extension=\"json\"\n", + " )\n", + " with open(output_filename_eye2, \"w\") as outfile:\n", + " json.dump(json_eye2, outfile, indent=4)\n", + " \n", + " #e2b_log.info(f\"files generated: {output_filename_eye1} and {output_filename_eye2}\")\n", + "\n", + "else:\n", + " output_filename_eye1 = generate_output_filename(\n", + " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", + " )\n", + " with open(output_filename_eye1, \"w\") as outfile:\n", + " json.dump(json_eye1, outfile, indent=4)\n", + " \n", + " #e2b_log.info(f\"file generated: {output_filename_eye1}\")\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 3ef81562df8d7da73edcdebe4f16b667e95dc569 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 08:32:01 +0200 Subject: [PATCH 03/13] [pre-commit.ci] pre-commit autoupdate (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e202ac..be642ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-ast - id: check-case-conflict From fe7df431a1fe6ca3a109113003e204880dfeb372 Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Wed, 10 Apr 2024 17:56:57 +0200 Subject: [PATCH 04/13] ignore test ipynb --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1aa6ebe..e58cc53 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ eye2bids/_version.py *events.json *eyetrack.json *eyetrack.tsv +2eyes.ipynb tmp # General From 9244429e01dea1755a32368ccbdc683caaab487b Mon Sep 17 00:00:00 2001 From: Julia-Katharina Pfarr <111446107+julia-pfarr@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:58:04 +0200 Subject: [PATCH 05/13] Delete eye2bids/2eyes.ipynb --- eye2bids/2eyes.ipynb | 664 ------------------------------------------- 1 file changed, 664 deletions(-) delete mode 100644 eye2bids/2eyes.ipynb diff --git a/eye2bids/2eyes.ipynb b/eye2bids/2eyes.ipynb deleted file mode 100644 index 71eabc5..0000000 --- a/eye2bids/2eyes.ipynb +++ /dev/null @@ -1,664 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "\n", - "import gzip\n", - "import json\n", - "import subprocess\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import yaml\n", - "from rich.prompt import Prompt\n", - "from yaml.loader import SafeLoader" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "events_asc_file = \"/Users/julia/Desktop/NOWA.nosync/eyelink/2eyes/sub-99_task-FreeView_run-01_eyeData_events.asc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def _load_asc_file(events_asc_file: str | Path) -> list[str]:\n", - " with open(events_asc_file) as f:\n", - " return f.readlines()\n", - "\n", - "\n", - "def _load_asc_file_as_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # dataframe for events, all\n", - " events = _load_asc_file(events_asc_file)\n", - " return pd.DataFrame([ms.split() for ms in events if ms.startswith(\"MSG\")])\n", - "\n", - "\n", - "def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # reduced dataframe without MSG and sample columns\n", - " df_ms = _load_asc_file_as_df(events_asc_file)\n", - " return pd.DataFrame(df_ms.iloc[0:, 2:])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "events = _load_asc_file(events_asc_file)\n", - "df_ms = _load_asc_file_as_df(events_asc_file)\n", - "df_ms_reduced = _load_asc_file_as_reduced_df(events_asc_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def _calibrations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"CALIBRATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationType(df: pd.DataFrame) -> list[int]:\n", - " return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationCount(df: pd.DataFrame) -> int:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return len(_calibrations(df)) // 2\n", - " return len(_calibrations(df))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def _get_calibration_positions(df: pd.DataFrame) -> list[int]:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )[::2]\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationPosition(df: pd.DataFrame) -> list[list[int]]:\n", - " cal_pos = _get_calibration_positions(df)\n", - " cal_num = len(cal_pos) // _extract_CalibrationCount(df)\n", - "\n", - " CalibrationPosition: list[list[int]] = []\n", - "\n", - " if len(cal_pos) == 0:\n", - " return CalibrationPosition\n", - "\n", - " CalibrationPosition.extend(\n", - " cal_pos[i : i + cal_num] for i in range(0, len(cal_pos), cal_num)\n", - " )\n", - " return CalibrationPosition" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationUnit(df: pd.DataFrame) -> str:\n", - " if len(_get_calibration_positions(df)) == 0:\n", - " return \"\"\n", - "\n", - " cal_unit = (\n", - " (df[df[2] == \"VALIDATE\"][[13]])\n", - " .iloc[0:1, 0:1]\n", - " .to_string(header=False, index=False)\n", - " )\n", - " if cal_unit == \"pix.\":\n", - " return \"pixel\"\n", - " elif cal_unit in [\"cm\", \"mm\"]:\n", - " return cal_unit\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_EyeTrackingMethod(events: list[str]) -> str:\n", - " return (\n", - " pd.DataFrame(\n", - " \" \".join([tm for tm in events if tm.startswith(\">>>>>>>\")])\n", - " .replace(\")\", \",\")\n", - " .split(\",\")\n", - " )\n", - " .iloc[1:2]\n", - " .to_string(header=False, index=False)\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_DeviceSerialNumber(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([sl for sl in events if sl.startswith(\"** SERIAL NUMBER:\")])\n", - " .replace(\"** SERIAL NUMBER: \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_PupilFitMethod(df: pd.DataFrame) -> str:\n", - " return (df[df[2] == \"ELCL_PROC\"]).iloc[0:1, 1:2].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_SamplingFrequency(df: pd.DataFrame) -> int:\n", - " return int(df[df[2] == \"RECCFG\"].iloc[0:1, 2:3].to_string(header=False, index=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def _validations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"VALIDATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "def _has_validation(df: pd.DataFrame) -> bool:\n", - " return not _validations(df).empty" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ManufacturersModelName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ml for ml in events if ml.startswith(\"** EYELINK\")])\n", - " .replace(\"** \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]:\n", - " list_res = (\n", - " (df[df[2] == \"GAZE_COORDS\"])\n", - " .iloc[0:1, 3:5]\n", - " .to_string(header=False, index=False)\n", - " .replace(\".00\", \"\")\n", - " .split(\" \")\n", - " )\n", - " return [eval(i) for i in list_res]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_TaskName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ts for ts in events if ts.startswith(\"** RECORDED BY\")])\n", - " .replace(\"** RECORDED BY \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StartTime(events: list[str]) -> int:\n", - " StartTime = (\n", - " np.array(pd.DataFrame([st.split() for st in events if st.startswith(\"START\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StartTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple start times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the first start time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StartTime[0]\n", - " return StartTime" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StopTime(events: list[str]) -> int:\n", - " StopTime = (\n", - " np.array(pd.DataFrame([so.split() for so in events if so.startswith(\"END\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StopTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple stop times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the last stop time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StopTime[-1]\n", - " return StopTime" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return \"Left\"\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CLG-BAF38 P-CR EYELINK II CL v5.12 May 12 2017 pixel HV13 CENTROID 1000 [767979] [819652]\n" - ] - } - ], - "source": [ - "DeviceSerialNumber = _extract_DeviceSerialNumber(events)\n", - "EyeTrackingMethod = _extract_EyeTrackingMethod(events)\n", - "ManufacturersModelName = _extract_ManufacturersModelName(events)\n", - "CalibrationUnit = _extract_CalibrationUnit(df_ms_reduced)\n", - "CalibrationType = _extract_CalibrationType(df_ms_reduced)\n", - "PupilFitMethod = _extract_PupilFitMethod(df_ms_reduced)\n", - "SamplingFrequency = _extract_SamplingFrequency(df_ms_reduced)\n", - "StartTime = _extract_StartTime(events)\n", - "StopTime = _extract_StopTime(events)\n", - "\n", - "print(DeviceSerialNumber, EyeTrackingMethod, ManufacturersModelName, CalibrationUnit, CalibrationType, PupilFitMethod, SamplingFrequency, StartTime, StopTime)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "def _2eyesmode(df: pd.DataFrame) -> bool:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"LR\":\n", - " two_eyes = True\n", - " return two_eyes" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "_2eyesmode(df_ms_reduced)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return eye1\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\" " - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " if _extract_CalibrationCount(df) > 1:\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " return np.array(_validations(df)[[11]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " json_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " json_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: json_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }\n" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Right',\n", - " 'AverageCalibrationError': [[0.35]],\n", - " 'MaximalCalibrationError': [[1.21]]}" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye2" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "base_json = {\n", - " \"Manufacturer\": \"SR-Research\",\n", - " \"DeviceSerialNumber\": _extract_DeviceSerialNumber(events),\n", - " \"EyeTrackingMethod\": _extract_EyeTrackingMethod(events),\n", - " \"ManufacturersModelName\": _extract_ManufacturersModelName(events),\n", - " \"CalibrationUnit\": _extract_CalibrationUnit(df_ms_reduced),\n", - " \"CalibrationType\": _extract_CalibrationType(df_ms_reduced),\n", - " \"PupilFitMethod\": _extract_PupilFitMethod(df_ms_reduced),\n", - " \"SamplingFrequency\": _extract_SamplingFrequency(df_ms_reduced),\n", - " \"StartTime\": _extract_StartTime(events),\n", - " \"StopTime\": _extract_StopTime(events),\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " metadata_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " metadata_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: \n", - " metadata_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Manufacturer': 'SR-Research',\n", - " 'DeviceSerialNumber': 'CLG-BAF38',\n", - " 'EyeTrackingMethod': 'P-CR',\n", - " 'ManufacturersModelName': 'EYELINK II CL v5.12 May 12 2017',\n", - " 'CalibrationUnit': 'pixel',\n", - " 'CalibrationType': 'HV13',\n", - " 'PupilFitMethod': 'CENTROID',\n", - " 'SamplingFrequency': 1000,\n", - " 'StartTime': [767979],\n", - " 'StopTime': [819652],\n", - " 'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1 = base_json | metadata_eye1\n", - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_output_filename(\n", - " output_dir: Path, input_file: Path, suffix: str, extension: str\n", - ") -> Path:\n", - " \"\"\"Generate output filename.\"\"\"\n", - " filename = Path(input_file).stem\n", - " if filename.endswith(suffix):\n", - " suffix = \"\"\n", - " return output_dir / f\"{filename}{suffix}.{extension}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode == True:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - "\n", - " output_filename_eye2 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye2_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye2, \"w\") as outfile:\n", - " json.dump(json_eye2, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"files generated: {output_filename_eye1} and {output_filename_eye2}\")\n", - "\n", - "else:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"file generated: {output_filename_eye1}\")\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From d5bef7fe3f719deb0a66371d04f4dea1624c31e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:04:06 +0000 Subject: [PATCH 06/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- eye2bids/edf2bids.py | 190 ++++++++++++++++++++++------------------- tests/test_edf2bids.py | 14 ++- 2 files changed, 115 insertions(+), 89 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 39dddbf..97a6b96 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -115,6 +115,7 @@ def _convert_edf_to_asc_samples(input_file: str | Path) -> Path: subprocess.run(["edf2asc", "-y", "-s", input_file, "-o", samples_asc_file]) return Path(samples_asc_file).with_suffix(".asc") + def _2eyesmode(df: pd.DataFrame) -> bool: eye = df[df[2] == "RECCFG"].iloc[0:1, 5:6].to_string(header=False, index=False) if eye == "LR": @@ -123,9 +124,11 @@ def _2eyesmode(df: pd.DataFrame) -> bool: two_eyes = False return two_eyes + def _calibrations(df: pd.DataFrame) -> pd.DataFrame: return df[df[3] == "CALIBRATION"] + def _extract_CalibrationType(df: pd.DataFrame) -> list[int]: return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False) @@ -245,7 +248,7 @@ def _extract_RecordedEye(df: pd.DataFrame) -> str: return "Right" elif eye == "LR": return ["Left", "Right"] - return "" + return "" def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]: @@ -319,6 +322,7 @@ def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame: df_ms = _load_asc_file_as_df(events_asc_file) return pd.DataFrame(df_ms.iloc[0:, 2:]) + def edf2bids( input_file: str | Path | None = None, metadata_file: str | Path | None = None, @@ -352,89 +356,94 @@ def edf2bids( with open(metadata_file) as f: metadata = yaml.load(f, Loader=SafeLoader) - # eye-physio.json Metadata base_json = { - "Columns": [ "x_coordinate", "y_coordinate", "pupil_size", "timestamp"], - "x_coordinate": { - "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", - "Units": "a.u." - }, - "y_coordinate": { - "Description": "Gaze position y-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", - "Units": "a.u." - }, - "pupil_size": { - "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", - "Units": "a.u." - }, - "timestamp": { - "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." - }, - "Manufacturer": "SR-Research", - "ManufacturersModelName": _extract_ManufacturersModelName(events), - "DeviceSerialNumber": _extract_DeviceSerialNumber(events), - "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), - "SoftwareVersion": metadata.get("SoftwareVersion"), - "EyeCameraSettings": metadata.get("EyeCameraSettings"), - "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), - "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), - "GazeMappingSettings": metadata.get("GazeMappingSettings"), - "RawDataFilters": metadata.get("RawDataFilters"), - "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), - "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), - "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), - "EyeTrackingMethod": _extract_EyeTrackingMethod(events), - "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), - "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), - "StartTime": _extract_StartTime(events), - "StopTime": _extract_StopTime(events), - "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), - "CalibrationType": _extract_CalibrationType(df_ms_reduced), - "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), - "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced) - } - + "Columns": ["x_coordinate", "y_coordinate", "pupil_size", "timestamp"], + "x_coordinate": { + "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u.", + }, + "y_coordinate": { + "Description": "Gaze position y-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", + "Units": "a.u.", + }, + "pupil_size": { + "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", + "Units": "a.u.", + }, + "timestamp": { + "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." + }, + "Manufacturer": "SR-Research", + "ManufacturersModelName": _extract_ManufacturersModelName(events), + "DeviceSerialNumber": _extract_DeviceSerialNumber(events), + "EnvironmentCoordinates": metadata.get("EnvironmentCoordinates"), + "SoftwareVersion": metadata.get("SoftwareVersion"), + "EyeCameraSettings": metadata.get("EyeCameraSettings"), + "EyeTrackerDistance": metadata.get("EyeTrackerDistance"), + "FeatureDetectionSettings": metadata.get("FeatureDetectionSettings"), + "GazeMappingSettings": metadata.get("GazeMappingSettings"), + "RawDataFilters": metadata.get("RawDataFilters"), + "SampleCoordinateSystem": metadata.get("SampleCoordinateSystem"), + "SampleCoordinateUnits": metadata.get("SampleCoordinateUnits"), + "ScreenAOIDefinition": metadata.get("ScreenAOIDefinition"), + "EyeTrackingMethod": _extract_EyeTrackingMethod(events), + "PupilFitMethod": _extract_PupilFitMethod(df_ms_reduced), + "SamplingFrequency": _extract_SamplingFrequency(df_ms_reduced), + "StartTime": _extract_StartTime(events), + "StopTime": _extract_StopTime(events), + "CalibrationUnit": _extract_CalibrationUnit(df_ms_reduced), + "CalibrationType": _extract_CalibrationType(df_ms_reduced), + "CalibrationCount": _extract_CalibrationCount(df_ms_reduced), + "CalibrationPosition": _extract_CalibrationPosition(df_ms_reduced), + } + if _2eyesmode(df_ms_reduced) == True: metadata_eye1 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[0]) - } - + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[0]), + } + metadata_eye2 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[1::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[1::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[1]) + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[1::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[1::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)[1]), } else: metadata_eye1 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) - } - + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)), + } + json_eye1 = base_json | metadata_eye1 if _2eyesmode(df_ms_reduced) == True: json_eye2 = base_json | metadata_eye2 # to json - + output_filename_eye1 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physio", + extension="json", + ) with open(output_filename_eye1, "w") as outfile: - json.dump(json_eye1, outfile, indent=4) + json.dump(json_eye1, outfile, indent=4) e2b_log.info(f"file generated: {output_filename_eye1}") if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physio", + extension="json", + ) with open(output_filename_eye2, "w") as outfile: json.dump(json_eye2, outfile, indent=4) - + e2b_log.info(f"file generated: {output_filename_eye2}") # physioevents.json Metadata @@ -443,16 +452,12 @@ def edf2bids( "Columns": ["onset", "duration", "trial_type", "blink", "message"], "Description": "Messages logged by the measurement device", "ForeignIndexColumn": "timestamp", - "blink": { - "Description": "One indicates if the eye was closed, zero if open." - }, - "message": { - "Description": "String messages logged by the eye-tracker." - }, + "blink": {"Description": "One indicates if the eye was closed, zero if open."}, + "message": {"Description": "String messages logged by the eye-tracker."}, "trial_type": { - "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." + "Description": "Event type as identified by the eye-tracker's model (either 'n/a' if not applicabble, 'fixation', or 'saccade')." }, - "TaskName": _extract_TaskName(events), + "TaskName": _extract_TaskName(events), "InstitutionAddress": metadata.get("InstitutionAddress"), "InstitutionName": metadata.get("InstitutionName"), "StimulusPresentation": { @@ -460,13 +465,15 @@ def edf2bids( "ScreenRefreshRate": metadata.get("ScreenRefreshRate"), "ScreenSize": metadata.get("ScreenSize"), "ScreenResolution": _extract_ScreenResolution(df_ms_reduced), - } + }, } - output_filename_eye1 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physioevents", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physioevents", + extension="json", + ) with open(output_filename_eye1, "w") as outfile: json.dump(events_json, outfile, indent=4) @@ -475,11 +482,14 @@ def edf2bids( if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physioevents", extension="json" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physioevents", + extension="json", + ) with open(output_filename_eye2, "w") as outfile: json.dump(events_json, outfile, indent=4) - + e2b_log.info(f"file generated: {output_filename_eye2}") # Samples to dataframe @@ -492,16 +502,23 @@ def edf2bids( ) samples = pd.read_csv(samples_asc_file, sep="\t", header=None) - samples_eye1 = pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]).map(lambda x: x.strip() if isinstance(x, str) else x).replace(".", np.nan, regex=False) - + samples_eye1 = ( + pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]) + .map(lambda x: x.strip() if isinstance(x, str) else x) + .replace(".", np.nan, regex=False) + ) + if _2eyesmode(df_ms_reduced) == True: samples_eye2 = pd.DataFrame(samples.iloc[:, [4, 5, 6, 0]]) # Samples to eye_physio.tsv.gz output_filename_eye1 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye1_physio", extension="tsv.gz" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye1_physio", + extension="tsv.gz", + ) content = samples_eye1.to_csv(sep="\t", index=False, na_rep="n/a", header=None) with gzip.open(output_filename_eye1, "wb") as f: f.write(content.encode()) @@ -511,19 +528,20 @@ def edf2bids( if _2eyesmode(df_ms_reduced) == True: output_filename_eye2 = generate_output_filename( - output_dir=output_dir, input_file=input_file, suffix="_recording-eye2_physio", extension="tsv.gz" - ) + output_dir=output_dir, + input_file=input_file, + suffix="_recording-eye2_physio", + extension="tsv.gz", + ) content = samples_eye2.to_csv(sep="\t", index=False, na_rep="n/a", header=None) with gzip.open(output_filename_eye2, "wb") as f: f.write(content.encode()) - - e2b_log.info(f"file generated: {output_filename_eye2}") + e2b_log.info(f"file generated: {output_filename_eye2}") # Messages and events to physioevents.tsv.gz - tbc - def generate_output_filename( output_dir: Path, input_file: Path, suffix: str, extension: str ) -> Path: diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 9ada11f..e3b57a8 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -52,7 +52,9 @@ def test_edf_end_to_end(metadata_file, eyelink_test_data_dir): output_dir=output_dir, ) - expected_events_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" + expected_events_sidecar = ( + output_dir / f"{input_file.stem}_recording-eye1_physioevents.json" + ) assert expected_events_sidecar.exists() with open(expected_events_sidecar) as f: events = json.load(f) @@ -88,6 +90,7 @@ def test_edf_nan_in_tsv(eyelink_test_data_dir): count = sum(i == "." for i in df[0]) assert count == 0 + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_2files_eye1(eyelink_test_data_dir): """Check that for datafile with 2eyes 2 eye1 file is created and check input. @@ -105,13 +108,16 @@ def test_2files_eye1(eyelink_test_data_dir): output_dir=output_dir, ) - expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye1_physio.json" + expected_eyetrack_sidecar = ( + output_dir / f"{input_file.stem}_recording-eye1_physio.json" + ) assert expected_eyetrack_sidecar.exists() with open(expected_eyetrack_sidecar) as f: eyetrack = json.load(f) assert eyetrack["AverageCalibrationError"] == [[0.29]] assert eyetrack["RecordedEye"] == "Left" + @pytest.mark.skipif(not _check_edf2asc_present(), reason="edf2asc missing") def test_2files_eye2(eyelink_test_data_dir): """Check that for datafile with 2eyes 2 eye2 file is created and check input. @@ -129,7 +135,9 @@ def test_2files_eye2(eyelink_test_data_dir): output_dir=output_dir, ) - expected_eyetrack_sidecar = output_dir / f"{input_file.stem}_recording-eye2_physio.json" + expected_eyetrack_sidecar = ( + output_dir / f"{input_file.stem}_recording-eye2_physio.json" + ) assert expected_eyetrack_sidecar.exists() with open(expected_eyetrack_sidecar) as f: eyetrack = json.load(f) From b5f971a7d02df1c9cb35595593fe8549e28cbe6f Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Thu, 11 Apr 2024 11:00:58 +0200 Subject: [PATCH 07/13] change how json dicts are combined because tests were failing --- eye2bids/2eyes.ipynb | 664 ------------------------------------------- eye2bids/edf2bids.py | 4 +- 2 files changed, 2 insertions(+), 666 deletions(-) delete mode 100644 eye2bids/2eyes.ipynb diff --git a/eye2bids/2eyes.ipynb b/eye2bids/2eyes.ipynb deleted file mode 100644 index 71eabc5..0000000 --- a/eye2bids/2eyes.ipynb +++ /dev/null @@ -1,664 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "\n", - "import gzip\n", - "import json\n", - "import subprocess\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import yaml\n", - "from rich.prompt import Prompt\n", - "from yaml.loader import SafeLoader" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "events_asc_file = \"/Users/julia/Desktop/NOWA.nosync/eyelink/2eyes/sub-99_task-FreeView_run-01_eyeData_events.asc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def _load_asc_file(events_asc_file: str | Path) -> list[str]:\n", - " with open(events_asc_file) as f:\n", - " return f.readlines()\n", - "\n", - "\n", - "def _load_asc_file_as_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # dataframe for events, all\n", - " events = _load_asc_file(events_asc_file)\n", - " return pd.DataFrame([ms.split() for ms in events if ms.startswith(\"MSG\")])\n", - "\n", - "\n", - "def _load_asc_file_as_reduced_df(events_asc_file: str | Path) -> pd.DataFrame:\n", - " # reduced dataframe without MSG and sample columns\n", - " df_ms = _load_asc_file_as_df(events_asc_file)\n", - " return pd.DataFrame(df_ms.iloc[0:, 2:])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "events = _load_asc_file(events_asc_file)\n", - "df_ms = _load_asc_file_as_df(events_asc_file)\n", - "df_ms_reduced = _load_asc_file_as_reduced_df(events_asc_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def _calibrations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"CALIBRATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationType(df: pd.DataFrame) -> list[int]:\n", - " return _calibrations(df).iloc[0:1, 2:3].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationCount(df: pd.DataFrame) -> int:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return len(_calibrations(df)) // 2\n", - " return len(_calibrations(df))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "def _get_calibration_positions(df: pd.DataFrame) -> list[int]:\n", - " if _extract_RecordedEye(df) == \"Both\":\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )[::2]\n", - " return (\n", - " np.array(df[df[2] == \"VALIDATE\"][8].str.split(\",\", expand=True))\n", - " .astype(int)\n", - " .tolist()\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationPosition(df: pd.DataFrame) -> list[list[int]]:\n", - " cal_pos = _get_calibration_positions(df)\n", - " cal_num = len(cal_pos) // _extract_CalibrationCount(df)\n", - "\n", - " CalibrationPosition: list[list[int]] = []\n", - "\n", - " if len(cal_pos) == 0:\n", - " return CalibrationPosition\n", - "\n", - " CalibrationPosition.extend(\n", - " cal_pos[i : i + cal_num] for i in range(0, len(cal_pos), cal_num)\n", - " )\n", - " return CalibrationPosition" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_CalibrationUnit(df: pd.DataFrame) -> str:\n", - " if len(_get_calibration_positions(df)) == 0:\n", - " return \"\"\n", - "\n", - " cal_unit = (\n", - " (df[df[2] == \"VALIDATE\"][[13]])\n", - " .iloc[0:1, 0:1]\n", - " .to_string(header=False, index=False)\n", - " )\n", - " if cal_unit == \"pix.\":\n", - " return \"pixel\"\n", - " elif cal_unit in [\"cm\", \"mm\"]:\n", - " return cal_unit\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_EyeTrackingMethod(events: list[str]) -> str:\n", - " return (\n", - " pd.DataFrame(\n", - " \" \".join([tm for tm in events if tm.startswith(\">>>>>>>\")])\n", - " .replace(\")\", \",\")\n", - " .split(\",\")\n", - " )\n", - " .iloc[1:2]\n", - " .to_string(header=False, index=False)\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_DeviceSerialNumber(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([sl for sl in events if sl.startswith(\"** SERIAL NUMBER:\")])\n", - " .replace(\"** SERIAL NUMBER: \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_PupilFitMethod(df: pd.DataFrame) -> str:\n", - " return (df[df[2] == \"ELCL_PROC\"]).iloc[0:1, 1:2].to_string(header=False, index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_SamplingFrequency(df: pd.DataFrame) -> int:\n", - " return int(df[df[2] == \"RECCFG\"].iloc[0:1, 2:3].to_string(header=False, index=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def _validations(df: pd.DataFrame) -> pd.DataFrame:\n", - " return df[df[3] == \"VALIDATION\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "def _has_validation(df: pd.DataFrame) -> bool:\n", - " return not _validations(df).empty" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ManufacturersModelName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ml for ml in events if ml.startswith(\"** EYELINK\")])\n", - " .replace(\"** \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_ScreenResolution(df: pd.DataFrame) -> list[int]:\n", - " list_res = (\n", - " (df[df[2] == \"GAZE_COORDS\"])\n", - " .iloc[0:1, 3:5]\n", - " .to_string(header=False, index=False)\n", - " .replace(\".00\", \"\")\n", - " .split(\" \")\n", - " )\n", - " return [eval(i) for i in list_res]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_TaskName(events: list[str]) -> str:\n", - " return (\n", - " \" \".join([ts for ts in events if ts.startswith(\"** RECORDED BY\")])\n", - " .replace(\"** RECORDED BY \", \"\")\n", - " .replace(\"\\n\", \"\")\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StartTime(events: list[str]) -> int:\n", - " StartTime = (\n", - " np.array(pd.DataFrame([st.split() for st in events if st.startswith(\"START\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StartTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple start times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the first start time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StartTime[0]\n", - " return StartTime" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_StopTime(events: list[str]) -> int:\n", - " StopTime = (\n", - " np.array(pd.DataFrame([so.split() for so in events if so.startswith(\"END\")])[1])\n", - " .astype(int)\n", - " .tolist()\n", - " )\n", - " if len(StopTime) > 1:\n", - " e2b_log.info(\n", - " \"\"\"Your input file contains multiple stop times.\\n\n", - " As this is not seen as good practice in eyetracking experiments, \\n\n", - " only the last stop time will be kept for the metadata file. \\n\n", - " Please consider changing your code accordingly\n", - " for future eyetracking experiments.\\n\"\"\"\n", - " )\n", - " return StopTime[-1]\n", - " return StopTime" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return \"Left\"\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CLG-BAF38 P-CR EYELINK II CL v5.12 May 12 2017 pixel HV13 CENTROID 1000 [767979] [819652]\n" - ] - } - ], - "source": [ - "DeviceSerialNumber = _extract_DeviceSerialNumber(events)\n", - "EyeTrackingMethod = _extract_EyeTrackingMethod(events)\n", - "ManufacturersModelName = _extract_ManufacturersModelName(events)\n", - "CalibrationUnit = _extract_CalibrationUnit(df_ms_reduced)\n", - "CalibrationType = _extract_CalibrationType(df_ms_reduced)\n", - "PupilFitMethod = _extract_PupilFitMethod(df_ms_reduced)\n", - "SamplingFrequency = _extract_SamplingFrequency(df_ms_reduced)\n", - "StartTime = _extract_StartTime(events)\n", - "StopTime = _extract_StopTime(events)\n", - "\n", - "print(DeviceSerialNumber, EyeTrackingMethod, ManufacturersModelName, CalibrationUnit, CalibrationType, PupilFitMethod, SamplingFrequency, StartTime, StopTime)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "def _2eyesmode(df: pd.DataFrame) -> bool:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"LR\":\n", - " two_eyes = True\n", - " return two_eyes" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "_2eyesmode(df_ms_reduced)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_RecordedEye(df: pd.DataFrame) -> str:\n", - " eye = df[df[2] == \"RECCFG\"].iloc[0:1, 5:6].to_string(header=False, index=False)\n", - " if eye == \"L\":\n", - " return eye1\n", - " elif eye == \"R\":\n", - " return \"Right\"\n", - " elif eye == \"LR\":\n", - " return [\"Left\", \"Right\"]\n", - " return \"\" " - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_AverageCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " if _extract_CalibrationCount(df) > 1:\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()\n", - " return np.array(_validations(df)[[9]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def _extract_MaximalCalibrationError(df: pd.DataFrame) -> list[float]:\n", - " if not _has_validation(df):\n", - " return []\n", - " return np.array(_validations(df)[[11]]).astype(float).tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " json_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " json_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: json_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }\n" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'RecordedEye': 'Right',\n", - " 'AverageCalibrationError': [[0.35]],\n", - " 'MaximalCalibrationError': [[1.21]]}" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye2" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "base_json = {\n", - " \"Manufacturer\": \"SR-Research\",\n", - " \"DeviceSerialNumber\": _extract_DeviceSerialNumber(events),\n", - " \"EyeTrackingMethod\": _extract_EyeTrackingMethod(events),\n", - " \"ManufacturersModelName\": _extract_ManufacturersModelName(events),\n", - " \"CalibrationUnit\": _extract_CalibrationUnit(df_ms_reduced),\n", - " \"CalibrationType\": _extract_CalibrationType(df_ms_reduced),\n", - " \"PupilFitMethod\": _extract_PupilFitMethod(df_ms_reduced),\n", - " \"SamplingFrequency\": _extract_SamplingFrequency(df_ms_reduced),\n", - " \"StartTime\": _extract_StartTime(events),\n", - " \"StopTime\": _extract_StopTime(events),\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode(df_ms_reduced) == True:\n", - " metadata_eye1 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[0]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[0::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[0::2])\n", - " }\n", - " metadata_eye2 = {\n", - " \"RecordedEye\": (_extract_RecordedEye(df_ms_reduced)[1]),\n", - " \"AverageCalibrationError\": (_extract_AverageCalibrationError(df_ms)[1::2]),\n", - " \"MaximalCalibrationError\": (_extract_MaximalCalibrationError(df_ms)[1::2])\n", - " }\n", - "else: \n", - " metadata_eye1 = {\n", - " \"RecordedEye\": _extract_RecordedEye(df_ms_reduced),\n", - " \"AverageCalibrationError\": _extract_AverageCalibrationError(df_ms),\n", - " \"MaximalCalibrationError\": _extract_MaximalCalibrationError(df_ms)\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Manufacturer': 'SR-Research',\n", - " 'DeviceSerialNumber': 'CLG-BAF38',\n", - " 'EyeTrackingMethod': 'P-CR',\n", - " 'ManufacturersModelName': 'EYELINK II CL v5.12 May 12 2017',\n", - " 'CalibrationUnit': 'pixel',\n", - " 'CalibrationType': 'HV13',\n", - " 'PupilFitMethod': 'CENTROID',\n", - " 'SamplingFrequency': 1000,\n", - " 'StartTime': [767979],\n", - " 'StopTime': [819652],\n", - " 'RecordedEye': 'Left',\n", - " 'AverageCalibrationError': [[0.29]],\n", - " 'MaximalCalibrationError': [[0.62]]}" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "json_eye1 = base_json | metadata_eye1\n", - "json_eye1" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_output_filename(\n", - " output_dir: Path, input_file: Path, suffix: str, extension: str\n", - ") -> Path:\n", - " \"\"\"Generate output filename.\"\"\"\n", - " filename = Path(input_file).stem\n", - " if filename.endswith(suffix):\n", - " suffix = \"\"\n", - " return output_dir / f\"{filename}{suffix}.{extension}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "metadata": {}, - "outputs": [], - "source": [ - "if _2eyesmode == True:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - "\n", - " output_filename_eye2 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye2_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye2, \"w\") as outfile:\n", - " json.dump(json_eye2, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"files generated: {output_filename_eye1} and {output_filename_eye2}\")\n", - "\n", - "else:\n", - " output_filename_eye1 = generate_output_filename(\n", - " output_dir=output_dir, input_file=input_file, suffix=\"recording-eye1_physio\", extension=\"json\"\n", - " )\n", - " with open(output_filename_eye1, \"w\") as outfile:\n", - " json.dump(json_eye1, outfile, indent=4)\n", - " \n", - " #e2b_log.info(f\"file generated: {output_filename_eye1}\")\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 39dddbf..2e3e0e1 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -414,9 +414,9 @@ def edf2bids( "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) } - json_eye1 = base_json | metadata_eye1 + json_eye1 = dict(base_json, **metadata_eye1) if _2eyesmode(df_ms_reduced) == True: - json_eye2 = base_json | metadata_eye2 + json_eye2 = dict(base_json, **metadata_eye2) # to json From 8d4a0ff17c35a9897ace518b543f22bcef8626e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:02:49 +0000 Subject: [PATCH 08/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- eye2bids/edf2bids.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 6b968df..ee2660c 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -412,12 +412,12 @@ def edf2bids( } else: metadata_eye1 = { - "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), - "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), - "RecordedEye": (_extract_RecordedEye(df_ms_reduced)) - } - - json_eye1 = dict(base_json, **metadata_eye1) + "AverageCalibrationError": (_extract_AverageCalibrationError(df_ms)[0::2]), + "MaximalCalibrationError": (_extract_MaximalCalibrationError(df_ms)[0::2]), + "RecordedEye": (_extract_RecordedEye(df_ms_reduced)), + } + + json_eye1 = dict(base_json, **metadata_eye1) if _2eyesmode(df_ms_reduced) == True: json_eye2 = dict(base_json, **metadata_eye2) From cbc441f3b31d3617c3b34a534e10f2a806018fb1 Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Fri, 12 Apr 2024 12:20:48 +0200 Subject: [PATCH 09/13] fix test test_edf_nan_in_tsv --- eye2bids/edf2bids.py | 4 ++-- tests/test_edf2bids.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 6b968df..9b8797e 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -503,13 +503,13 @@ def edf2bids( samples = pd.read_csv(samples_asc_file, sep="\t", header=None) samples_eye1 = ( - pd.DataFrame(samples.iloc[:, [2, 1, 3, 0]]) + pd.DataFrame(samples.iloc[:, 0:4]) .map(lambda x: x.strip() if isinstance(x, str) else x) .replace(".", np.nan, regex=False) ) if _2eyesmode(df_ms_reduced) == True: - samples_eye2 = pd.DataFrame(samples.iloc[:, [4, 5, 6, 0]]) + samples_eye2 = pd.DataFrame(samples.iloc[:, [0, 4, 5, 6]]) # Samples to eye_physio.tsv.gz diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index e3b57a8..74452d1 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -86,7 +86,7 @@ def test_edf_nan_in_tsv(eyelink_test_data_dir): ) expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - df = pd.read_csv(expected_eyetrack_tsv, sep="\t") + df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header = None) count = sum(i == "." for i in df[0]) assert count == 0 @@ -163,7 +163,7 @@ def test_number_columns_2eyes_tsv(eyelink_test_data_dir): ) expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - df = pd.read_csv(expected_eyetrack_tsv, sep="\t") + df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header = None) number_columns = len(df.columns) assert number_columns == 4 @@ -187,7 +187,7 @@ def test_number_columns_1eye_tsv(eyelink_test_data_dir): ) expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - df = pd.read_csv(expected_eyetrack_tsv, sep="\t") + df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header = None) number_columns = len(df.columns) assert number_columns == 4 From 11c6ca019753a8e7ae33140edf813da2c3e1dc49 Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Fri, 12 Apr 2024 12:21:23 +0200 Subject: [PATCH 10/13] add pandas version and python3.12 --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a1d717a..25ba8bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,11 +18,12 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11" + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12" ] dependencies = [ "numpy", - "pandas", + "pandas>2.0", "pyaml", "rich_argparse", "rich" From e8a4bcb900ed6edebf86c666e1d9bb999ada9e83 Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Fri, 12 Apr 2024 12:25:42 +0200 Subject: [PATCH 11/13] put timestamp first in samples --- eye2bids/edf2bids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eye2bids/edf2bids.py b/eye2bids/edf2bids.py index 475ec24..40a243b 100644 --- a/eye2bids/edf2bids.py +++ b/eye2bids/edf2bids.py @@ -359,6 +359,9 @@ def edf2bids( # eye-physio.json Metadata base_json = { "Columns": ["x_coordinate", "y_coordinate", "pupil_size", "timestamp"], + "timestamp": { + "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." + }, "x_coordinate": { "Description": "Gaze position x-coordinate of the recorded eye, in the coordinate units specified in the corresponding metadata sidecar.", "Units": "a.u.", @@ -371,9 +374,6 @@ def edf2bids( "Description": "Pupil area of the recorded eye as calculated by the eye-tracker in arbitrary units (see EyeLink's documentation for conversion).", "Units": "a.u.", }, - "timestamp": { - "Description": "Timestamp issued by the eye-tracker indexing the continuous recordings corresponding to the sampled eye." - }, "Manufacturer": "SR-Research", "ManufacturersModelName": _extract_ManufacturersModelName(events), "DeviceSerialNumber": _extract_DeviceSerialNumber(events), From 9029933e9162641d958d1aeecacabdc1f99b6222 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 10:29:02 +0000 Subject: [PATCH 12/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_edf2bids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_edf2bids.py b/tests/test_edf2bids.py index 74452d1..aa231b0 100644 --- a/tests/test_edf2bids.py +++ b/tests/test_edf2bids.py @@ -86,7 +86,7 @@ def test_edf_nan_in_tsv(eyelink_test_data_dir): ) expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header = None) + df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header=None) count = sum(i == "." for i in df[0]) assert count == 0 @@ -163,7 +163,7 @@ def test_number_columns_2eyes_tsv(eyelink_test_data_dir): ) expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header = None) + df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header=None) number_columns = len(df.columns) assert number_columns == 4 @@ -187,7 +187,7 @@ def test_number_columns_1eye_tsv(eyelink_test_data_dir): ) expected_eyetrack_tsv = output_dir / f"{input_file.stem}_recording-eye1_physio.tsv.gz" - df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header = None) + df = pd.read_csv(expected_eyetrack_tsv, sep="\t", header=None) number_columns = len(df.columns) assert number_columns == 4 From b44d7d7cabd2c2bedcf486c6838144ef535a2b1d Mon Sep 17 00:00:00 2001 From: Julia Pfarr Date: Fri, 12 Apr 2024 12:47:04 +0200 Subject: [PATCH 13/13] remove python3.8 from supported python versions --- .github/workflows/tests.yml | 2 +- pyproject.toml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45d9fa9..3939b49 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 25ba8bd..ca3feb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ classifiers = [ "Operating System :: POSIX", "Operating System :: Unix", "Operating System :: MacOS", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -33,7 +32,7 @@ dynamic = ["version"] license = {text = "MIT"} name = "eye2bids" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" [project.optional-dependencies] # A combination of dependencies useful for developers