data_formatter.py

import pandas as pd
import sys
import glob
from tqdm import tqdm

# Specify the paths to the raw data from a single day.
PATH = "../data/complete_data/raw_data"

from tempfile import TemporaryFile


def list_to_df(temp):
    lat = []
    lon = []
    a_x = []
    a_y = []
    v = []
    frame = []
    for i in range(
        7, len(temp) - 6, 6
    ):  # start range from 3 for sample, and 7 for complete. Check this to make it consistent with the format of raw data
        lon.append(temp[i + 2])  # check lat
        lat.append(temp[i + 1])  # check lon
        v.append(temp[i + 3])
        a_x.append(temp[i + 4])
        a_y.append(temp[i + 5])
        frame.append(temp[i + 6])
    df_meta = pd.DataFrame(
        {"id": [temp[0]], "type": [temp[1]], "dist": [temp[6]], "avg_speed": [temp[7]]}
    )
    df_trajectory = pd.DataFrame(
        {
            "id": temp[0],
            "frame": frame,
            "lon": lon,
            "lat": lat,
            "v": v,
            "a_x": a_x,
            "a_y": a_y,
        }
    )
    return df_meta, df_trajectory


def write_list(file):
    with open(file) as f, TemporaryFile("w+") as t:
        next(f)
        list_meta = []
        list_trajectory = []
        for line in tqdm(f):
            h, ln = line, len(line.split("; "))
            temp = h.strip().split("; ")
            m, t = list_to_df(temp)
            list_meta.append(m)
            list_trajectory.append(t)
    return list_meta, list_trajectory


if __name__ == "__main__":
    paths = glob.glob(PATH + "/*.csv")
    print(paths)
    for file in paths:
        print(file)
        filename = file.split("_")[-2]
        # filenames will be the data collection hours e.g. 0900-0930
        print(filename)
        list_meta, list_trajectory = write_list(file)
        # save meta file
        pd.concat(list_meta).to_csv(
            "../data/long/meta_" + filename + ".csv", index=None
        )

        # save trajectory data
        pd.concat(list_trajectory).to_csv(
            "../data/long/" + filename + ".csv", index=None
        )