diff --git a/.codespellrc b/.codespellrc index d558690a..20e3b294 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,4 +1,4 @@ [codespell] -skip = .git,Gemfile.lock,_sass,_site,tools/convert_to_md.py +skip = .git,Gemfile.lock,_sass,_site,tools/convert_to_md.py,_pages/bids_timeline.html ignore-words-list = foo,bar,baz builtin = clear,rare,en-GB_to_en-US diff --git a/.gitignore b/.gitignore index 64bf13d8..08538db2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ Gemfile.lock _site/* .DS_Store +__pycache__ .sass-cache tools/inputs tools/*.html diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25df9165..a92a437c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - - id: check-added-large-files + # - id: check-added-large-files - id: check-case-conflict - repo: https://github.com/codespell-project/codespell @@ -15,3 +15,10 @@ repos: hooks: - id: codespell args: ["--config=.codespellrc"] + + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + name: black + args: [--config, pyproject.toml, --verbose] diff --git a/_data/beps.yml b/_data/beps.yml index b1b4aaa4..c4eec970 100644 --- a/_data/beps.yml +++ b/_data/beps.yml @@ -13,6 +13,9 @@ content: - raw blocking: Looking for a new leader. + google_doc_created: 2017-04 + pull_request_created: + pull_request_merged: - number: "011" title: Structural preprocessing derivatives @@ -32,6 +35,9 @@ blocking: | Blocked by BIDS-validator extending and PyBIDS implementation for common derivatives + google_doc_created: 2017-08 + pull_request_created: 2020-06 + pull_request_merged: - number: "012" title: Functional preprocessing derivatives @@ -46,6 +52,9 @@ content: - derivative blocking: Blocked by BIDS-validator extending and PyBIDS implementation + google_doc_created: 2018-10 + pull_request_created: 2020-06 + pull_request_merged: - number: "014" title: Affine transformations and nonlinear field warps @@ -59,6 +68,9 @@ content: - derivative blocking: In progress. + google_doc_created: 2017-08 + pull_request_created: + pull_request_merged: - number: "016" title: Diffusion weighted imaging derivatives @@ -73,6 +85,9 @@ content: - derivative blocking: None + google_doc_created: + pull_request_created: + pull_request_merged: - number: "017" title: Generic BIDS connectivity data schema @@ -85,6 +100,9 @@ content: - derivative blocking: None + google_doc_created: 2017-05 + pull_request_created: + pull_request_merged: - number: "020" title: Eye Tracking including Gaze Position and Pupil Size @@ -96,6 +114,9 @@ content: - raw blocking: None + google_doc_created: 2018-04 + pull_request_created: 2022-06 + pull_request_merged: - number: "021" title: Common Electrophysiological Derivatives @@ -113,6 +134,9 @@ content: - derivative blocking: Mostly on hold. + google_doc_created: 2018-05 + pull_request_created: + pull_request_merged: - number: "022" title: Magnetic Resonance Spectroscopy (MRS) @@ -127,6 +151,9 @@ content: - raw blocking: None + google_doc_created: 2018-05 + pull_request_created: 2023-06 + pull_request_merged: - number: "023" title: PET Preprocessing derivatives @@ -142,6 +169,9 @@ content: - derivative blocking: Progressing and soliciting input from senior PET experts. + google_doc_created: 2018-08 + pull_request_created: + pull_request_merged: - number: "024" title: Computed Tomography scan (CT) @@ -151,6 +181,9 @@ content: - raw blocking: None + google_doc_created: 2018-11 + pull_request_created: + pull_request_merged: - number: "026" title: Microelectrode Recordings @@ -160,6 +193,9 @@ content: - raw blocking: Searching for a new leader. + google_doc_created: 2018-04 + pull_request_created: + pull_request_merged: - number: "027" title: BIDS Applications 2.0 @@ -174,6 +210,9 @@ content: - metadata blocking: None + google_doc_created: 2019-05 + pull_request_created: + pull_request_merged: - number: "028" title: Provenance @@ -190,6 +229,9 @@ content: - metadata blocking: None + google_doc_created: 2018-08 + pull_request_created: + pull_request_merged: - number: "032" title: Animal electrophysiology @@ -209,6 +251,9 @@ content: - raw blocking: None + google_doc_created: 2020-12 + pull_request_created: 2022-11 + pull_request_merged: - number: "033" title: Advanced Diffusion Weighted Imaging (aDWI) @@ -226,6 +271,9 @@ content: - raw blocking: None + google_doc_created: 2021-04 + pull_request_created: + pull_request_merged: - number: "034" title: Computational modeling @@ -239,6 +287,9 @@ - derivative - metadata blocking: None + google_doc_created: 2021-02 + pull_request_created: 2021-08 + pull_request_merged: - number: "035" title: Modular extensions for individual participant data mega-analyses with non-compliant derivatives @@ -253,6 +304,9 @@ content: - derivative blocking: None + google_doc_created: 2021-12 + pull_request_created: + pull_request_merged: - number: "036" title: Phenotypic Data Guidelines @@ -267,6 +321,9 @@ content: - raw blocking: More reviews and discussions are necessary. + google_doc_created: 2021-10 + pull_request_created: + pull_request_merged: - number: "037" title: Non-Invasive Brain Stimulation (NIBS) @@ -279,6 +336,9 @@ content: - raw blocking: None + google_doc_created: 2022-09 + pull_request_created: + pull_request_merged: - number: "038" title: Atlases @@ -291,6 +351,9 @@ content: - raw blocking: None + google_doc_created: 2022-09 + pull_request_created: + pull_request_merged: - number: "039" title: Dimensionality reduction-based networks @@ -304,6 +367,9 @@ content: - raw blocking: None + google_doc_created: 2021-10 + pull_request_created: + pull_request_merged: - number: "040" title: Functional Ultrasound (fUS) @@ -315,6 +381,9 @@ content: - raw blocking: None + google_doc_created: 2023-03 + pull_request_created: + pull_request_merged: - number: "041" title: Statistical Model Derivatives @@ -324,3 +393,6 @@ content: - derivative blocking: None + google_doc_created: 2022-08 + pull_request_created: + pull_request_merged: diff --git a/_data/beps_completed.yml b/_data/beps_completed.yml index fc76ce96..21047def 100644 --- a/_data/beps_completed.yml +++ b/_data/beps_completed.yml @@ -13,13 +13,33 @@ - name: Kirstie Whitaker content: - raw + google_doc_created: 2017-02 + pull_request_created: 2020-06 + pull_request_merged: 2021-02 + +- number: "002" + title: BIDS Models Specification + leads: + - name: Tal Yarkoni + - name: Chris Markiewcz + content: + - file format + outcome: "Became an [independent specification with validator](https://bids-standard.github.io/stats-models/) to execute statistical models, with minimal configuration and intervention required from the user." + google_doc_created: 2016-09 + pull_request_created: 2018-10 + pull_request_merged: 2023-08 - number: "003" title: Common Derivatives leads: - name: Chris Markiewicz + former_leads: + - name: Chris Gorgolewski content: - derivative + google_doc_created: 2016-02 + pull_request_created: 2018-12 + pull_request_merged: 2020-06 - number: "005" title: Arterial Spin Labeling (ASL) @@ -30,6 +50,9 @@ - name: Marco Castellaro content: - raw + google_doc_created: 2017-05 + pull_request_created: 2020-10 + pull_request_merged: 2021-02 - number: "006" title: Electroencephalography (EEG) @@ -39,6 +62,9 @@ - name: Robert Oostenveld content: - raw + google_doc_created: 2017-06 + pull_request_created: 2018-12 + pull_request_merged: 2019-03 - number: "007" title: Hierarchical Event Descriptor (HED) Tags @@ -46,6 +72,9 @@ - name: Chris Gorgolewski content: - raw + google_doc_created: 2017-07 + pull_request_created: # only ever existed as a google doc + pull_request_merged: 2018-04 - number: "008" title: Magnetoencephalography (MEG) @@ -53,6 +82,9 @@ - name: Guiomar Niso content: - raw + google_doc_created: 2016-03 + pull_request_created: # only ever existed as a google doc + pull_request_merged: 2018-04 - number: "009" title: Positron Emission Tomography (PET) @@ -60,6 +92,9 @@ - name: Melanie Ganz content: - raw + google_doc_created: 2016-03 + pull_request_created: 2020-10 + pull_request_merged: 2021-04 - number: "010" title: intracranial Electroencephalography (iEEG) @@ -68,6 +103,9 @@ - name: Dora Hermes content: - raw + google_doc_created: 2017-04 + pull_request_created: 2018-12 + pull_request_merged: 2019-03 - number: "018" title: Genetic information @@ -77,6 +115,9 @@ - name: Thomas Nichols content: - raw + google_doc_created: 2017-09 + pull_request_created: 2019-07 + pull_request_merged: 2020-04 - number: "029" title: Virtual and physical motion data @@ -85,6 +126,10 @@ - name: Julius Welzel content: - raw + blocking: None + google_doc_created: 2019-11 + pull_request_created: 2022-01 + pull_request_merged: 2023-03 - number: "030" title: Near Infrared Spectroscopy (NIRS) @@ -93,6 +138,9 @@ - name: Luca Pollonini content: - raw + google_doc_created: 2020-04 + pull_request_created: 2021-05 + pull_request_merged: 2022-10 - number: "031" title: Microscopy @@ -101,4 +149,6 @@ - name: Julien Cohen-Adad content: - raw - + google_doc_created: 2020-06 + pull_request_created: 2021-09 + pull_request_merged: 2022-02 diff --git a/_data/beps_other.yml b/_data/beps_other.yml index ad3171cb..1f56953c 100644 --- a/_data/beps_other.yml +++ b/_data/beps_other.yml @@ -13,15 +13,6 @@ # content: # - raw -- number: "002" - title: BIDS Models Specification - leads: - - name: Tal Yarkoni - - name: Chris Markiewcz - content: - - file format - outcome: "Became an [independent specification with validator](https://bids-standard.github.io/stats-models/) to execute statistical models, with minimal configuration and intervention required from the user." - - number: "013" title: Resting state fMRI derivatives leads: @@ -47,6 +38,9 @@ content: - metadata outcome: Closed in favor of possible direct support of DICOM within BIDS (see this [issue](https://github.com/bids-standard/bids-specification/issues/1552) and this [pull request](https://github.com/bids-standard/bids-specification/pull/1551)). + google_doc_created: 2017-10 + pull_request_created: + pull_request_merged: - number: "025" title: Medical Imaging Data structure (MIDS) diff --git a/_layouts/default.html b/_layouts/default.html index d08fd2aa..5181ae19 100644 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -85,13 +85,13 @@ - {% if site.show_footer %} - {% endif %} + + + + {% include tracking.html %} diff --git a/_pages/get_involved.md b/_pages/get_involved.md index 7237c689..af80118e 100644 --- a/_pages/get_involved.md +++ b/_pages/get_involved.md @@ -64,6 +64,54 @@ Below is a table of BEPs that have been merged. The references of the final publication for those BEPS can be found in the BIDS [specification](https://bids-specification.readthedocs.io/en/latest/01-introduction.html#datatype-specific-publications). + + +```mermaid +gantt + title completed BEP timeline + dateFormat YYYY-MM + tickInterval 6month + axisFormat %b-%Y + section BEP001 + Google doc :2017-02, 2020-06 + Pull request :2020-06, 2021-02 + section BEP002 + Google doc :2016-09, 2018-10 + Pull request :2018-10, 2023-08 + section BEP003 + Google doc :2018-10, 2018-12 + Pull request :2018-12, 2020-06 + section BEP005 + Google doc :2017-05, 2020-10 + Pull request :2020-10, 2021-02 + section BEP006 + Google doc :2017-06, 2018-12 + Pull request :2018-12, 2019-03 + section BEP007 + Google doc :2017-07, 2018-04 + section BEP008 + Google doc :2016-03, 2018-04 + section BEP009 + Google doc :2016-03, 2020-10 + Pull request :2020-10, 2021-04 + section BEP010 + Google doc :2017-04, 2018-12 + Pull request :2018-12, 2019-03 + section BEP018 + Google doc :2017-09, 2019-07 + Pull request :2019-07, 2020-04 + section BEP029 + Google doc :2019-11, 2022-01 + Pull request :2022-01, 2023-03 + section BEP030 + Google doc :2020-04, 2021-05 + Pull request :2021-05, 2022-10 + section BEP031 + Google doc :2020-06, 2021-09 + Pull request :2021-09, 2022-02 +``` + + {% include beps_completed_table.html beps=site.data.beps_completed %}
diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8389af11 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 99 +target-version = ['py39'] +exclude = ''' +( + /( + \.git + | env + )/ +) +''' diff --git a/tools/bep_gant_chart.py b/tools/bep_gant_chart.py new file mode 100644 index 00000000..f0530a5b --- /dev/null +++ b/tools/bep_gant_chart.py @@ -0,0 +1,198 @@ +"""Creates a gant chart for the completed BEPs. + +Also include a timeline of the main BIDS events. +""" +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go +import ruamel.yaml as yaml +from pyzotero import zotero + +INCLUDE_PATCHES = False +START_DATE = "2014-10" +DATE_FORMAT = "%Y-%m" +Y_AXIS_VALUE = "" + + +def root_dir() -> Path: + return Path(__file__).parent.parent + + +def data_dir() -> Path: + return root_dir() / "_data" + + +def create_bep_timeline() -> type[go.Figure]: + completd_beps = data_dir() / "beps_completed.yml" + + with open(completd_beps, "r") as f: + data = yaml.safe_load(f) + + df = [] + + # iterate over data in reverse order + for bep in reversed(data): + StartDoc = bep.get("google_doc_created") + StartPR = bep.get("pull_request_created") + Finish = bep["pull_request_merged"] + + BEP = f"{bep['number']} - {bep['title']} " + + if StartDoc and StartPR: + df.append( + dict( + BEP=BEP, + Start=StartDoc, + Finish=StartPR, + Resource="Google Doc", + ) + ) + if StartPR and Finish: + df.append( + dict( + BEP=BEP, + Start=StartPR, + Finish=Finish, + Resource="Pull Request", + ) + ) + if not StartPR: + df.append( + dict( + BEP=BEP, + Start=StartDoc, + Finish=Finish, + Resource="Google Doc", + ) + ) + + df = pd.DataFrame(df) + + fig = px.timeline( + df, + x_start="Start", + x_end="Finish", + y="BEP", + color="Resource", + title="BIDS timeline", + ) + + return fig + + +def plot_time_line(fig: go.Figure) -> type[go.Figure]: + dates = [datetime.strptime(START_DATE, DATE_FORMAT), datetime.now()] + y = [Y_AXIS_VALUE, Y_AXIS_VALUE] + + fig.add_trace( + go.Scatter( + x=dates, + y=y, + mode="lines", + showlegend=False, + line=dict(color="black", width=2), + hoverinfo="skip", + ) + ) + + return fig + + +def plot_releases( + fig: go.figure, timeline: pd.DataFrame, include_patches: bool = INCLUDE_PATCHES +) -> type[go.Figure]: + mask = timeline["type"] == "release" + if not include_patches: + mask = mask & (timeline["name"].str.endswith("0")) + bids_releases = timeline["name"][mask].tolist() + bids_releases_dates = timeline["date"][mask].tolist() + + dates = [datetime.strptime(d, DATE_FORMAT) for d in bids_releases_dates] + y = [Y_AXIS_VALUE for _ in dates] + + fig.add_trace( + go.Scatter( + name="Releases", + x=dates, + y=y, + mode="markers+text", + marker=dict(color="black", size=10), + text=bids_releases, + hoverinfo="x+text", + textposition="bottom center", + ), + ) + + return fig + + +def add_publications_to_timeline(fig: go.Figure) -> type[go.Figure]: + """Get publication info from Zotero and add to timeline.""" + zot = zotero.Zotero(library_id="5111637", library_type="group") + items = zot.everything(zot.top()) + + titles = [] + dois = [] + dates = [] + for item in items: + titles.append(item["data"].get("title")) + dois.append(item["data"]["DOI"]) + dates.append(item["data"]["date"]) + + fig.add_trace( + go.Scatter( + name="Publications", + x=dates, + y=[Y_AXIS_VALUE for _ in dates], + mode="markers", + marker=dict(color="blue", size=10), + hoverinfo="x+text", + text=titles, + ), + ) + + return fig + + +def main(): + fig = create_bep_timeline() + + fig = plot_time_line(fig) + + timeline = pd.read_csv(root_dir() / "tools" / "timeline.csv") + fig = plot_releases(fig, timeline, include_patches=INCLUDE_PATCHES) + + # Add events timeline + sub_df = timeline[timeline["type"] == "event"] + + fig.add_trace( + go.Scatter( + name="Event", + x=sub_df["date"], + y=[Y_AXIS_VALUE for _ in sub_df["date"]], + mode="markers", + marker=dict(color="green", size=10), + hoverinfo="x+text", + text=sub_df["name"], + ), + ) + + fig = add_publications_to_timeline(fig) + + fig.update_layout(legend_font_size=15) + fig.update_layout(title=dict(font=dict(size=30))) + fig.update_layout(yaxis=dict(tickfont=dict(size=15))) + + fig.show() + + # save as html + fig.write_html(root_dir() / "_pages" / "bids_timeline.html") + + +if __name__ == "__main__": + main() diff --git a/tools/convert_to_md.py b/tools/convert_to_md.py index 256b7153..5fdc53f1 100644 --- a/tools/convert_to_md.py +++ b/tools/convert_to_md.py @@ -39,10 +39,6 @@ def rename_files(input_folder): print("Renaming files".upper()) for file in input_folder.glob("*.md"): - - if str(file).endswith("README.md"): - continue - print(f" Processing {file}") with open(file, "r") as f: @@ -59,12 +55,7 @@ def rename_files(input_folder): month, day = line.split(" ")[2:] - day = ( - day.replace("th", "") - .replace("rd", "") - .replace("nd", "") - .replace("st", "") - ) + day = day.replace("th", "").replace("rd", "").replace("nd", "").replace("st", "") if month in MONTH_MAPPING: month = MONTH_MAPPING[month] @@ -86,7 +77,6 @@ def rename_files(input_folder): def sanitize_md(input_folder): """Clean up the markdown files.""" for file in input_folder.glob("*.md"): - if str(file).endswith("README.md"): continue diff --git a/tools/insert_mermaid_bep_gantt.py b/tools/insert_mermaid_bep_gantt.py new file mode 100644 index 00000000..b6aa629b --- /dev/null +++ b/tools/insert_mermaid_bep_gantt.py @@ -0,0 +1,129 @@ +"""Creates a gant chart for the completed BEPs""" +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import ruamel.yaml as yaml +from rich import print + + +INCLUDE_PATCHES = False +START_DATE = "2014-10" +DATE_FORMAT = "%Y-%m" +Y_AXIS_VALUE = "" + + +def root_dir() -> Path: + return Path(__file__).parent.parent + + +def data_dir() -> Path: + return root_dir() / "_data" + + +def target_file() -> Path: + return root_dir() / "_pages" / "get_involved.md" + + +def get_bep_timeline() -> pd.DataFrame: + completd_beps = data_dir() / "beps_completed.yml" + + with open(completd_beps, "r") as f: + data = yaml.safe_load(f) + + df = [] + + # iterate over data in reverse order + for bep in data: + StartDoc = bep.get("google_doc_created") + StartPR = bep.get("pull_request_created") + Finish = bep["pull_request_merged"] + + BEP = f"{bep['number']} - {bep['title']} " + + if StartDoc and StartPR: + df.append( + dict( + BEP=BEP, + Start=StartDoc, + Finish=StartPR, + Resource="Google Doc", + ) + ) + if StartPR and Finish: + df.append( + dict( + BEP=BEP, + Start=StartPR, + Finish=Finish, + Resource="Pull Request", + ) + ) + if not StartPR: + df.append( + dict( + BEP=BEP, + Start=StartDoc, + Finish=Finish, + Resource="Google Doc", + ) + ) + + df = pd.DataFrame(df) + + return df + + +def main(): + df = get_bep_timeline() + + text = """```mermaid +gantt + title completed BEP timeline + dateFormat YYYY-MM + tickInterval 6month + axisFormat %b-%Y +""" + + for bep in df.BEP.unique(): + bep_df = df[df.BEP == bep] + google_start = bep_df[bep_df.Resource == "Google Doc"].Start.values[0] + google_end = bep_df[bep_df.Resource == "Google Doc"].Finish.values[0] + + text += f""" section BEP{bep[:3]} + Google doc :{google_start}, {google_end} +""" + + pr_df = bep_df[bep_df.Resource == "Pull Request"] + if len(pr_df) > 0: + pr_start = pr_df.Start.values[0] + pr_end = pr_df.Finish.values[0] + + text += f""" Pull request :{pr_start}, {pr_end} +""" + + text += "```\n" + + print(text) + + # insert into target file + with open(target_file(), "r") as f: + lines = f.readlines() + + inser_mermaid_here = False + with open(target_file(), "w") as f: + for line in lines: + if line.startswith(""): + inser_mermaid_here = True + f.write("\n") + f.write(text) + if line.startswith(""): + inser_mermaid_here = False + + if not inser_mermaid_here: + f.write(line) + + +if __name__ == "__main__": + main() diff --git a/tools/requirements.txt b/tools/requirements.txt index c1f5f713..e4ab27ea 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1 +1,5 @@ beautifulsoup4 +plotly +ruamel.yaml +pandas +pyzotero diff --git a/tools/timeline.csv b/tools/timeline.csv new file mode 100644 index 00000000..72737c2d --- /dev/null +++ b/tools/timeline.csv @@ -0,0 +1,25 @@ +name,date,type,url +INCF / NIDM / NIDASH,2015-01,event,"n/a" +First draft,2015-09,event,"n/a" +BIDS Paper,2016-06,publication,https://doi.org/10.1038/sdata.2016.44 +BIDS App Paper,2017-03,publication,https://doi.org/10.1371/journal.pcbi.1005209 +OpenNeuro "launch",2018-02,event,"n/a" +First commit on the BIDS specification github repository,2018-09,event,"n/a" +First BIDS streering group election,2019-09,event,"n/a" +1.0.0,2016-06,release,"n/a" +1.0.1,2017-03,release,"n/a" +1.0.2,2017-07,release,"n/a" +1.1.0,2018-04,release,"n/a" +1.1.1,2018-06,release,"n/a" +1.1.2,2019-01,release,https://bids-specification.readthedocs.io/en/v1.1.2 +1.2.0,2019-05,release,https://bids-specification.readthedocs.io/en/v1.2.0 +1.2.1,2019-08,release,https://bids-specification.readthedocs.io/en/v1.2.1 +1.2.2,2020-02,release,https://bids-specification.readthedocs.io/en/v1.2.2 +1.3.0,2020-04,release,https://bids-specification.readthedocs.io/en/v1.3.0 +1.4.0,2020-06,release,https://bids-specification.readthedocs.io/en/v1.4.0 +1.4.1,2020-10,release,https://bids-specification.readthedocs.io/en/v1.4.1 +1.5.0,2021-02,release,https://bids-specification.readthedocs.io/en/v1.5.0 +1.6.0,2021-04,release,https://bids-specification.readthedocs.io/en/v1.6.0 +1.7.0,2022-02,release,https://bids-specification.readthedocs.io/en/v1.7.0 +1.8.0,2022-10,release,https://bids-specification.readthedocs.io/en/v1.8.0 +1.9.0,2023-11,release,https://bids-specification.readthedocs.io/en/v1.9.0