Separate 20_markdown.obsidian.personal.notation.ipynb into `51_mark…

…down.obsidian.personal.notation.parse` and `52_markdown.obsidian.personal.notation.in_standard_information_note`
hyunjongkimmath · Sep 6, 2024 · 1b50cb0 · 1b50cb0
1 parent 4343dca
commit 1b50cb0
Show file tree

Hide file tree

Showing 14 changed files with 2,666 additions and 1,508 deletions.
diff --git a/README.md b/README.md
@@ -116,7 +116,8 @@ from pathlib import Path
 import shutil
 import tempfile
 
-from trouver.helper.tests import _test_directory, text_from_file
+from trouver.helper.tests import _test_directory
+from trouver.helper.files_and_folders import  text_from_file
 from trouver.latex.convert import (
     divide_preamble, divide_latex_text, custom_commands,
     setup_reference_from_latex_parts
@@ -238,8 +239,6 @@ else:
     information_note_type_model = from_pretrained_fastai(repo_id)
 ```
 
-    Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
-
 ``` python
 sample_prediction_1 = information_note_type_model.predict(r'Let $L/K$ be an field extension. An element $\alpha \in L$ is said to be algebraic over $K$ if there exists some polynomial $f(x) \in K[x]$ such that $f(\alpha) = 0$.')
 print(sample_prediction_1) 
@@ -726,7 +725,8 @@ For the example below, there is at least one information note with
 notations already marked with double asterisks `**`.
 
 ``` python
-from trouver.markdown.obsidian.personal.notation import make_notation_notes_from_double_asts, notation_notes_linked_in_see_also_section
+from trouver.markdown.obsidian.personal.notation import make_notation_notes_from_double_asts
+from trouver.markdown.obsidian.personal.notation.in_standard_information_note import notation_notes_linked_in_see_also_section
 from trouver.markdown.obsidian.personal.machine_learning.notation_summarization import append_summary_to_notation_note
 ```
 

diff --git a/nbs/20_markdown.obsidian.personal.notation.ipynb b/nbs/20_markdown.obsidian.personal.notation.ipynb
diff --git a/nbs/25_markdown.obsidian.personal.machine_learning.notation_summarization.ipynb b/nbs/25_markdown.obsidian.personal.machine_learning.notation_summarization.ipynb
@@ -48,7 +48,8 @@
     "from trouver.markdown.obsidian.links import ObsidianLink\n",
     "from trouver.markdown.obsidian.personal.machine_learning.database_update import append_to_database\n",
     "from trouver.markdown.obsidian.personal.note_processing import process_standard_information_note\n",
-    "from trouver.markdown.obsidian.personal.notation import parse_notation_note, main_of_notation, _notation_string_no_metadata, _raw_notation\n",
+    "from trouver.markdown.obsidian.personal.notation import _notation_string_no_metadata, _raw_notation\n",
+    "from trouver.markdown.obsidian.personal.notation.parse import main_of_notation, parse_notation_note\n",
     "from trouver.markdown.obsidian.personal.note_type import note_is_of_type\n",
     "from trouver.markdown.obsidian.vault import VaultNote"
    ]
@@ -179,6 +180,7 @@
     "        notation_note: VaultNote,\n",
     "        vault: PathLike,\n",
     "        # gather_data_for_blank_content_notation_notes: bool = False,\n",
+    "        check_for_actual_summarization: bool = True # If `True`, then return `None` if `notation_note` is determined to not have a summary or if the summary is autogenerated.\n",
     "        ) -> Union[dict[str, str], None]: # The keys to the dict are \"Notation note name\", \"Notation\", \"Latex in oiriginal\", \"Summary\", and \"Main note name\". However, `None` is returned instead of a `dict` if the notation note is determined to have not been summarized, if the main note of the notation note does not exist, or the notation note is marked with the `_auto/notation_summary` tag.\n",
     "    \"\"\"Obtain notation summzarization data from the notation note.\n",
     "\n",
@@ -219,7 +221,8 @@
     "\n",
     "    - Union[dict[str, str], None]\n",
     "\n",
-    "        - If the notation note is determined to have been summarized\n",
+    "        - If `check_for_actual_summarization` is `False` or\n",
+    "          the notation note is determined to have been summarized\n",
     "          (i.e. contains text beyond `<notation> denotes ` and does not\n",
     "          have the `#_meta/TODO` tag) then the output is a `dict` whose\n",
     "          key-value pairs are\n",
@@ -235,9 +238,9 @@
     "        - Otherwise, the output is `None.\n",
     "    \"\"\"\n",
     "    metadata, notation_str, main_of_notation, main_mf, _ = parse_notation_note(notation_note, vault)\n",
-    "    if not _notation_has_been_summarized(main_mf):\n",
+    "    if check_for_actual_summarization and not _notation_has_been_summarized(main_mf):\n",
     "        return None\n",
-    "    if _notation_note_has_auto_summary_tag(metadata):\n",
+    "    if check_for_actual_summarization and _notation_note_has_auto_summary_tag(metadata):\n",
     "        return None\n",
     "\n",
     "    latex_in_original = get_latex_in_original_from_parsed_notation_note_data(\n",
@@ -269,7 +272,9 @@
     "        ) -> bool:\n",
     "    text = str(main_mf).strip()\n",
     "    return len(text) > 0 and '#_meta/TODO' not in text\n",
-    "\n"
+    "\n",
+    "\n",
+    "# TODO: Test the case of `check_for_actual_summarization` is `False`"
    ]
   },
   {
@@ -1338,6 +1343,14 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\hyunj\\Documents\\Development\\Python\\trouver_py310_venv\\lib\\site-packages\\bs4\\__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
+      "  warnings.warn(\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -1349,14 +1362,6 @@
       "---\n",
       "$\\mathscr{O}_{\\text {Proj } S_{*}}(n)$ [[foag_15.2.1|denotes]] This is a mocked summary\n"
      ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "c:\\Users\\hyunj\\Documents\\Development\\Python\\trouver_py310_venv\\lib\\site-packages\\bs4\\__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
-      "  warnings.warn(\n"
-     ]
     }
    ],
    "source": [
@@ -2005,7 +2010,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Your max_length is set to 200, but you input_length is only 126. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=63)\n"
+      "Your max_length is set to 200, but your input_length is only 126. Since this is a summarization task, where outputs shorter than the input are typically wanted, you might consider decreasing max_length manually, e.g. summarizer('...', max_length=63)\n"
      ]
     },
     {
@@ -2017,7 +2022,7 @@
       "latex_in_original: [\"\\\\mathscr{O}_{\\\\text {Proj } S_{*}}(n)\"]\n",
       "tags: [_auto/notation_summary]\n",
       "---\n",
-      "$\\mathscr{O}_{\\text {Proj } S_{*}}(n)$ [[foag_15.2.1|denotes]] $\\mathscr{O}_{\\text {Proj} S_{*}}(n)$ where $\\bullet$ is a graded ring generated in degree $1$.\n"
+      "$\\mathscr{O}_{\\text {Proj } S_{*}}(n)$ [[foag_15.2.1|denotes]] $\\mathscr{O}_{\\text {Proj} S_{*}}(n)$ where $S_{*}$ is a graded ring generated in degree $1$.\n"
      ]
     }
    ],

diff --git a/nbs/34_markdown.obsidian.machine_learning.notation_linking.ipynb b/nbs/34_markdown.obsidian.machine_learning.notation_linking.ipynb
@@ -57,7 +57,8 @@
     "from trouver.markdown.markdown.file import MarkdownFile, MarkdownLineEnum\n",
     "from trouver.markdown.obsidian.links import MARKDOWNLINK_CAPTURE_PATTERN\n",
     "from trouver.markdown.obsidian.personal.information_notes import reference_of_information_note\n",
-    "from trouver.markdown.obsidian.personal.notation import parse_notation_note, notation_notes_linked_in_see_also_section\n",
+    "from trouver.markdown.obsidian.personal.notation.in_standard_information_note import notation_notes_linked_in_see_also_section\n",
+    "from trouver.markdown.obsidian.personal.notation.parse import parse_notation_note\n",
     "from trouver.markdown.obsidian.personal.note_processing import process_standard_information_note\n",
     "from trouver.markdown.obsidian.personal.note_type import note_is_of_type, type_of_note, PersonalNoteTypeEnum\n",
     "from trouver.markdown.obsidian.personal.notes import notes_linked_in_notes_linked_in_note\n",