Repackaging as a Python library

ibleaman · Feb 3, 2022 · 33b9db0 · 33b9db0
1 parent bbd86e8
commit 33b9db0
Show file tree

Hide file tree

Showing 13 changed files with 230 additions and 51 deletions.
diff --git a/AUTHORS b/AUTHORS
@@ -0,0 +1,4 @@
+Isaac L. Bleaman <[email protected]>
+(additional authors to be added here)
+
+ILB wrote many of these functions for personal use, in a few different programming languages, over the years. They were put together in a single .py file in 2021 for REYD, a Yiddish text-to-speech project, and then packaged as a standalone Python library in 2022.
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,7 @@
+Copyright 2021-2022 Isaac L. Bleaman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,63 @@
+# yiddish
+
+A Python library for processing Yiddish text
+
+Isaac L. Bleaman (<[email protected]>) and contributors (see commit history)
+
+## What is this for?
+
+This library includes functions to carry out common tasks when dealing with Yiddish text. For example, you might wish to replace precombined Unicode characters (such as אַ, U+FB2E) with their decomposed versions (אַ, which is U+05D0 followed by U+05B7). Or you might wish to transliterate YIVO Yiddish text
+(`איבער` to `iber`) or render it in the orthography used more commonly in the 
+Hasidic community (`שנײיִק` to `שנייאיג`).
+
+See the source file, `yiddish.py`, for the full list of supported functions.
+
+## How to install
+
+    pip install yiddish
+
+## How to cite
+
+If you'd like to cite `yiddish` in a publication, you can include a link to the source:
+    https://github.com/ibleaman/yiddish
+
+## Example
+
+```python
+import yiddish
+
+output = ''
+
+string = 'אונדזער גאַנצע משפּחה װױנט אין די פֿאַראײניקטע שטאַטן.'
+
+output += yiddish.replace_with_precombined(string) + '\n'
+output += yiddish.respell_loshn_koydesh(string) + '\n'
+output += yiddish.strip_diacritics(string) +  '\n'
+output += yiddish.transliterate(string) +  '\n'
+output += yiddish.transliterate(string, loshn_koydesh=True) +  '\n'
+output += yiddish.hasidify(string)
+
+output += '\n\n'
+
+string_two = 'shloymele hot khasene gehat mit rokhls tokhter leye.'
+
+output += yiddish.detransliterate(string_two) + '\n'
+output += yiddish.detransliterate(string_two, loshn_koydesh=True)
+
+print(output)
+
+```
+
+Output:
+
+```
+אונדזער גאַנצע משפּחה װױנט אין די פֿאַראײניקטע שטאַטן.
+אונדזער גאַנצע מישפּאָכע װױנט אין די פֿאַראײניקטע שטאַטן.
+אונדזער גאנצע משפחה וווינט אין די פאראייניקטע שטאטן.
+undzer gantse mshpkhh voynt in di fareynikte shtatn.
+undzer gantse mishpokhe voynt in di fareynikte shtatn.
+אונזער גאנצע משפחה וואוינט אין די פאראייניקטע שטאטן.
+
+שלױמעלע האָט כאַסענע געהאַט מיט ראָכלס טאָכטער לײע.
+שלמהלע האָט חתונה געהאַט מיט רחלס טאָכטער לאה.
+```
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"
diff --git a/respell_yiddish.py b/respell_yiddish.py
diff --git a/setup.py b/setup.py
@@ -0,0 +1,21 @@
+from setuptools import setup
+
+with open('README.md', 'r', encoding='utf-8') as fh:
+    long_description = fh.read()
+
+setup(
+    name='yiddish',
+    version='0.0.1',
+    author='Isaac L. Bleaman',
+    author_email='[email protected]',
+    description='A Python library for processing Yiddish text',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://github.com/ibleaman/yiddish',
+    install_requires=[
+        "pandas>=1.1.0",
+    ],
+    license='MIT',
+    packages=['yiddish'],
+    test_suite='tests',
+)
diff --git a/yiddish.egg-info/PKG-INFO b/yiddish.egg-info/PKG-INFO
@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: yiddish
+Version: 0.0.5
+Summary: A Python library for processing Yiddish text
+Home-page: https://github.com/ibleaman/yiddish
+Author: Isaac L. Bleaman
+Author-email: [email protected]
+License: MIT
+Platform: UNKNOWN
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: AUTHORS
+
+# yiddish
+
+A Python library for processing Yiddish text
+
+Isaac L. Bleaman (<[email protected]>) and contributors (see commit history)
+
+## What is this for?
+
+This library includes functions to accomplish common tasks when dealing with Yiddish text. For example, you might wish to replace precombined Unicode characters (such as אַ, U+FB2E) with their decomposed versions (אַ, which is U+05D0 followed by U+05B7). Or you might wish to transliterate YIVO Yiddish text
+(`איבער` to `iber`) or render it in the orthography used more commonly in the 
+Hasidic community (`שנײיִק` -> `שנייאיג`).
+
+See the source file, `yiddish.py`, for the full list of supported functions.
+
+## How to install
+
+    pip install yiddish
+
+## How to cite
+
+If you'd like to cite `yiddish` in a publication, you can include a link to the source:
+    https://github.com/ibleaman/yiddish
+
+## Example
+
+```python
+import yiddish
+
+output = ''
+
+string = 'די גאַנצע משפּחה װױנט אין די פֿאַראײניקטע שטאַטן.'
+
+output += yiddish.replace_with_precombined(string) + '\n'
+output += yiddish.strip_diacritics(string) +  '\n'
+output += yiddish.transliterate(string) +  '\n'
+output += yiddish.transliterate(string, loshn_koydesh=True) +  '\n'
+output += yiddish.hasidify(string)
+
+
+string_two = 'moyshe-leyb hot khasene gehat mit esters tokhter.'
+
+output += yiddish.detransliterate(string_two) + '\n'
+output += yiddish.detransliterate(string_two, loshn_koydesh=True)
+
+print(output)
+
+```
+
+Output:
+
+```
+
+```
+
diff --git a/yiddish.egg-info/SOURCES.txt b/yiddish.egg-info/SOURCES.txt
@@ -0,0 +1,12 @@
+AUTHORS
+LICENSE
+README.md
+pyproject.toml
+setup.py
+yiddish/__init__.py
+yiddish/yiddish.py
+yiddish.egg-info/PKG-INFO
+yiddish.egg-info/SOURCES.txt
+yiddish.egg-info/dependency_links.txt
+yiddish.egg-info/requires.txt
+yiddish.egg-info/top_level.txt
diff --git a/yiddish.egg-info/dependency_links.txt b/yiddish.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/yiddish.egg-info/requires.txt b/yiddish.egg-info/requires.txt
@@ -0,0 +1 @@
+pandas>=1.1.0
diff --git a/yiddish.egg-info/top_level.txt b/yiddish.egg-info/top_level.txt
@@ -0,0 +1 @@
+yiddish
diff --git a/yiddish/__init__.py b/yiddish/__init__.py
@@ -0,0 +1,11 @@
+from .yiddish import (
+  replace_with_precombined,
+  replace_with_decomposed,
+  replace_punctuation,
+  strip_diacritics,
+  transliterate,
+  detransliterate,
+  romanise_german,
+  respell_loshn_koydesh,
+  hasidify,
+)
diff --git a/yiddish_text_tools.py → yiddish/yiddish.py b/yiddish_text_tools.py → yiddish/yiddish.py
@@ -1,9 +1,10 @@
-# Functions for processing Yiddish text, written in the YIVO orthography
+# yiddish
+# A Python library for processing Yiddish text
 # Author: Isaac L. Bleaman ([email protected])
 
 import re
 from urllib.request import urlopen
-import pandas as pd
+import csv
 
 ##########
 # encoding
@@ -83,7 +84,10 @@ def strip_diacritics(string): # and replace with decomposed
     for entry in entries.split(','):
         if entry not in reverse_lk:
             reverse_lk[entry] = key
-
+
+germanic_semitic_homographs = ["אין", "צום", "בין", "ברי", "מיד", "קין", "שער", "מעגן", "צו", "מאַנס", "טוען", "מערער"]
+
+less_common_lk_pronunciations = ["אַדױשעם", "כאַנוקע", "גדױלע", "כאַװײרע", "מיכיע", "כאָװער", "אָרעװ", "מאָסער", "כיִעס", "זקאָנים", "נעװאָלע", "מאַשלעם", "כפֿאָצים", "כאַכאָמע", "טאַנאָיִם", "יאָסעף", "יאָסעפֿס", "יאָסעפֿן"]
 
 translit_table = [ # all are precombined
     ('א', ''),
@@ -141,8 +145,11 @@ def transliterate(string, loshn_koydesh=False):
         tokens = re.findall(r"[אאַאָבבֿגדהוװוּױזחטייִײײַככּךלמםנןסעפּפֿףצץקרששׂתּת\-־]+|[^אאַאָבבֿגדהוװוּױזחטייִײײַככּךלמםנןסעפּפֿףצץקרששׂתּת\-־]", romanized)
         new_tokens = []
         for token in tokens:
-            if token in lk:
-                new_tokens.append(lk[token][0].replace('־', '-'))
+            if token in lk and token not in germanic_semitic_homographs:
+                if lk[token][0] in less_common_lk_pronunciations and len(lk[token]) > 1:
+                    new_tokens.append(lk[token][1].replace('־', '-'))
+                else:
+                    new_tokens.append(lk[token][0].replace('־', '-'))
             else:
                 new_tokens.append(token)
 
@@ -411,13 +418,15 @@ def romanise_german(text):
 # for TTS: respell orthographic words phonetically
 ##################################################
 
+# Note: input text WILL become precombined
 def respell_loshn_koydesh(text):
+    text = replace_with_precombined(text)
     # loop over keys, in reverse order from longest keys to shortest
     for key in sorted(list(lk.keys()), key=len, reverse=True):
         # skip Germanic homographs, which are usually phonetic
-        if key not in ["אין", "צום", "בין", "ברי", "מיד", "קין", "שער", "מעגן", "צו", "מאַנס", "טוען", "מערער"]:
-            # skip less common LK variants, in favor of more common ones
-            if lk[key][0] in ["אַדױשעם", "כאַנוקע", "גדױלע", "כאַװײרע", "מיכיע", "כאָװער", "אָרעװ", "מאָסער", "כיִעס", "זקאָנים", "נעװאָלע", "מאַשלעם", "כפֿאָצים", "כאַכאָמע", "טאַנאָיִם", "יאָסעף", "יאָסעפֿס", "יאָסעפֿן"] and len(lk[key]) > 1:
+        if key not in germanic_semitic_homographs:
+            # skip less common LK pronunciations, in favor of more common ones
+            if lk[key][0] in less_common_lk_pronunciations and len(lk[key]) > 1:
                 replacement = lk[key][1]
             else:
                 replacement = lk[key][0]
@@ -454,37 +463,34 @@ def respell_loshn_koydesh(text):
 # note: all replacements are based on
 # looking for precombined characters
 #######################################
-hasidifier_lexicon = pd.ExcelFile('https://docs.google.com/spreadsheets/d/1x_KLOaUfnCBVVWEb523QIJZIvk65Pp0ly9MPMVLUVOI/export?format=xlsx')
+hasidify_lexicon = 'https://raw.githubusercontent.com/ibleaman/hasidify_lexicon/master/'
 
-whole_word_variants = pd.read_excel(hasidifier_lexicon, 'whole_word_variants')
-whole_word_variants = dict(zip([replace_with_precombined(word) for word in whole_word_variants['Find']], [replace_with_precombined(word) for word in whole_word_variants['Replace']]))
+whole_word_variants = list(csv.reader(urlopen(hasidify_lexicon + 'whole_word_variants.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+whole_word_variants = dict(zip([replace_with_precombined(row[0]) for row in whole_word_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in whole_word_variants if row[1] != 'Replace']))
 
-prefix_variants = pd.read_excel(hasidifier_lexicon, 'prefix_variants')
-prefix_variants = dict(zip([replace_with_precombined(word) for word in prefix_variants['Find']], [replace_with_precombined(word) for word in prefix_variants['Replace']]))
+prefix_variants = list(csv.reader(urlopen(hasidify_lexicon + 'prefix_variants.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+prefix_variants = dict(zip([replace_with_precombined(row[0]) for row in prefix_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in prefix_variants if row[1] != 'Replace']))
 
-suffix_variants = pd.read_excel(hasidifier_lexicon, 'suffix_variants')
-suffix_variants = dict(zip([replace_with_precombined(word) for word in suffix_variants['Find']], [replace_with_precombined(word) for word in suffix_variants['Replace']]))
+suffix_variants = list(csv.reader(urlopen(hasidify_lexicon + 'suffix_variants.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+suffix_variants = dict(zip([replace_with_precombined(row[0]) for row in suffix_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in suffix_variants if row[1] != 'Replace']))
 
-anywhere_variants = pd.read_excel(hasidifier_lexicon, 'anywhere_variants')
-anywhere_variants = dict(zip([replace_with_precombined(word) for word in anywhere_variants['Find']], [replace_with_precombined(word) for word in anywhere_variants['Replace']]))
+anywhere_variants = list(csv.reader(urlopen(hasidify_lexicon + 'anywhere_variants.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+anywhere_variants = dict(zip([replace_with_precombined(row[0]) for row in anywhere_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in anywhere_variants if row[1] != 'Replace']))
 
-lkizmen = pd.read_excel(hasidifier_lexicon, 'lkizmen')
-lkizmen = lkizmen['Words'].tolist()
-lkizmen = [replace_with_precombined(word) for word in lkizmen]
+lkizmen = list(csv.reader(urlopen(hasidify_lexicon + 'lkizmen.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+lkizmen = [replace_with_precombined(row[0]) for row in lkizmen if row[0] != 'Words']
 
-word_group_variants = pd.read_excel(hasidifier_lexicon, 'word_group_variants')
-word_group_variants = dict(zip([replace_with_precombined(word) for word in word_group_variants['Find']], [replace_with_precombined(word) for word in word_group_variants['Replace']]))
+word_group_variants = list(csv.reader(urlopen(hasidify_lexicon + 'word_group_variants.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+word_group_variants = dict(zip([replace_with_precombined(row[0]) for row in word_group_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in word_group_variants if row[1] != 'Replace']))
 
-ik_exceptions = pd.read_excel(hasidifier_lexicon, 'ik_exceptions')
-ik_exceptions = ik_exceptions['Words'].tolist()
-ik_exceptions = [replace_with_precombined(word) for word in ik_exceptions]
+ik_exceptions = list(csv.reader(urlopen(hasidify_lexicon + 'ik_exceptions.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+ik_exceptions = [replace_with_precombined(row[0]) for row in ik_exceptions if row[0] != 'Words']
 
-lekh_exceptions = pd.read_excel(hasidifier_lexicon, 'lekh_exceptions')
-lekh_exceptions = lekh_exceptions['Words'].tolist()
-lekh_exceptions = [replace_with_precombined(word) for word in lekh_exceptions]
+lekh_exceptions = list(csv.reader(urlopen(hasidify_lexicon + 'lekh_exceptions.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+lekh_exceptions = [replace_with_precombined(row[0]) for row in lekh_exceptions if row[0] != 'Words']
 
-last_minute_fixes = pd.read_excel(hasidifier_lexicon, 'last_minute_fixes')
-last_minute_fixes = dict(zip(last_minute_fixes['Find'], last_minute_fixes['Replace']))
+last_minute_fixes = list(csv.reader(urlopen(hasidify_lexicon + 'last_minute_fixes.csv').read().decode('utf-8').replace('\r', '').splitlines()))
+last_minute_fixes = dict(zip([replace_with_precombined(row[0]) for row in last_minute_fixes if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in last_minute_fixes if row[1] != 'Replace']))
 
 reformatting = [
     ('וּװוּ', 'ואוואו'),