From 33b9db0e60796f7cb371f95dc2c5011ac6bc4105 Mon Sep 17 00:00:00 2001 From: "Isaac L. Bleaman" Date: Wed, 2 Feb 2022 18:50:54 -0800 Subject: [PATCH] Repackaging as a Python library --- AUTHORS | 4 ++ LICENSE | 7 +++ README.md | 63 +++++++++++++++++++ pyproject.toml | 6 ++ respell_yiddish.py | 21 ------- setup.py | 21 +++++++ yiddish.egg-info/PKG-INFO | 67 +++++++++++++++++++++ yiddish.egg-info/SOURCES.txt | 12 ++++ yiddish.egg-info/dependency_links.txt | 1 + yiddish.egg-info/requires.txt | 1 + yiddish.egg-info/top_level.txt | 1 + yiddish/__init__.py | 11 ++++ yiddish_text_tools.py => yiddish/yiddish.py | 66 +++++++++++--------- 13 files changed, 230 insertions(+), 51 deletions(-) create mode 100644 AUTHORS create mode 100644 LICENSE create mode 100644 README.md create mode 100644 pyproject.toml delete mode 100644 respell_yiddish.py create mode 100644 setup.py create mode 100644 yiddish.egg-info/PKG-INFO create mode 100644 yiddish.egg-info/SOURCES.txt create mode 100644 yiddish.egg-info/dependency_links.txt create mode 100644 yiddish.egg-info/requires.txt create mode 100644 yiddish.egg-info/top_level.txt create mode 100644 yiddish/__init__.py rename yiddish_text_tools.py => yiddish/yiddish.py (81%) diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..a4dfef3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +Isaac L. Bleaman +(additional authors to be added here) + +ILB wrote many of these functions for personal use, in a few different programming languages, over the years. They were put together in a single .py file in 2021 for REYD, a Yiddish text-to-speech project, and then packaged as a standalone Python library in 2022. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bbdc7e4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2021-2022 Isaac L. Bleaman + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..de53820 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# yiddish + +A Python library for processing Yiddish text + +Isaac L. Bleaman () and contributors (see commit history) + +## What is this for? + +This library includes functions to carry out common tasks when dealing with Yiddish text. For example, you might wish to replace precombined Unicode characters (such as אַ, U+FB2E) with their decomposed versions (אַ, which is U+05D0 followed by U+05B7). Or you might wish to transliterate YIVO Yiddish text +(`איבער` to `iber`) or render it in the orthography used more commonly in the +Hasidic community (`שנײיִק` to `שנייאיג`). + +See the source file, `yiddish.py`, for the full list of supported functions. + +## How to install + + pip install yiddish + +## How to cite + +If you'd like to cite `yiddish` in a publication, you can include a link to the source: + https://github.com/ibleaman/yiddish + +## Example + +```python +import yiddish + +output = '' + +string = 'אונדזער גאַנצע משפּחה װױנט אין די פֿאַראײניקטע שטאַטן.' + +output += yiddish.replace_with_precombined(string) + '\n' +output += yiddish.respell_loshn_koydesh(string) + '\n' +output += yiddish.strip_diacritics(string) + '\n' +output += yiddish.transliterate(string) + '\n' +output += yiddish.transliterate(string, loshn_koydesh=True) + '\n' +output += yiddish.hasidify(string) + +output += '\n\n' + +string_two = 'shloymele hot khasene gehat mit rokhls tokhter leye.' + +output += yiddish.detransliterate(string_two) + '\n' +output += yiddish.detransliterate(string_two, loshn_koydesh=True) + +print(output) + +``` + +Output: + +``` +אונדזער גאַנצע משפּחה װױנט אין די פֿאַראײניקטע שטאַטן. +אונדזער גאַנצע מישפּאָכע װױנט אין די פֿאַראײניקטע שטאַטן. +אונדזער גאנצע משפחה וווינט אין די פאראייניקטע שטאטן. +undzer gantse mshpkhh voynt in di fareynikte shtatn. +undzer gantse mishpokhe voynt in di fareynikte shtatn. +אונזער גאנצע משפחה וואוינט אין די פאראייניקטע שטאטן. + +שלױמעלע האָט כאַסענע געהאַט מיט ראָכלס טאָכטער לײע. +שלמהלע האָט חתונה געהאַט מיט רחלס טאָכטער לאה. +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..374b58c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/respell_yiddish.py b/respell_yiddish.py deleted file mode 100644 index d29c5be..0000000 --- a/respell_yiddish.py +++ /dev/null @@ -1,21 +0,0 @@ -import argparse -import re -import yiddish_text_tools - -parser = argparse.ArgumentParser(description="respeller for Yiddish") -parser.add_argument("-i", "--input", required=True, help="text file") -parser.add_argument("-o", "--output", required=True, help="where to save reformatted text file") -parser.add_argument("-l", "--lk", action='store_true') -args = parser.parse_args() - -with open(args.input, "r") as content_file: - input_string = content_file.read() - -output_string = yiddish_text_tools.replace_with_precombined(input_string) -output_string = yiddish_text_tools.replace_punctuation(output_string) - -if args.lk: - output_string = yiddish_text_tools.respell_loshn_koydesh(output_string) - -with open(args.output, "w") as text_file: - text_file.write(output_string) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6e630de --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup + +with open('README.md', 'r', encoding='utf-8') as fh: + long_description = fh.read() + +setup( + name='yiddish', + version='0.0.1', + author='Isaac L. Bleaman', + author_email='bleaman@berkeley.edu', + description='A Python library for processing Yiddish text', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/ibleaman/yiddish', + install_requires=[ + "pandas>=1.1.0", + ], + license='MIT', + packages=['yiddish'], + test_suite='tests', +) diff --git a/yiddish.egg-info/PKG-INFO b/yiddish.egg-info/PKG-INFO new file mode 100644 index 0000000..0b05b90 --- /dev/null +++ b/yiddish.egg-info/PKG-INFO @@ -0,0 +1,67 @@ +Metadata-Version: 2.1 +Name: yiddish +Version: 0.0.5 +Summary: A Python library for processing Yiddish text +Home-page: https://github.com/ibleaman/yiddish +Author: Isaac L. Bleaman +Author-email: bleaman@berkeley.edu +License: MIT +Platform: UNKNOWN +Description-Content-Type: text/markdown +License-File: LICENSE +License-File: AUTHORS + +# yiddish + +A Python library for processing Yiddish text + +Isaac L. Bleaman () and contributors (see commit history) + +## What is this for? + +This library includes functions to accomplish common tasks when dealing with Yiddish text. For example, you might wish to replace precombined Unicode characters (such as אַ, U+FB2E) with their decomposed versions (אַ, which is U+05D0 followed by U+05B7). Or you might wish to transliterate YIVO Yiddish text +(`איבער` to `iber`) or render it in the orthography used more commonly in the +Hasidic community (`שנײיִק` -> `שנייאיג`). + +See the source file, `yiddish.py`, for the full list of supported functions. + +## How to install + + pip install yiddish + +## How to cite + +If you'd like to cite `yiddish` in a publication, you can include a link to the source: + https://github.com/ibleaman/yiddish + +## Example + +```python +import yiddish + +output = '' + +string = 'די גאַנצע משפּחה װױנט אין די פֿאַראײניקטע שטאַטן.' + +output += yiddish.replace_with_precombined(string) + '\n' +output += yiddish.strip_diacritics(string) + '\n' +output += yiddish.transliterate(string) + '\n' +output += yiddish.transliterate(string, loshn_koydesh=True) + '\n' +output += yiddish.hasidify(string) + + +string_two = 'moyshe-leyb hot khasene gehat mit esters tokhter.' + +output += yiddish.detransliterate(string_two) + '\n' +output += yiddish.detransliterate(string_two, loshn_koydesh=True) + +print(output) + +``` + +Output: + +``` + +``` + diff --git a/yiddish.egg-info/SOURCES.txt b/yiddish.egg-info/SOURCES.txt new file mode 100644 index 0000000..b629ab0 --- /dev/null +++ b/yiddish.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +AUTHORS +LICENSE +README.md +pyproject.toml +setup.py +yiddish/__init__.py +yiddish/yiddish.py +yiddish.egg-info/PKG-INFO +yiddish.egg-info/SOURCES.txt +yiddish.egg-info/dependency_links.txt +yiddish.egg-info/requires.txt +yiddish.egg-info/top_level.txt \ No newline at end of file diff --git a/yiddish.egg-info/dependency_links.txt b/yiddish.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/yiddish.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/yiddish.egg-info/requires.txt b/yiddish.egg-info/requires.txt new file mode 100644 index 0000000..5fb9d46 --- /dev/null +++ b/yiddish.egg-info/requires.txt @@ -0,0 +1 @@ +pandas>=1.1.0 diff --git a/yiddish.egg-info/top_level.txt b/yiddish.egg-info/top_level.txt new file mode 100644 index 0000000..c31c1d5 --- /dev/null +++ b/yiddish.egg-info/top_level.txt @@ -0,0 +1 @@ +yiddish diff --git a/yiddish/__init__.py b/yiddish/__init__.py new file mode 100644 index 0000000..7a7432d --- /dev/null +++ b/yiddish/__init__.py @@ -0,0 +1,11 @@ +from .yiddish import ( + replace_with_precombined, + replace_with_decomposed, + replace_punctuation, + strip_diacritics, + transliterate, + detransliterate, + romanise_german, + respell_loshn_koydesh, + hasidify, +) \ No newline at end of file diff --git a/yiddish_text_tools.py b/yiddish/yiddish.py similarity index 81% rename from yiddish_text_tools.py rename to yiddish/yiddish.py index 650797b..9e30574 100644 --- a/yiddish_text_tools.py +++ b/yiddish/yiddish.py @@ -1,9 +1,10 @@ -# Functions for processing Yiddish text, written in the YIVO orthography +# yiddish +# A Python library for processing Yiddish text # Author: Isaac L. Bleaman (bleaman@berkeley.edu) import re from urllib.request import urlopen -import pandas as pd +import csv ########## # encoding @@ -83,7 +84,10 @@ def strip_diacritics(string): # and replace with decomposed for entry in entries.split(','): if entry not in reverse_lk: reverse_lk[entry] = key - + +germanic_semitic_homographs = ["אין", "צום", "בין", "ברי", "מיד", "קין", "שער", "מעגן", "צו", "מאַנס", "טוען", "מערער"] + +less_common_lk_pronunciations = ["אַדױשעם", "כאַנוקע", "גדױלע", "כאַװײרע", "מיכיע", "כאָװער", "אָרעװ", "מאָסער", "כיִעס", "זקאָנים", "נעװאָלע", "מאַשלעם", "כפֿאָצים", "כאַכאָמע", "טאַנאָיִם", "יאָסעף", "יאָסעפֿס", "יאָסעפֿן"] translit_table = [ # all are precombined ('א', ''), @@ -141,8 +145,11 @@ def transliterate(string, loshn_koydesh=False): tokens = re.findall(r"[אאַאָבבֿגדהוװוּױזחטייִײײַככּךלמםנןסעפּפֿףצץקרששׂתּת\-־]+|[^אאַאָבבֿגדהוװוּױזחטייִײײַככּךלמםנןסעפּפֿףצץקרששׂתּת\-־]", romanized) new_tokens = [] for token in tokens: - if token in lk: - new_tokens.append(lk[token][0].replace('־', '-')) + if token in lk and token not in germanic_semitic_homographs: + if lk[token][0] in less_common_lk_pronunciations and len(lk[token]) > 1: + new_tokens.append(lk[token][1].replace('־', '-')) + else: + new_tokens.append(lk[token][0].replace('־', '-')) else: new_tokens.append(token) @@ -411,13 +418,15 @@ def romanise_german(text): # for TTS: respell orthographic words phonetically ################################################## +# Note: input text WILL become precombined def respell_loshn_koydesh(text): + text = replace_with_precombined(text) # loop over keys, in reverse order from longest keys to shortest for key in sorted(list(lk.keys()), key=len, reverse=True): # skip Germanic homographs, which are usually phonetic - if key not in ["אין", "צום", "בין", "ברי", "מיד", "קין", "שער", "מעגן", "צו", "מאַנס", "טוען", "מערער"]: - # skip less common LK variants, in favor of more common ones - if lk[key][0] in ["אַדױשעם", "כאַנוקע", "גדױלע", "כאַװײרע", "מיכיע", "כאָװער", "אָרעװ", "מאָסער", "כיִעס", "זקאָנים", "נעװאָלע", "מאַשלעם", "כפֿאָצים", "כאַכאָמע", "טאַנאָיִם", "יאָסעף", "יאָסעפֿס", "יאָסעפֿן"] and len(lk[key]) > 1: + if key not in germanic_semitic_homographs: + # skip less common LK pronunciations, in favor of more common ones + if lk[key][0] in less_common_lk_pronunciations and len(lk[key]) > 1: replacement = lk[key][1] else: replacement = lk[key][0] @@ -454,37 +463,34 @@ def respell_loshn_koydesh(text): # note: all replacements are based on # looking for precombined characters ####################################### -hasidifier_lexicon = pd.ExcelFile('https://docs.google.com/spreadsheets/d/1x_KLOaUfnCBVVWEb523QIJZIvk65Pp0ly9MPMVLUVOI/export?format=xlsx') +hasidify_lexicon = 'https://raw.githubusercontent.com/ibleaman/hasidify_lexicon/master/' -whole_word_variants = pd.read_excel(hasidifier_lexicon, 'whole_word_variants') -whole_word_variants = dict(zip([replace_with_precombined(word) for word in whole_word_variants['Find']], [replace_with_precombined(word) for word in whole_word_variants['Replace']])) +whole_word_variants = list(csv.reader(urlopen(hasidify_lexicon + 'whole_word_variants.csv').read().decode('utf-8').replace('\r', '').splitlines())) +whole_word_variants = dict(zip([replace_with_precombined(row[0]) for row in whole_word_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in whole_word_variants if row[1] != 'Replace'])) -prefix_variants = pd.read_excel(hasidifier_lexicon, 'prefix_variants') -prefix_variants = dict(zip([replace_with_precombined(word) for word in prefix_variants['Find']], [replace_with_precombined(word) for word in prefix_variants['Replace']])) +prefix_variants = list(csv.reader(urlopen(hasidify_lexicon + 'prefix_variants.csv').read().decode('utf-8').replace('\r', '').splitlines())) +prefix_variants = dict(zip([replace_with_precombined(row[0]) for row in prefix_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in prefix_variants if row[1] != 'Replace'])) -suffix_variants = pd.read_excel(hasidifier_lexicon, 'suffix_variants') -suffix_variants = dict(zip([replace_with_precombined(word) for word in suffix_variants['Find']], [replace_with_precombined(word) for word in suffix_variants['Replace']])) +suffix_variants = list(csv.reader(urlopen(hasidify_lexicon + 'suffix_variants.csv').read().decode('utf-8').replace('\r', '').splitlines())) +suffix_variants = dict(zip([replace_with_precombined(row[0]) for row in suffix_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in suffix_variants if row[1] != 'Replace'])) -anywhere_variants = pd.read_excel(hasidifier_lexicon, 'anywhere_variants') -anywhere_variants = dict(zip([replace_with_precombined(word) for word in anywhere_variants['Find']], [replace_with_precombined(word) for word in anywhere_variants['Replace']])) +anywhere_variants = list(csv.reader(urlopen(hasidify_lexicon + 'anywhere_variants.csv').read().decode('utf-8').replace('\r', '').splitlines())) +anywhere_variants = dict(zip([replace_with_precombined(row[0]) for row in anywhere_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in anywhere_variants if row[1] != 'Replace'])) -lkizmen = pd.read_excel(hasidifier_lexicon, 'lkizmen') -lkizmen = lkizmen['Words'].tolist() -lkizmen = [replace_with_precombined(word) for word in lkizmen] +lkizmen = list(csv.reader(urlopen(hasidify_lexicon + 'lkizmen.csv').read().decode('utf-8').replace('\r', '').splitlines())) +lkizmen = [replace_with_precombined(row[0]) for row in lkizmen if row[0] != 'Words'] -word_group_variants = pd.read_excel(hasidifier_lexicon, 'word_group_variants') -word_group_variants = dict(zip([replace_with_precombined(word) for word in word_group_variants['Find']], [replace_with_precombined(word) for word in word_group_variants['Replace']])) +word_group_variants = list(csv.reader(urlopen(hasidify_lexicon + 'word_group_variants.csv').read().decode('utf-8').replace('\r', '').splitlines())) +word_group_variants = dict(zip([replace_with_precombined(row[0]) for row in word_group_variants if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in word_group_variants if row[1] != 'Replace'])) -ik_exceptions = pd.read_excel(hasidifier_lexicon, 'ik_exceptions') -ik_exceptions = ik_exceptions['Words'].tolist() -ik_exceptions = [replace_with_precombined(word) for word in ik_exceptions] +ik_exceptions = list(csv.reader(urlopen(hasidify_lexicon + 'ik_exceptions.csv').read().decode('utf-8').replace('\r', '').splitlines())) +ik_exceptions = [replace_with_precombined(row[0]) for row in ik_exceptions if row[0] != 'Words'] -lekh_exceptions = pd.read_excel(hasidifier_lexicon, 'lekh_exceptions') -lekh_exceptions = lekh_exceptions['Words'].tolist() -lekh_exceptions = [replace_with_precombined(word) for word in lekh_exceptions] +lekh_exceptions = list(csv.reader(urlopen(hasidify_lexicon + 'lekh_exceptions.csv').read().decode('utf-8').replace('\r', '').splitlines())) +lekh_exceptions = [replace_with_precombined(row[0]) for row in lekh_exceptions if row[0] != 'Words'] -last_minute_fixes = pd.read_excel(hasidifier_lexicon, 'last_minute_fixes') -last_minute_fixes = dict(zip(last_minute_fixes['Find'], last_minute_fixes['Replace'])) +last_minute_fixes = list(csv.reader(urlopen(hasidify_lexicon + 'last_minute_fixes.csv').read().decode('utf-8').replace('\r', '').splitlines())) +last_minute_fixes = dict(zip([replace_with_precombined(row[0]) for row in last_minute_fixes if row[0] != 'Find'], [replace_with_precombined(row[1]) for row in last_minute_fixes if row[1] != 'Replace'])) reformatting = [ ('וּװוּ', 'ואוואו'),