Move nonempty_string_arg and module_deprecation_helper to snscrape.utils

JustAnotherArchivist · Jun 7, 2023 · 1b726ae · 1b726ae
1 parent 6d2dfa7
commit 1b726ae
Show file tree

Hide file tree

Showing 10 changed files with 50 additions and 38 deletions.
diff --git a/snscrape/base.py b/snscrape/base.py
@@ -11,6 +11,7 @@
 import random
 import requests
 import requests.adapters
+import snscrape.utils
 import snscrape.version
 import urllib3.connection
 import time
@@ -20,17 +21,6 @@
 _logger = logging.getLogger(__name__)
 
 
-def _module_deprecation_helper(all, **names):
-	def __getattr__(name):
-		if name in names:
-			warnings.warn(f'{name} is deprecated, use {names[name].__name__} instead', DeprecatedFeatureWarning, stacklevel = 2)
-			return names[name]
-		raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
-	def __dir__():
-		return sorted(all + list(names.keys()))
-	return __getattr__, __dir__
-
-
 class DeprecatedFeatureWarning(FutureWarning):
 	pass
 
@@ -297,14 +287,4 @@ def _cli_construct(cls, argparseArgs, *args, **kwargs):
 		return cls(*args, **kwargs, retries = argparseArgs.retries)
 
 
-def nonempty_string(name):
-	def f(s):
-		s = s.strip()
-		if s:
-			return s
-		raise ValueError('must not be an empty string')
-	f.__name__ = name
-	return f
-
-
-__getattr__, __dir__ = _module_deprecation_helper(__all__, Entity = Item)
+__getattr__, __dir__ = snscrape.utils.module_deprecation_helper(__all__, Entity = Item)
diff --git a/snscrape/modules/facebook.py b/snscrape/modules/facebook.py
@@ -8,6 +8,7 @@
 import logging
 import re
 import snscrape.base
+import snscrape.utils
 import typing
 import urllib.parse
 
@@ -206,7 +207,7 @@ def get_items(self):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'A Facebook username or user ID')
+		subparser.add_argument('username', type = snscrape.utils.nonempty_string_arg('username'), help = 'A Facebook username or user ID')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -357,7 +358,7 @@ def get_items(self):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('group', type = snscrape.base.nonempty_string('group'), help = 'A group name or ID')
+		subparser.add_argument('group', type = snscrape.utils.nonempty_string_arg('group'), help = 'A group name or ID')
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/modules/instagram.py b/snscrape/modules/instagram.py
@@ -8,6 +8,7 @@
 import logging
 import re
 import snscrape.base
+import snscrape.utils
 import typing
 
 
@@ -192,7 +193,7 @@ def parse_num(s):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'An Instagram username (no leading @)')
+		subparser.add_argument('username', type = snscrape.utils.nonempty_string_arg('username'), help = 'An Instagram username (no leading @)')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -214,7 +215,7 @@ def __init__(self, hashtag, **kwargs):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('hashtag', type = snscrape.base.nonempty_string('hashtag'), help = 'An Instagram hashtag (no leading #)')
+		subparser.add_argument('hashtag', type = snscrape.utils.nonempty_string_arg('hashtag'), help = 'An Instagram hashtag (no leading #)')
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/modules/mastodon.py b/snscrape/modules/mastodon.py
@@ -8,6 +8,7 @@
 import json
 import logging
 import snscrape.base
+import snscrape.utils
 import time
 import typing
 import urllib.parse
@@ -289,7 +290,7 @@ def get_items(self):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('account', type = snscrape.base.nonempty_string('account'), help = 'A Mastodon account. This can be either a URL to the profile page or a string of the form @[email protected]')
+		subparser.add_argument('account', type = snscrape.utils.nonempty_string_arg('account'), help = 'A Mastodon account. This can be either a URL to the profile page or a string of the form @[email protected]')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -333,7 +334,7 @@ def get_items(self):
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
 		subparser.add_argument('--thread', action = 'store_true', help = 'Collect thread around the toot referenced by the URL')
-		subparser.add_argument('url', type = snscrape.base.nonempty_string('url'), help = 'A URL for a toot')
+		subparser.add_argument('url', type = snscrape.utils.nonempty_string_arg('url'), help = 'A URL for a toot')
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/modules/reddit.py b/snscrape/modules/reddit.py
@@ -6,6 +6,7 @@
 import logging
 import re
 import snscrape.base
+import snscrape.utils
 import snscrape.version
 import string
 import time
@@ -224,7 +225,7 @@ def _cli_setup_parser(cls, subparser):
 		subparser.add_argument('--before', metavar = 'TIMESTAMP', type = int, help = 'Fetch results before a Unix timestamp')
 		subparser.add_argument('--after', metavar = 'TIMESTAMP', type = int, help = 'Fetch results after a Unix timestamp')
 		name = cls.name.split('-', 1)[1]
-		subparser.add_argument(name, type = snscrape.base.nonempty_string(name))
+		subparser.add_argument(name, type = snscrape.utils.nonempty_string_arg(name))
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -272,7 +273,7 @@ def get_items(self):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('submissionId', type = snscrape.base.nonempty_string('submissionId'))
+		subparser.add_argument('submissionId', type = snscrape.utils.nonempty_string_arg('submissionId'))
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
@@ -7,6 +7,7 @@
 import logging
 import re
 import snscrape.base
+import snscrape.utils
 import typing
 import urllib.parse
 
@@ -196,7 +197,7 @@ def parse_num(s):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('channel', type = snscrape.base.nonempty_string('channel'), help = 'A channel name')
+		subparser.add_argument('channel', type = snscrape.utils.nonempty_string_arg('channel'), help = 'A channel name')
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py
@@ -1701,7 +1701,7 @@ def _cli_setup_parser(cls, subparser):
 		group.add_argument('--top', action = 'store_true', default = False, help = 'Search top tweets instead of live/chronological')
 		group.add_argument('--user', action = 'store_true', default = False, help = 'Search users instead of tweets')
 		subparser.add_argument('--max-empty-pages', dest = 'maxEmptyPages', metavar = 'N', type = int, default = 20, help = 'Stop after N empty pages from Twitter; set to 0 to disable')
-		subparser.add_argument('query', type = snscrape.base.nonempty_string('query'), help = 'A Twitter search string')
+		subparser.add_argument('query', type = snscrape.utils.nonempty_string_arg('query'), help = 'A Twitter search string')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -1850,7 +1850,7 @@ def __init__(self, hashtag, **kwargs):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('hashtag', type = snscrape.base.nonempty_string('hashtag'), help = 'A Twitter hashtag (without #)')
+		subparser.add_argument('hashtag', type = snscrape.utils.nonempty_string_arg('hashtag'), help = 'A Twitter hashtag (without #)')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -1866,7 +1866,7 @@ def __init__(self, cashtag, **kwargs):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('cashtag', type = snscrape.base.nonempty_string('cashtag'), help = 'A Twitter cashtag (without $)')
+		subparser.add_argument('cashtag', type = snscrape.utils.nonempty_string_arg('cashtag'), help = 'A Twitter cashtag (without $)')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -1996,7 +1996,7 @@ def __init__(self, listName, **kwargs):
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('list', type = snscrape.base.nonempty_string('list'), help = 'A Twitter list ID or a string of the form "username/listname" (replace spaces with dashes)')
+		subparser.add_argument('list', type = snscrape.utils.nonempty_string_arg('list'), help = 'A Twitter list ID or a string of the form "username/listname" (replace spaces with dashes)')
 
 	@classmethod
 	def _cli_from_args(cls, args):
@@ -2150,4 +2150,4 @@ def get_items(self):
 					yield Trend(name = trend['name'], metaDescription = trend['trendMetadata'].get('metaDescription'), domainContext = trend['trendMetadata']['domainContext'])
 
 
-__getattr__, __dir__ = snscrape.base._module_deprecation_helper(__all__, DescriptionURL = TextLink)
+__getattr__, __dir__ = snscrape.utils.module_deprecation_helper(__all__, DescriptionURL = TextLink)
diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py
@@ -10,6 +10,7 @@
 import logging
 import re
 import snscrape.base
+import snscrape.utils
 import typing
 import urllib.parse
 try:
@@ -384,7 +385,7 @@ def parse_num(s: str) -> typing.Tuple[int, int]:
 
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
-		subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'A VK username')
+		subparser.add_argument('username', type = snscrape.utils.nonempty_string_arg('username'), help = 'A VK username')
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/modules/weibo.py b/snscrape/modules/weibo.py
@@ -5,6 +5,7 @@
 import logging
 import re
 import snscrape.base
+import snscrape.utils
 import typing
 
 
@@ -146,7 +147,7 @@ def _get_entity(self):
 	@classmethod
 	def _cli_setup_parser(cls, subparser):
 		subparser.add_argument('--name', dest = 'isName', action = 'store_true', help = 'Use username instead of user ID')
-		subparser.add_argument('user', type = snscrape.base.nonempty_string('user'), help = 'A user ID')
+		subparser.add_argument('user', type = snscrape.utils.nonempty_string_arg('user'), help = 'A user ID')
 
 	@classmethod
 	def _cli_from_args(cls, args):

diff --git a/snscrape/utils.py b/snscrape/utils.py
@@ -14,3 +14,28 @@ def snake_to_camel(**kwargs):
 			keyParts[i] = keyParts[i][:1].upper() + keyParts[i][1:]
 		out[''.join(keyParts)] = value
 	return out
+
+
+def nonempty_string_arg(name):
+	'''An argparse argument type factory for a non-empty string argument. The supplied `name` is used for the internal function name, resulting in better error messages.'''
+
+	def f(s):
+		s = s.strip()
+		if s:
+			return s
+		raise ValueError('must not be an empty string')
+	f.__name__ = name
+	return f
+
+
+def module_deprecation_helper(all, **names):
+	'''A helper function to generate the relevant module __getattr__ and __dir__ functions for handling deprecated names'''
+
+	def __getattr__(name):
+		if name in names:
+			warnings.warn(f'{name} is deprecated, use {names[name].__name__} instead', DeprecatedFeatureWarning, stacklevel = 2)
+			return names[name]
+		raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
+	def __dir__():
+		return sorted(all + list(names.keys()))
+	return __getattr__, __dir__