From 1b726aede262c649d7165843a0d16116f02ec929 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Wed, 7 Jun 2023 14:41:37 +0000 Subject: [PATCH] Move nonempty_string_arg and module_deprecation_helper to snscrape.utils --- snscrape/base.py | 24 ++---------------------- snscrape/modules/facebook.py | 5 +++-- snscrape/modules/instagram.py | 5 +++-- snscrape/modules/mastodon.py | 5 +++-- snscrape/modules/reddit.py | 5 +++-- snscrape/modules/telegram.py | 3 ++- snscrape/modules/twitter.py | 10 +++++----- snscrape/modules/vkontakte.py | 3 ++- snscrape/modules/weibo.py | 3 ++- snscrape/utils.py | 25 +++++++++++++++++++++++++ 10 files changed, 50 insertions(+), 38 deletions(-) diff --git a/snscrape/base.py b/snscrape/base.py index f42809de..12404c80 100644 --- a/snscrape/base.py +++ b/snscrape/base.py @@ -11,6 +11,7 @@ import random import requests import requests.adapters +import snscrape.utils import snscrape.version import urllib3.connection import time @@ -20,17 +21,6 @@ _logger = logging.getLogger(__name__) -def _module_deprecation_helper(all, **names): - def __getattr__(name): - if name in names: - warnings.warn(f'{name} is deprecated, use {names[name].__name__} instead', DeprecatedFeatureWarning, stacklevel = 2) - return names[name] - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - def __dir__(): - return sorted(all + list(names.keys())) - return __getattr__, __dir__ - - class DeprecatedFeatureWarning(FutureWarning): pass @@ -297,14 +287,4 @@ def _cli_construct(cls, argparseArgs, *args, **kwargs): return cls(*args, **kwargs, retries = argparseArgs.retries) -def nonempty_string(name): - def f(s): - s = s.strip() - if s: - return s - raise ValueError('must not be an empty string') - f.__name__ = name - return f - - -__getattr__, __dir__ = _module_deprecation_helper(__all__, Entity = Item) +__getattr__, __dir__ = snscrape.utils.module_deprecation_helper(__all__, Entity = Item) diff --git a/snscrape/modules/facebook.py b/snscrape/modules/facebook.py index c2839f18..584f622c 100644 --- a/snscrape/modules/facebook.py +++ b/snscrape/modules/facebook.py @@ -8,6 +8,7 @@ import logging import re import snscrape.base +import snscrape.utils import typing import urllib.parse @@ -206,7 +207,7 @@ def get_items(self): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'A Facebook username or user ID') + subparser.add_argument('username', type = snscrape.utils.nonempty_string_arg('username'), help = 'A Facebook username or user ID') @classmethod def _cli_from_args(cls, args): @@ -357,7 +358,7 @@ def get_items(self): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('group', type = snscrape.base.nonempty_string('group'), help = 'A group name or ID') + subparser.add_argument('group', type = snscrape.utils.nonempty_string_arg('group'), help = 'A group name or ID') @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/modules/instagram.py b/snscrape/modules/instagram.py index 1cd5db79..49733335 100644 --- a/snscrape/modules/instagram.py +++ b/snscrape/modules/instagram.py @@ -8,6 +8,7 @@ import logging import re import snscrape.base +import snscrape.utils import typing @@ -192,7 +193,7 @@ def parse_num(s): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'An Instagram username (no leading @)') + subparser.add_argument('username', type = snscrape.utils.nonempty_string_arg('username'), help = 'An Instagram username (no leading @)') @classmethod def _cli_from_args(cls, args): @@ -214,7 +215,7 @@ def __init__(self, hashtag, **kwargs): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('hashtag', type = snscrape.base.nonempty_string('hashtag'), help = 'An Instagram hashtag (no leading #)') + subparser.add_argument('hashtag', type = snscrape.utils.nonempty_string_arg('hashtag'), help = 'An Instagram hashtag (no leading #)') @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/modules/mastodon.py b/snscrape/modules/mastodon.py index 653e83a1..1d8d04c3 100644 --- a/snscrape/modules/mastodon.py +++ b/snscrape/modules/mastodon.py @@ -8,6 +8,7 @@ import json import logging import snscrape.base +import snscrape.utils import time import typing import urllib.parse @@ -289,7 +290,7 @@ def get_items(self): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('account', type = snscrape.base.nonempty_string('account'), help = 'A Mastodon account. This can be either a URL to the profile page or a string of the form @account@instance.example.org') + subparser.add_argument('account', type = snscrape.utils.nonempty_string_arg('account'), help = 'A Mastodon account. This can be either a URL to the profile page or a string of the form @account@instance.example.org') @classmethod def _cli_from_args(cls, args): @@ -333,7 +334,7 @@ def get_items(self): @classmethod def _cli_setup_parser(cls, subparser): subparser.add_argument('--thread', action = 'store_true', help = 'Collect thread around the toot referenced by the URL') - subparser.add_argument('url', type = snscrape.base.nonempty_string('url'), help = 'A URL for a toot') + subparser.add_argument('url', type = snscrape.utils.nonempty_string_arg('url'), help = 'A URL for a toot') @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/modules/reddit.py b/snscrape/modules/reddit.py index f93b96e6..c6938de3 100644 --- a/snscrape/modules/reddit.py +++ b/snscrape/modules/reddit.py @@ -6,6 +6,7 @@ import logging import re import snscrape.base +import snscrape.utils import snscrape.version import string import time @@ -224,7 +225,7 @@ def _cli_setup_parser(cls, subparser): subparser.add_argument('--before', metavar = 'TIMESTAMP', type = int, help = 'Fetch results before a Unix timestamp') subparser.add_argument('--after', metavar = 'TIMESTAMP', type = int, help = 'Fetch results after a Unix timestamp') name = cls.name.split('-', 1)[1] - subparser.add_argument(name, type = snscrape.base.nonempty_string(name)) + subparser.add_argument(name, type = snscrape.utils.nonempty_string_arg(name)) @classmethod def _cli_from_args(cls, args): @@ -272,7 +273,7 @@ def get_items(self): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('submissionId', type = snscrape.base.nonempty_string('submissionId')) + subparser.add_argument('submissionId', type = snscrape.utils.nonempty_string_arg('submissionId')) @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 5f9ffe65..191a63ff 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -7,6 +7,7 @@ import logging import re import snscrape.base +import snscrape.utils import typing import urllib.parse @@ -196,7 +197,7 @@ def parse_num(s): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('channel', type = snscrape.base.nonempty_string('channel'), help = 'A channel name') + subparser.add_argument('channel', type = snscrape.utils.nonempty_string_arg('channel'), help = 'A channel name') @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index e1a41661..1803ab7c 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -1701,7 +1701,7 @@ def _cli_setup_parser(cls, subparser): group.add_argument('--top', action = 'store_true', default = False, help = 'Search top tweets instead of live/chronological') group.add_argument('--user', action = 'store_true', default = False, help = 'Search users instead of tweets') subparser.add_argument('--max-empty-pages', dest = 'maxEmptyPages', metavar = 'N', type = int, default = 20, help = 'Stop after N empty pages from Twitter; set to 0 to disable') - subparser.add_argument('query', type = snscrape.base.nonempty_string('query'), help = 'A Twitter search string') + subparser.add_argument('query', type = snscrape.utils.nonempty_string_arg('query'), help = 'A Twitter search string') @classmethod def _cli_from_args(cls, args): @@ -1850,7 +1850,7 @@ def __init__(self, hashtag, **kwargs): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('hashtag', type = snscrape.base.nonempty_string('hashtag'), help = 'A Twitter hashtag (without #)') + subparser.add_argument('hashtag', type = snscrape.utils.nonempty_string_arg('hashtag'), help = 'A Twitter hashtag (without #)') @classmethod def _cli_from_args(cls, args): @@ -1866,7 +1866,7 @@ def __init__(self, cashtag, **kwargs): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('cashtag', type = snscrape.base.nonempty_string('cashtag'), help = 'A Twitter cashtag (without $)') + subparser.add_argument('cashtag', type = snscrape.utils.nonempty_string_arg('cashtag'), help = 'A Twitter cashtag (without $)') @classmethod def _cli_from_args(cls, args): @@ -1996,7 +1996,7 @@ def __init__(self, listName, **kwargs): @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('list', type = snscrape.base.nonempty_string('list'), help = 'A Twitter list ID or a string of the form "username/listname" (replace spaces with dashes)') + subparser.add_argument('list', type = snscrape.utils.nonempty_string_arg('list'), help = 'A Twitter list ID or a string of the form "username/listname" (replace spaces with dashes)') @classmethod def _cli_from_args(cls, args): @@ -2150,4 +2150,4 @@ def get_items(self): yield Trend(name = trend['name'], metaDescription = trend['trendMetadata'].get('metaDescription'), domainContext = trend['trendMetadata']['domainContext']) -__getattr__, __dir__ = snscrape.base._module_deprecation_helper(__all__, DescriptionURL = TextLink) +__getattr__, __dir__ = snscrape.utils.module_deprecation_helper(__all__, DescriptionURL = TextLink) diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py index 1b577ae9..2d600f6f 100644 --- a/snscrape/modules/vkontakte.py +++ b/snscrape/modules/vkontakte.py @@ -10,6 +10,7 @@ import logging import re import snscrape.base +import snscrape.utils import typing import urllib.parse try: @@ -384,7 +385,7 @@ def parse_num(s: str) -> typing.Tuple[int, int]: @classmethod def _cli_setup_parser(cls, subparser): - subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'A VK username') + subparser.add_argument('username', type = snscrape.utils.nonempty_string_arg('username'), help = 'A VK username') @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/modules/weibo.py b/snscrape/modules/weibo.py index 654fd622..04b90032 100644 --- a/snscrape/modules/weibo.py +++ b/snscrape/modules/weibo.py @@ -5,6 +5,7 @@ import logging import re import snscrape.base +import snscrape.utils import typing @@ -146,7 +147,7 @@ def _get_entity(self): @classmethod def _cli_setup_parser(cls, subparser): subparser.add_argument('--name', dest = 'isName', action = 'store_true', help = 'Use username instead of user ID') - subparser.add_argument('user', type = snscrape.base.nonempty_string('user'), help = 'A user ID') + subparser.add_argument('user', type = snscrape.utils.nonempty_string_arg('user'), help = 'A user ID') @classmethod def _cli_from_args(cls, args): diff --git a/snscrape/utils.py b/snscrape/utils.py index 31509243..0d872832 100644 --- a/snscrape/utils.py +++ b/snscrape/utils.py @@ -14,3 +14,28 @@ def snake_to_camel(**kwargs): keyParts[i] = keyParts[i][:1].upper() + keyParts[i][1:] out[''.join(keyParts)] = value return out + + +def nonempty_string_arg(name): + '''An argparse argument type factory for a non-empty string argument. The supplied `name` is used for the internal function name, resulting in better error messages.''' + + def f(s): + s = s.strip() + if s: + return s + raise ValueError('must not be an empty string') + f.__name__ = name + return f + + +def module_deprecation_helper(all, **names): + '''A helper function to generate the relevant module __getattr__ and __dir__ functions for handling deprecated names''' + + def __getattr__(name): + if name in names: + warnings.warn(f'{name} is deprecated, use {names[name].__name__} instead', DeprecatedFeatureWarning, stacklevel = 2) + return names[name] + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + def __dir__(): + return sorted(all + list(names.keys())) + return __getattr__, __dir__