From 002168d2ffda36c4537bd3fe0b50aaacacf3ac0a Mon Sep 17 00:00:00 2001 From: Aripov Sergey Date: Tue, 30 Nov 2021 17:09:15 +0500 Subject: [PATCH 1/3] =?UTF-8?q?=D0=97=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LinkTestsResults.md | 0 TokensTestsResult.md | 19 +++++++ links_utils.py | 36 ++++++++++++ main.py | 0 ppe_token_generator_test.py | 0 res.txt | 104 +++++++++++++++++++++++++++++++++++ sync_link_test.py | 0 sync_token_generator_test.py | 0 tokens.py | 0 tpe_link_test.py | 0 10 files changed, 159 insertions(+) create mode 100644 LinkTestsResults.md create mode 100644 TokensTestsResult.md create mode 100644 links_utils.py create mode 100644 main.py create mode 100644 ppe_token_generator_test.py create mode 100644 res.txt create mode 100644 sync_link_test.py create mode 100644 sync_token_generator_test.py create mode 100644 tokens.py create mode 100644 tpe_link_test.py diff --git a/LinkTestsResults.md b/LinkTestsResults.md new file mode 100644 index 0000000..e69de29 diff --git a/TokensTestsResult.md b/TokensTestsResult.md new file mode 100644 index 0000000..7d1f39d --- /dev/null +++ b/TokensTestsResult.md @@ -0,0 +1,19 @@ +# Синхронный тест +Время: 145153211000 ns + +Загрузка cpu: <1% в среднем, не больше 2% + +# ThreadPoolExecutor - 5 +Среднее время (5 повторов): 34274738620 ns + +Загрузка cpu: >1% в среднем, не больше 5% + +# ThreadPoolExecutor - 10 +Среднее время (5 повторов): 21303062040 ns + +Загрузка cpu: >2.5% в среднем, не больше 5% + +# ThreadPoolExecutor - 100 +Среднее время (5 повторов): 6163196780 ns + +Загрузка cpu: >6% в среднем, не больше 15% \ No newline at end of file diff --git a/links_utils.py b/links_utils.py new file mode 100644 index 0000000..bea6ce6 --- /dev/null +++ b/links_utils.py @@ -0,0 +1,36 @@ +import os +from urllib.request import Request, urlopen +from bs4 import BeautifulSoup +from tqdm import tqdm + +url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0' + +if not os.path.exists('res.txt'): + res = open('res.txt', 'w', encoding='utf8') + + for i in tqdm(range(100)): + html = urlopen(url).read().decode('utf8') + soup = BeautifulSoup(html, 'html.parser') + links = soup.find_all('a') + + for l in links: + href = l.get('href') + if href and href.startswith('http') and 'wiki' not in href: + print(href, file=res) + + res.close() + + +def get_links(): + with open('res.txt', 'r', encoding='utf-8') as f: + return f.read().split('\n') + + +def load_url(url, timeout): + request = Request( + url, + headers={ + 'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'}, + ) + resp = urlopen(request, timeout=timeout) + return resp.code diff --git a/main.py b/main.py new file mode 100644 index 0000000..e69de29 diff --git a/ppe_token_generator_test.py b/ppe_token_generator_test.py new file mode 100644 index 0000000..e69de29 diff --git a/res.txt b/res.txt new file mode 100644 index 0000000..5f631eb --- /dev/null +++ b/res.txt @@ -0,0 +1,104 @@ +https://www.openstreetmap.org/?mlat=44.0589&mlon=5.2342&zoom=12 +https://www.openstreetmap.org/?mlat=44.0589&mlon=5.2342&zoom=12 +https://www.insee.fr/fr/statistiques/2011101?geo=COM-84148 +http://www.villes-sur-auzon.fr +http://www.insee.fr/fr/themes/tableau_local.asp?ref_id=POP&millesime=2010&nivgeo=COM&codgeo=84148 +http://www.insee.fr/fr/methodes/nomenclatures/cog/fichecommunale.asp?codedep=84&codecom=148&codecan=15 +https://www.openstreetmap.org/?mlat=34.94972&mlon=-90.47083&zoom=12 +https://www.openstreetmap.org/?mlat=34.94972&mlon=-90.47083&zoom=12 +https://edits.nationalmap.gov/apps/gaz-domestic/public/summary/83178 +https://edits.nationalmap.gov/apps/gaz-domestic/public/summary/2404739 +http://factfinder.census.gov +https://www.openstreetmap.org/?mlat=42.16667&mlon=26.53333&zoom=12 +https://www.openstreetmap.org/?mlat=42.16667&mlon=26.53333&zoom=12 +http://grao.bg/tna/tab02.txt +https://www.transfermarkt.com/transfermarkt/profil/spieler/223974 +https://fbref.com/en/players/7f609bfc/ +https://int.soccerway.com/matches/2017/09/28/europe/uefa-cup/fk-razgrad-2000/tsg-1899-hoffenheim-ev/2580369/ +https://int.soccerway.com/matches/2017/10/14/germany/bundesliga/tsg-1899-hoffenheim-ev/fc-augsburg/2480328/ +http://int.soccerway.com/matches/2016/07/11/europe/uefa-u19-championship/portugal-under-19/austria-under-19/2229976/ +http://int.soccerway.com/matches/2016/07/14/europe/uefa-u19-championship/italy-under-19/austria-under-19/2229978/ +http://int.soccerway.com/matches/2016/07/17/europe/uefa-u19-championship/austria-under-19/germany-under-19/2229980/ +https://int.soccerway.com/matches/2019/06/10/europe/european-championship-qualification/macedonia-fyr/austria/2942836/ +https://int.soccerway.com/matches/2019/06/17/europe/uefa-u21-championship/serbia-u21/austria-under-21/2940777/ +https://int.soccerway.com/matches/2019/06/20/europe/uefa-u21-championship/denmark-under-21/austria-under-21/2940779/ +https://int.soccerway.com/matches/2019/06/23/europe/uefa-u21-championship/austria-under-21/germany-under-21/2940780/ +https://www.national-football-teams.com/player/74577.html +https://www.sports.ru/tags/161060597/ +https://www.weltfussball.de/spieler_profil/stefan-posch/ +https://www.kicker.de/profil-90213/spieler +https://www.fussballdaten.de/person/stefan-posch/ +https://www.dfb.de/datencenter/personen/stefan-posch/spieler +https://eu-football.info/_player.php?id=30676 +https://fbref.com/en/players/7f609bfc/ +https://www.kicker.de/stefan-posch/spieler +https://www.national-football-teams.com/player/74577.html +https://int.soccerway.com/players/stefan-posch/316755/ +https://www.transfermarkt.com/transfermarkt/profil/spieler/223974 +https://www.worldfootball.net/player_summary/stefan-posch/ +http://kanashchuk.livejournal.com +http://amateurphotographer.ru/?p=5287 +http://lenta.ru/news/2011/03/02/prize/ +http://www.mdf.ru/contests/silvercam/silvercam2009/sk_architecture09/emptiness_sk09/ +http://www.mdf.ru/contests/silvercam/silvercam2009/sk_events09/riots_sk09/ +http://www.mdf.ru/contests/silvercam/silvercam2009/sk_faces09/herodayzm_sk09/ +http://www.silvercam.ru/ +http://www.mdf.ru/contests/silvercam/silvercam2005/press144.html +http://www.museum.ru/N32915 +https://web.archive.org/web/20070701141757/http://foto.potrebitel.ru/data/4/26/p174serebr.shtml +http://www.businesspravo.ru/Docum/DocumShow_DocumID_99954.html +http://www.silvercamera.ru/blogs/ +https://www.imdb.com/name/nm0813961/ +http://www.elkesommeronline.com/en/biography.htm +https://www.filmportal.de/ecd3a6af1b9a4304852c1d52c9853abe +https://www.fembio.org/biographie.php/frau/frauendatenbank?fem_id=25641 +https://www.munzinger.de/search/go/document.jsp?id=00000011630 +http://www.elkesommeronline.com/en/biography.htm +https://web.archive.org/web/20161005032346/http://www.elkesommeronline.com/en/biography.htm +http://articles.latimes.com/1993-12-09/local/me-50_1_gabor-sommer-feud +http://www.elkesommeronline.com/en/biography7.htm +http://www.elkesommeronline.com/en/biography11.htm +http://www.helia-d.us/fansreviews.html +http://web.archive.org/web/20200804224920/http://www.helia-d.us/fansreviews.html +https://music.yandex.ru/artist/473608 +https://www.allmovie.com/artist/p66862 +https://www.allocine.fr/personne/fichepersonne_gen_cpersonne=24633.html +https://www.csfd.cz/tvurce/6487 +https://www.dfi.dk/viden-om-film/filmdatabasen/person/114484 +https://www.discogs.com/artist/647316 +https://www.filmportal.de/ecd3a6af1b9a4304852c1d52c9853abe +https://www.imdb.com/name/nm0813961 +https://www.kinopoisk.ru/name/119809/ +https://musicbrainz.org/artist/638dbe3c-198e-4def-ab40-4fd63de92aad +https://rkd.nl/nl/explore/artists/328834 +https://www.britannica.com/biography/Elke-Sommer +https://nndb.com/people/239/000024167 +http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX1546865 +https://catalogue.bnf.fr/ark:/12148/cb140392273 +https://d-nb.info/gnd/118913085 +http://data.beeldengeluid.nl/gtaa/156101 +http://isni-url.oclc.nl/isni/000000010912768X +https://id.loc.gov/authorities/n84123407 +https://data.bibliotheken.nl/id/thes/p073476285 +https://viaf.org/processed/NUKAT%7Cn2009046388 +https://www.idref.fr/070758085 +https://viaf.org/viaf/69130469 +https://www.worldcat.org/identities/containsVIAFID/69130469 +https://research.frick.org/directory/detail/2659 +https://www.nytimes.com/2001/09/01/arts/ethel-scull-a-patron-of-pop-and-minimal-art-dies-at-79.html +https://www.worldcat.org/issn/0362-4331 +https://www.worldcat.org/issn/1553-8095 +https://www.worldcat.org/issn/1542-667X +https://www.nytimes.com/2001/09/01/arts/ethel-scull-a-patron-of-pop-and-minimal-art-dies-at-79.html?_r=0 +http://www.metmuseum.org/collections/search-the-collections/210010373 +http://www.artnews.com/2010/07/01/the-five-year-plan/ +http://www.artinamericamagazine.com/reviews/the-scull-collection +http://www.highbeam.com/doc/1G1-197049240.html +https://web.archive.org/web/20140611030857/http://www.highbeam.com/doc/1G1-197049240.html +https://www.nytimes.com/2001/09/01/arts/ethel-scull-a-patron-of-pop-and-minimal-art-dies-at-79.html +http://www.rfi.fr/contenu/20091103-warhol-couleur-argente +http://jmapps.ne.jp/apmoa2/det.html?data_id=6322 +https://www.findagrave.com/cgi-bin/fg.cgi?page=gr&GRid=5794150 +http://isni-url.oclc.nl/isni/0000000020792528 +https://viaf.org/viaf/25741873 +https://www.worldcat.org/identities/containsVIAFID/25741873 diff --git a/sync_link_test.py b/sync_link_test.py new file mode 100644 index 0000000..e69de29 diff --git a/sync_token_generator_test.py b/sync_token_generator_test.py new file mode 100644 index 0000000..e69de29 diff --git a/tokens.py b/tokens.py new file mode 100644 index 0000000..e69de29 diff --git a/tpe_link_test.py b/tpe_link_test.py new file mode 100644 index 0000000..e69de29 From ed3719100d14c1beb7a7beaeb8372cec923c18ad Mon Sep 17 00:00:00 2001 From: Aripov Sergey Date: Fri, 3 Dec 2021 18:06:56 +0500 Subject: [PATCH 2/3] =?UTF-8?q?=D0=9D=D0=B5=D0=B4=D0=BE=D1=81=D1=82=D0=B0?= =?UTF-8?q?=D1=8E=D1=89=D0=B8=D0=B5=20=D1=84=D0=B0=D0=B9=D0=BB=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LinkTestsResults.md | 19 ++++++++++++++ TokensTestsResult.md | 33 ++++++++++++++++++------- links_utils.py | 5 +++- main.py | 48 ++++++++++++++++++++++++++++++++++++ ppe_token_generator_test.py | 12 +++++++++ sync_link_test.py | 16 ++++++++++++ sync_token_generator_test.py | 5 ++++ tokens.py | 12 +++++++++ tpe_link_test.py | 19 ++++++++++++++ 9 files changed, 159 insertions(+), 10 deletions(-) diff --git a/LinkTestsResults.md b/LinkTestsResults.md index e69de29..7d1f39d 100644 --- a/LinkTestsResults.md +++ b/LinkTestsResults.md @@ -0,0 +1,19 @@ +# Синхронный тест +Время: 145153211000 ns + +Загрузка cpu: <1% в среднем, не больше 2% + +# ThreadPoolExecutor - 5 +Среднее время (5 повторов): 34274738620 ns + +Загрузка cpu: >1% в среднем, не больше 5% + +# ThreadPoolExecutor - 10 +Среднее время (5 повторов): 21303062040 ns + +Загрузка cpu: >2.5% в среднем, не больше 5% + +# ThreadPoolExecutor - 100 +Среднее время (5 повторов): 6163196780 ns + +Загрузка cpu: >6% в среднем, не больше 15% \ No newline at end of file diff --git a/TokensTestsResult.md b/TokensTestsResult.md index 7d1f39d..e134662 100644 --- a/TokensTestsResult.md +++ b/TokensTestsResult.md @@ -1,19 +1,34 @@ +Генерирование 10 монеток + # Синхронный тест -Время: 145153211000 ns +Время: 159032290900 ns + +Загрузка cpu: ~24% + +# ThreadPoolExecutor - 2 +Среднее время (5 повторов): 107437799580 ns -Загрузка cpu: <1% в среднем, не больше 2% +Загрузка cpu: ~45% + +# ThreadPoolExecutor - 4 +Среднее время (5 повторов): 95689853860 ns + +Загрузка cpu: ~75% # ThreadPoolExecutor - 5 -Среднее время (5 повторов): 34274738620 ns +Среднее время (5 повторов): 94443180060 ns -Загрузка cpu: >1% в среднем, не больше 5% +Загрузка cpu: до 90% # ThreadPoolExecutor - 10 -Среднее время (5 повторов): 21303062040 ns +Среднее время (5 повторов): 97867832680 ns + +Загрузка cpu: до 100% -Загрузка cpu: >2.5% в среднем, не больше 5% +# ThreadPoolExecutor - 61 +Среднее время (5 повторов): 103766266200 ns -# ThreadPoolExecutor - 100 -Среднее время (5 повторов): 6163196780 ns +Загрузка cpu: до 100% -Загрузка cpu: >6% в среднем, не больше 15% \ No newline at end of file +--- +По мере нахождения монеток процессом загрузка процессора уменьшалась +- пропорционально количеству процессов \ No newline at end of file diff --git a/links_utils.py b/links_utils.py index bea6ce6..4c0e07d 100644 --- a/links_utils.py +++ b/links_utils.py @@ -7,8 +7,10 @@ if not os.path.exists('res.txt'): res = open('res.txt', 'w', encoding='utf8') - + links_count = 0 for i in tqdm(range(100)): + if links_count>100: + continue html = urlopen(url).read().decode('utf8') soup = BeautifulSoup(html, 'html.parser') links = soup.find_all('a') @@ -16,6 +18,7 @@ for l in links: href = l.get('href') if href and href.startswith('http') and 'wiki' not in href: + links_count += 1 print(href, file=res) res.close() diff --git a/main.py b/main.py index e69de29..0648b6f 100644 --- a/main.py +++ b/main.py @@ -0,0 +1,48 @@ +import time + +import ppe_token_generator_test +import sync_link_test, tpe_link_test +import sync_token_generator_test + + +def perform_test(test, repeats, *arg): + start_time = time.perf_counter_ns() + for i in range(repeats): + test(*arg) + end_time = time.perf_counter_ns() + return (end_time-start_time)/repeats + + +def links_tests(): + print(f'Sync time {perform_test(sync_link_test.test, 1)}') + + workers = 5 + print(f'TPE with {workers} max_workers time {perform_test(tpe_link_test.test, 5, workers)}') + + workers = 10 + print(f'TPE with {workers} max_workers time {perform_test(tpe_link_test.test, 5, workers)}') + + workers = 100 + print(f'TPE with {workers} max_workers time {perform_test(tpe_link_test.test, 5, workers)}') + + +def tokens_tests(): + #print(f'Sync time {perform_test(sync_token_generator_test.test, 1, 10)}') + + workers = 2 + #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + + workers = 4 + #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + + workers = 5 + #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + + workers = 10 + #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + + workers = 61 + print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 1, 10, workers)}') + +if __name__ == '__main__': + tokens_tests() diff --git a/ppe_token_generator_test.py b/ppe_token_generator_test.py index e69de29..769aa95 100644 --- a/ppe_token_generator_test.py +++ b/ppe_token_generator_test.py @@ -0,0 +1,12 @@ +import concurrent.futures + +import tokens + + +def test(tokens_num, max_workers): + result = [] + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(tokens.get_token) for _ in range(0, tokens_num)] + for future in concurrent.futures.as_completed(futures): + result += future.result() + return result diff --git a/sync_link_test.py b/sync_link_test.py index e69de29..e4e6293 100644 --- a/sync_link_test.py +++ b/sync_link_test.py @@ -0,0 +1,16 @@ +from urllib.request import Request, urlopen + +import links_utils + +links = links_utils.get_links() + + +def test(): + results=[] + for url in links: + try: + results += links_utils.load_url(url, 2) + except Exception as e: + results += (url, e) + + return results diff --git a/sync_token_generator_test.py b/sync_token_generator_test.py index e69de29..3dcc8c3 100644 --- a/sync_token_generator_test.py +++ b/sync_token_generator_test.py @@ -0,0 +1,5 @@ +import tokens + + +def test(tokens_num): + return [tokens.get_token() for _ in range(0, tokens_num)] diff --git a/tokens.py b/tokens.py index e69de29..ee38714 100644 --- a/tokens.py +++ b/tokens.py @@ -0,0 +1,12 @@ +from hashlib import md5 +from random import choice + + +def get_token(): + while True: + s = "".join([choice("0123456789") for i in range(50)]) + h = md5(s.encode('utf8')).hexdigest() + + if h.endswith("00000"): + return s, h + \ No newline at end of file diff --git a/tpe_link_test.py b/tpe_link_test.py index e69de29..ed58ed5 100644 --- a/tpe_link_test.py +++ b/tpe_link_test.py @@ -0,0 +1,19 @@ +import concurrent.futures + +import links_utils + +links = links_utils.get_links() + + +def test(max_workers): + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + # Start the load operations and mark each future with its URL + future_to_url = {executor.submit(links_utils.load_url, url, 2): url for url in links} + for future in concurrent.futures.as_completed(future_to_url): + url = future_to_url[future] + try: + results += future.result() + except Exception as e: + results += (url, e) + return results From ebb84131f6bc0c9dee09183d203d4c55a89fa40e Mon Sep 17 00:00:00 2001 From: Aripov Sergey Date: Fri, 3 Dec 2021 18:08:37 +0500 Subject: [PATCH 3/3] =?UTF-8?q?=D0=A4=D0=B8=D0=BA=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 0648b6f..555c774 100644 --- a/main.py +++ b/main.py @@ -27,22 +27,23 @@ def links_tests(): def tokens_tests(): - #print(f'Sync time {perform_test(sync_token_generator_test.test, 1, 10)}') + print(f'Sync time {perform_test(sync_token_generator_test.test, 1, 10)}') workers = 2 - #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') workers = 4 - #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') workers = 5 - #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') workers = 10 - #print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') + print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') workers = 61 - print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 1, 10, workers)}') + print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}') if __name__ == '__main__': + links_tests() tokens_tests()