Арипов Сергей, АТ-13

LinkTestsResults.md

@@ -0,0 +1,19 @@
+# Синхронный тест
+Время: 145153211000 ns
+
+Загрузка cpu: <1% в среднем, не больше 2%
+
+# ThreadPoolExecutor - 5
+Среднее время (5 повторов): 34274738620 ns
+
+Загрузка cpu: >1% в среднем, не больше 5%
+
+# ThreadPoolExecutor - 10
+Среднее время (5 повторов): 21303062040 ns
+
+Загрузка cpu: >2.5% в среднем, не больше 5%
+
+# ThreadPoolExecutor - 100
+Среднее время (5 повторов): 6163196780 ns
+
+Загрузка cpu: >6% в среднем, не больше 15%

TokensTestsResult.md

@@ -0,0 +1,34 @@
+Генерирование 10 монеток
+
+# Синхронный тест
+Время: 159032290900 ns
+
+Загрузка cpu: ~24%
+
+# ThreadPoolExecutor - 2
+Среднее время (5 повторов): 107437799580 ns
+
+Загрузка cpu: ~45%
+
+# ThreadPoolExecutor - 4
+Среднее время (5 повторов): 95689853860 ns
+
+Загрузка cpu: ~75%
+
+# ThreadPoolExecutor - 5
+Среднее время (5 повторов): 94443180060 ns
+
+Загрузка cpu: до 90%
+
+# ThreadPoolExecutor - 10
+Среднее время (5 повторов): 97867832680 ns
+
+Загрузка cpu: до 100%
+
+# ThreadPoolExecutor - 61
+Среднее время (5 повторов): 103766266200 ns
+
+Загрузка cpu: до 100%
+
+---
+По мере нахождения монеток процессом загрузка процессора уменьшалась +- пропорционально количеству процессов

links_utils.py

@@ -0,0 +1,39 @@
+import os
+from urllib.request import Request, urlopen
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+
+url = 'https://ru.wikipedia.org/wiki/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%BB%D1%83%D1%87%D0%B0%D0%B9%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0'
+
+if not os.path.exists('res.txt'):
+    res = open('res.txt', 'w', encoding='utf8')
+    links_count = 0
+    for i in tqdm(range(100)):
+        if links_count>100:
+            continue
+        html = urlopen(url).read().decode('utf8')
+        soup = BeautifulSoup(html, 'html.parser')
+        links = soup.find_all('a')
+
+        for l in links:
+            href = l.get('href')
+            if href and href.startswith('http') and 'wiki' not in href:
+                links_count += 1
+                print(href, file=res)
+
+    res.close()
+
+
+def get_links():
+    with open('res.txt', 'r', encoding='utf-8') as f:
+        return f.read().split('\n')
+
+
+def load_url(url, timeout):
+    request = Request(
+            url,
+            headers={
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 9.0; Win65; x64; rv:97.0) Gecko/20105107 Firefox/92.0'},
+            )
+    resp = urlopen(request, timeout=timeout)
+    return resp.code

main.py

@@ -0,0 +1,49 @@
+import time
+
+import ppe_token_generator_test
+import sync_link_test, tpe_link_test
+import sync_token_generator_test
+
+
+def perform_test(test, repeats, *arg):
+    start_time = time.perf_counter_ns()
+    for i in range(repeats):
+        test(*arg)
+    end_time = time.perf_counter_ns()
+    return (end_time-start_time)/repeats
+
+
+def links_tests():
+    print(f'Sync time {perform_test(sync_link_test.test, 1)}')
+
+    workers = 5
+    print(f'TPE with {workers} max_workers time {perform_test(tpe_link_test.test, 5, workers)}')
+
+    workers = 10
+    print(f'TPE with {workers} max_workers time {perform_test(tpe_link_test.test, 5, workers)}')
+
+    workers = 100
+    print(f'TPE with {workers} max_workers time {perform_test(tpe_link_test.test, 5, workers)}')
+
+
+def tokens_tests():
+    print(f'Sync time {perform_test(sync_token_generator_test.test, 1, 10)}')
+
+    workers = 2
+    print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}')
+
+    workers = 4
+    print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}')
+
+    workers = 5
+    print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}')
+
+    workers = 10
+    print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}')
+
+    workers = 61
+    print(f'TPE with {workers} max_workers time {perform_test(ppe_token_generator_test.test, 5, 10, workers)}')
+
+if __name__ == '__main__':
+    links_tests()
+    tokens_tests()

ppe_token_generator_test.py

@@ -0,0 +1,12 @@
+import concurrent.futures
+
+import tokens
+
+
+def test(tokens_num, max_workers):
+    result = []
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(tokens.get_token) for _ in range(0, tokens_num)]
+        for future in concurrent.futures.as_completed(futures):
+            result += future.result()
+    return result

res.txt

@@ -0,0 +1,104 @@
+https://www.openstreetmap.org/?mlat=44.0589&mlon=5.2342&zoom=12
+https://www.openstreetmap.org/?mlat=44.0589&mlon=5.2342&zoom=12
+https://www.insee.fr/fr/statistiques/2011101?geo=COM-84148
+http://www.villes-sur-auzon.fr
+http://www.insee.fr/fr/themes/tableau_local.asp?ref_id=POP&millesime=2010&nivgeo=COM&codgeo=84148
+http://www.insee.fr/fr/methodes/nomenclatures/cog/fichecommunale.asp?codedep=84&codecom=148&codecan=15
+https://www.openstreetmap.org/?mlat=34.94972&mlon=-90.47083&zoom=12
+https://www.openstreetmap.org/?mlat=34.94972&mlon=-90.47083&zoom=12
+https://edits.nationalmap.gov/apps/gaz-domestic/public/summary/83178
+https://edits.nationalmap.gov/apps/gaz-domestic/public/summary/2404739
+http://factfinder.census.gov
+https://www.openstreetmap.org/?mlat=42.16667&mlon=26.53333&zoom=12
+https://www.openstreetmap.org/?mlat=42.16667&mlon=26.53333&zoom=12
+http://grao.bg/tna/tab02.txt
+https://www.transfermarkt.com/transfermarkt/profil/spieler/223974
+https://fbref.com/en/players/7f609bfc/
+https://int.soccerway.com/matches/2017/09/28/europe/uefa-cup/fk-razgrad-2000/tsg-1899-hoffenheim-ev/2580369/
+https://int.soccerway.com/matches/2017/10/14/germany/bundesliga/tsg-1899-hoffenheim-ev/fc-augsburg/2480328/
+http://int.soccerway.com/matches/2016/07/11/europe/uefa-u19-championship/portugal-under-19/austria-under-19/2229976/
+http://int.soccerway.com/matches/2016/07/14/europe/uefa-u19-championship/italy-under-19/austria-under-19/2229978/
+http://int.soccerway.com/matches/2016/07/17/europe/uefa-u19-championship/austria-under-19/germany-under-19/2229980/
+https://int.soccerway.com/matches/2019/06/10/europe/european-championship-qualification/macedonia-fyr/austria/2942836/
+https://int.soccerway.com/matches/2019/06/17/europe/uefa-u21-championship/serbia-u21/austria-under-21/2940777/
+https://int.soccerway.com/matches/2019/06/20/europe/uefa-u21-championship/denmark-under-21/austria-under-21/2940779/
+https://int.soccerway.com/matches/2019/06/23/europe/uefa-u21-championship/austria-under-21/germany-under-21/2940780/
+https://www.national-football-teams.com/player/74577.html
+https://www.sports.ru/tags/161060597/
+https://www.weltfussball.de/spieler_profil/stefan-posch/
+https://www.kicker.de/profil-90213/spieler
+https://www.fussballdaten.de/person/stefan-posch/
+https://www.dfb.de/datencenter/personen/stefan-posch/spieler
+https://eu-football.info/_player.php?id=30676
+https://fbref.com/en/players/7f609bfc/
+https://www.kicker.de/stefan-posch/spieler
+https://www.national-football-teams.com/player/74577.html
+https://int.soccerway.com/players/stefan-posch/316755/
+https://www.transfermarkt.com/transfermarkt/profil/spieler/223974
+https://www.worldfootball.net/player_summary/stefan-posch/
+http://kanashchuk.livejournal.com
+http://amateurphotographer.ru/?p=5287
+http://lenta.ru/news/2011/03/02/prize/
+http://www.mdf.ru/contests/silvercam/silvercam2009/sk_architecture09/emptiness_sk09/
+http://www.mdf.ru/contests/silvercam/silvercam2009/sk_events09/riots_sk09/
+http://www.mdf.ru/contests/silvercam/silvercam2009/sk_faces09/herodayzm_sk09/
+http://www.silvercam.ru/
+http://www.mdf.ru/contests/silvercam/silvercam2005/press144.html
+http://www.museum.ru/N32915
+https://web.archive.org/web/20070701141757/http://foto.potrebitel.ru/data/4/26/p174serebr.shtml
+http://www.businesspravo.ru/Docum/DocumShow_DocumID_99954.html
+http://www.silvercamera.ru/blogs/
+https://www.imdb.com/name/nm0813961/
+http://www.elkesommeronline.com/en/biography.htm
+https://www.filmportal.de/ecd3a6af1b9a4304852c1d52c9853abe
+https://www.fembio.org/biographie.php/frau/frauendatenbank?fem_id=25641
+https://www.munzinger.de/search/go/document.jsp?id=00000011630
+http://www.elkesommeronline.com/en/biography.htm
+https://web.archive.org/web/20161005032346/http://www.elkesommeronline.com/en/biography.htm
+http://articles.latimes.com/1993-12-09/local/me-50_1_gabor-sommer-feud
+http://www.elkesommeronline.com/en/biography7.htm
+http://www.elkesommeronline.com/en/biography11.htm
+http://www.helia-d.us/fansreviews.html
+http://web.archive.org/web/20200804224920/http://www.helia-d.us/fansreviews.html
+https://music.yandex.ru/artist/473608
+https://www.allmovie.com/artist/p66862
+https://www.allocine.fr/personne/fichepersonne_gen_cpersonne=24633.html
+https://www.csfd.cz/tvurce/6487
+https://www.dfi.dk/viden-om-film/filmdatabasen/person/114484
+https://www.discogs.com/artist/647316
+https://www.filmportal.de/ecd3a6af1b9a4304852c1d52c9853abe
+https://www.imdb.com/name/nm0813961
+https://www.kinopoisk.ru/name/119809/
+https://musicbrainz.org/artist/638dbe3c-198e-4def-ab40-4fd63de92aad
+https://rkd.nl/nl/explore/artists/328834
+https://www.britannica.com/biography/Elke-Sommer
+https://nndb.com/people/239/000024167
+http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX1546865
+https://catalogue.bnf.fr/ark:/12148/cb140392273
+https://d-nb.info/gnd/118913085
+http://data.beeldengeluid.nl/gtaa/156101
+http://isni-url.oclc.nl/isni/000000010912768X
+https://id.loc.gov/authorities/n84123407
+https://data.bibliotheken.nl/id/thes/p073476285
+https://viaf.org/processed/NUKAT%7Cn2009046388
+https://www.idref.fr/070758085
+https://viaf.org/viaf/69130469
+https://www.worldcat.org/identities/containsVIAFID/69130469
+https://research.frick.org/directory/detail/2659
+https://www.nytimes.com/2001/09/01/arts/ethel-scull-a-patron-of-pop-and-minimal-art-dies-at-79.html
+https://www.worldcat.org/issn/0362-4331
+https://www.worldcat.org/issn/1553-8095
+https://www.worldcat.org/issn/1542-667X
+https://www.nytimes.com/2001/09/01/arts/ethel-scull-a-patron-of-pop-and-minimal-art-dies-at-79.html?_r=0
+http://www.metmuseum.org/collections/search-the-collections/210010373
+http://www.artnews.com/2010/07/01/the-five-year-plan/
+http://www.artinamericamagazine.com/reviews/the-scull-collection
+http://www.highbeam.com/doc/1G1-197049240.html
+https://web.archive.org/web/20140611030857/http://www.highbeam.com/doc/1G1-197049240.html
+https://www.nytimes.com/2001/09/01/arts/ethel-scull-a-patron-of-pop-and-minimal-art-dies-at-79.html
+http://www.rfi.fr/contenu/20091103-warhol-couleur-argente
+http://jmapps.ne.jp/apmoa2/det.html?data_id=6322
+https://www.findagrave.com/cgi-bin/fg.cgi?page=gr&GRid=5794150
+http://isni-url.oclc.nl/isni/0000000020792528
+https://viaf.org/viaf/25741873
+https://www.worldcat.org/identities/containsVIAFID/25741873

sync_link_test.py

@@ -0,0 +1,16 @@
+from urllib.request import Request, urlopen
+
+import links_utils
+
+links = links_utils.get_links()
+
+
+def test():
+    results=[]
+    for url in links:
+        try:
+            results += links_utils.load_url(url, 2)
+        except Exception as e:
+            results += (url, e)
+
+    return results

sync_token_generator_test.py

@@ -0,0 +1,5 @@
+import tokens
+
+
+def test(tokens_num):
+    return [tokens.get_token() for _ in range(0, tokens_num)]

tokens.py

@@ -0,0 +1,12 @@
+from hashlib import md5
+from random import choice
+
+
+def get_token():
+    while True:
+        s = "".join([choice("0123456789") for i in range(50)])
+        h = md5(s.encode('utf8')).hexdigest()
+
+        if h.endswith("00000"):
+            return s, h
+

tpe_link_test.py

@@ -0,0 +1,19 @@
+import concurrent.futures
+
+import links_utils
+
+links = links_utils.get_links()
+
+
+def test(max_workers):
+    results = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Start the load operations and mark each future with its URL
+        future_to_url = {executor.submit(links_utils.load_url, url, 2): url for url in links}
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                results += future.result()
+            except Exception as e:
+                results += (url, e)
+    return results