Skip to content

Commit

Permalink
separate crawler and analysis
Browse files Browse the repository at this point in the history
  • Loading branch information
@SebastianElvis
SebastianElvis committed Jan 17, 2023
1 parent e483c03 commit 84f238d
Show file tree
Hide file tree
Showing 4 changed files with 904 additions and 419 deletions.
214 changes: 214 additions & 0 deletions analysis.ipynb
View file

Large diffs are not rendered by default.

260 changes: 260 additions & 0 deletions crawler.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from python_graphql_client import GraphqlClient\n",
"import json\n",
"import networkx as nx\n",
"import matplotlib.pyplot as plt\n",
"import math\n",
"import numpy"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'data': {'blockchain': [{'name': 'Stride', '__typename': 'flat_blockchains'}]}}\n"
]
}
],
"source": [
"endpoint = 'https://api2.mapofzones.com/v1/graphql'\n",
"\n",
"# Instantiate the client with an endpoint.\n",
"client = GraphqlClient(endpoint=endpoint)\n",
"\n",
"# Create the query string and variables required for the request.\n",
"query = \"\"\"\n",
" query ZoneName($zone: String!) {\n",
" blockchain: flat_blockchains(where: {network_id: {_eq: $zone}}) {\n",
" name\n",
" __typename\n",
" }\n",
" }\n",
"\"\"\"\n",
"\n",
"variables = {\"zone\":\"stride-1\"}\n",
"\n",
"# Synchronous request\n",
"data = client.execute(query=query, variables=variables)\n",
"print(data) "
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"query = \"\"\"\n",
"query AseetsTable {\n",
" assets: flat_tokens {\n",
" blockchain: blockchainByBlockchain {\n",
" name\n",
" __typename\n",
" }\n",
" symbol\n",
" logoUrl: logo_url\n",
" price\n",
" price24hDiffPercent: price_day_diff_percent\n",
" price7dDiffPercent: price_week_diff_percent\n",
" marketCap: market_cap\n",
" volume24h: token_day_trading_volume\n",
" volume24hDiffPercent: token_day_trading_volume_diff_percent\n",
" onChainSupply: on_chain_supply\n",
" priceChart: token_charts(\n",
" where: {chart_type: {_eq: \"price_weekly\"}}\n",
" order_by: {point_index: asc}\n",
" ) {\n",
" price: point_value\n",
" __typename\n",
" }\n",
" __typename\n",
" }\n",
"}\n",
"\"\"\"\n",
"\n",
"variables = {}\n",
"\n",
"data = client.execute(query=query, variables=variables)\n",
"with open('data/marketcap.json', 'w') as f:\n",
" json.dump(data, f)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"query = \"\"\"\n",
"query ZonesMap($period: Int!, $isMainnet: Boolean!) {\n",
" zonesStats: flat_blockchains(where: {is_mainnet: {_eq: $isMainnet}}) {\n",
" ...ZoneBaseInfoV2\n",
" isMainnet: is_mainnet\n",
" switchedStats: blockchain_switched_stats(\n",
" where: {timeframe: {_eq: $period}, is_mainnet: {_eq: $isMainnet}}\n",
" ) {\n",
" ibcVolume: ibc_cashflow\n",
" ibcVolumeIn: ibc_cashflow_in\n",
" ibcVolumeOut: ibc_cashflow_out\n",
" ibcVolumeRating: ibc_cashflow_rating\n",
" ibcTransfersRating: ibc_transfers_rating\n",
" dauRating: active_addresses_cnt_rating\n",
" totalTxsRating: txs_rating\n",
" __typename\n",
" }\n",
" __typename\n",
" }\n",
" zonesGraphs: flat_blockchain_relations(\n",
" where: {blockchain: {is_mainnet: {_eq: $isMainnet}}, blockchainByBlockchainSource: {is_mainnet: {_eq: $isMainnet}}, timeframe: {_eq: $period}}\n",
" ) {\n",
" source: blockchain_source\n",
" target: blockchain_target\n",
" ibcVolume: ibc_cashflow\n",
" __typename\n",
" }\n",
"}\n",
"\n",
"fragment ZoneBaseInfoV2 on flat_blockchains {\n",
" zone: network_id\n",
" logoUrl: logo_url\n",
" name: name\n",
" __typename\n",
"}\n",
"\"\"\"\n",
"\n",
"variables = {\"period\": 24, \"isMainnet\": True}\n",
"data = client.execute(query=query, variables=variables)\n",
"with open('data/zonemap.json', 'w') as f:\n",
" json.dump(data, f)"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [],
"source": [
"mapdata = dict()\n",
"\n",
"with open('data/zonemap.json', 'r') as f:\n",
" mapdata = json.load(f)\n",
"\n",
"zones = mapdata['data']['zonesStats']\n",
"zones_map = {zone['zone']: zone['name'] for zone in zones}\n",
"channels = mapdata['data']['zonesGraphs']"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"query = \"\"\"\n",
"query ZonesListZonePeers($source: String!, $period: Int!, $isMainnet: Boolean!) {\n",
" zones: flat_blockchains(\n",
" where: {is_mainnet: {_eq: $isMainnet}, network_id: {_neq: $source}, channelsStatsByCounterpartyBlockchain: {blockchain: {_eq: $source}}}\n",
" ) {\n",
" data: channelsStatsByCounterpartyBlockchain_aggregate(\n",
" where: {blockchain: {_eq: $source}, timeframe: {_eq: $period}}\n",
" ) {\n",
" aggregate {\n",
" sum {\n",
" ibcTransfers: ibc_transfers\n",
" ibcTransfersPending: ibc_transfers_pending\n",
" ibcTransfersFailed: ibc_transfers_failed\n",
" ibcVolume: ibc_cashflow\n",
" ibcVolumePending: ibc_cashflow_pending\n",
" ibcVolumeIn: ibc_cashflow_in\n",
" ibcVolumeInPending: ibc_cashflow_in_pending\n",
" ibcVolumeOut: ibc_cashflow_out\n",
" ibcVolumeOutPending: ibc_cashflow_out_pending\n",
" __typename\n",
" }\n",
" __typename\n",
" }\n",
" zoneChannels: nodes {\n",
" zoneCounterparty: blockchainByCounterpartyBlockchain {\n",
" zone: network_id\n",
" name\n",
" logoUrl: logo_url\n",
" isUpToDate: is_synced\n",
" __typename\n",
" }\n",
" zone: blockchain\n",
" zoneCounterpartyChannelId: counterparty_channel_id\n",
" channelId: channel_id\n",
" clientId: client_id\n",
" connectionId: connection_id\n",
" isOpened: is_channel_open\n",
" ibcTransfers: ibc_transfers\n",
" ibcTransfersPending: ibc_transfers_pending\n",
" ibcTransfersFailed: ibc_transfers_failed\n",
" ibcTransfersSuccessRate: ibc_transfers_success_rate\n",
" ibcVolume: ibc_cashflow\n",
" ibcVolumePending: ibc_cashflow_pending\n",
" ibcVolumeIn: ibc_cashflow_in\n",
" ibcVolumeInPending: ibc_cashflow_in_pending\n",
" ibcVolumeOut: ibc_cashflow_out\n",
" ibcVolumeOutPending: ibc_cashflow_out_pending\n",
" __typename\n",
" }\n",
" __typename\n",
" }\n",
" __typename\n",
" }\n",
"}\n",
"\"\"\"\n",
"\n",
"for zone in zones:\n",
" variables = {\n",
" \"source\": zone['zone'],\n",
" \"period\": 24,\n",
" \"isMainnet\": True\n",
" }\n",
" data = client.execute(query=query, variables=variables)\n",
" with open('data/peers/%s.json' % zone['zone'], 'w') as f:\n",
" json.dump(data, f)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.9 (default, Apr 13 2022, 08:48:07) \n[Clang 13.1.6 (clang-1316.0.21.2.5)]"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Loading

0 comments on commit 84f238d

Please sign in to comment.