Skip to content

Commit

Permalink
fix: fix futures_inventory_99
Browse files Browse the repository at this point in the history
  • Loading branch information
@albertandking
albertandking committed Dec 23, 2024
1 parent a5e4040 commit 93fc8ac
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 432 deletions.
24 changes: 0 additions & 24 deletions akshare/futures/cons.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,30 +64,6 @@
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
}

sample_headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Host": "service.99qh.com",
"Origin": "http://service.99qh.com",
"Referer": "http://www.99qh.com/d/store.aspx",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
}

qh_headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Content-Length": "8429",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "service.99qh.com",
"Origin": "http://service.99qh.com",
"Pragma": "no-cache",
"Cookie": "__utma=181566328.985082941.1656754961.1656754961.1656754961.1; __utmc=181566328; __utmz=181566328.1656754961.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ASP.NET_SessionId=42k0mpzfu3fv5cxqmtrwc20y; tgw_l7_route=b26adbec28f4b4e1f7290033d59c43a7; __utmt=1; __utmb=181566328.2.10.1656754961",
"Referer": "http://service.99qh.com/Storage/Storage.aspx?page=99qh",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
}
# 奇货可查
QHKC_INDEX_URL = "https://www.qhkch.com/ajax/index_show.php"
QHKC_INDEX_TREND_URL = "https://qhkch.com/ajax/indexes_trend.php"
Expand Down
333 changes: 61 additions & 272 deletions akshare/futures/futures_inventory_99.py
View file
Original file line number Diff line number Diff line change
@@ -1,297 +1,86 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
Date: 2024/1/10 19:00
Date: 2024/12/23 20:00
Desc: 99 期货网-大宗商品库存数据
http://www.99qh.com/d/store.aspx
https://www.99qh.com/
"""
from io import StringIO

import json
from functools import lru_cache
from datetime import datetime
import pandas as pd
import requests
from bs4 import BeautifulSoup

from akshare.futures.cons import (
qh_headers,
sample_headers,
)

@lru_cache(maxsize=32)
def __get_99_symbol_map() -> pd.DataFrame:
"""
99 期货网-品种代码对照表
https://www.99qh.com/data/stockIn?productId=12
:return: 品种代码对照表
:rtype: pandas.DataFrame
"""
url = "https://www.99qh.com/data/stockIn"
r = requests.get(url)
soup = BeautifulSoup(r.text, features="lxml")
raw_data = soup.find(attrs={"id": "__NEXT_DATA__"}).text
data_json = json.loads(raw_data)
df_list = []
for i, item in enumerate(
data_json["props"]["pageProps"]["data"]["varietyListData"]
):
temp_df = pd.DataFrame(
data_json["props"]["pageProps"]["data"]["varietyListData"][i]["productList"]
)
df_list.append(temp_df)

big_df = pd.concat(df_list, ignore_index=True)
return big_df


def futures_inventory_99(
exchange: str = "大连商品交易所", symbol: str = "豆一"
) -> pd.DataFrame:
def futures_inventory_99(symbol: str = "豆一") -> pd.DataFrame:
"""
99 期货网-大宗商品库存数据
http://www.99qh.com/d/store.aspx
:param exchange: 交易所名称; choice of {"上海期货交易所", "郑州商品交易所", "大连商品交易所", "LME", "NYMEX", "CBOT", "NYBOT", "TOCOM", "上海国际能源交易中心", "OSE"}
:type exchange: str
https://www.99qh.com/data/stockIn?productId=12
:param symbol: 交易所对应的具体品种; 如：大连商品交易所的 豆一
:type symbol: str
:return: 大宗商品库存数据
:rtype: pandas.DataFrame
"""
data_code = {
"1": [
"1",
"2",
"3",
"12",
"32",
"36",
"37",
"40",
"42",
"47",
"56",
"63",
"69",
"70",
"79",
"85",
],
"2": [
"4",
"14",
"29",
"31",
"33",
"38",
"44",
"45",
"50",
"51",
"52",
"55",
"59",
"64",
"66",
"67",
"75",
"76",
"81",
"82",
"87",
"92",
"95",
],
"3": [
"6",
"7",
"8",
"15",
"30",
"34",
"35",
"39",
"43",
"53",
"57",
"58",
"61",
"62",
"68",
"80",
"84",
"86",
"88",
"89",
"94",
],
"4": ["9", "10", "16", "17", "18", "23", "28"],
"5": ["11", "20", "21"],
"6": ["13", "24", "25", "26", "27"],
"7": ["19"],
"8": ["22"],
"10": ["78", "83", "90", "93"],
"11": ["91"],
}
data_name = {
"1": [
"铜",
"铝",
"橡胶",
"燃料油",
"锌",
"黄金",
"螺纹钢",
"线材",
"铅",
"白银",
"石油沥青",
"热轧卷板",
"锡",
"镍",
"纸浆",
"不锈钢",
],
"2": [
"强麦",
"一号棉",
"白糖",
"PTA",
"菜籽油",
"早籼稻",
"甲醇",
"普麦",
"玻璃",
"油菜籽",
"菜籽粕",
"动力煤",
"粳稻",
"晚籼稻",
"硅铁",
"锰硅",
"棉纱",
"苹果",
"红枣",
"尿素",
"纯碱",
"短纤",
"花生",
],
"3": [
"豆一",
"豆二",
"豆粕",
"玉米",
"豆油",
"聚乙烯",
"棕榈油",
"聚氯乙烯",
"焦炭",
"焦煤",
"铁矿石",
"鸡蛋",
"胶合板",
"聚丙烯",
"玉米淀粉",
"乙二醇",
"粳米",
"苯乙烯",
"纤维板",
"液化石油气",
"生猪",
],
"4": ["LME铜", "LME铝", "LME镍", "LME铅", "LME锌", "LME锡", "LME铝合金"],
"5": ["COMEX铜", "COMEX金", "COMEX银"],
"6": ["CBOT大豆", "CBOT小麦", "CBOT玉米", "CBOT燕麦", "CBOT糙米"],
"7": ["NYBOT2号棉"],
"8": ["TOCOM橡胶"],
"10": ["原油", "20号胶", "低硫燃料油", "国际铜"],
"11": ["OSE橡胶"],
}
temp_out_exchange_name = {
"1": "上海期货交易所",
"2": "郑州商品交易所",
"3": "大连商品交易所",
"4": "LME",
"5": "NYMEX",
"6": "CBOT",
"7": "NYBOT",
"8": "TOCOM",
"10": "上海国际能源交易中心",
"11": "OSE",
}
exchange_map = {
value: key for key, value in temp_out_exchange_name.items()
temp_df = __get_99_symbol_map()
symbol_map = dict(zip(temp_df["name"], temp_df["productId"]))

url = "https://centerapi.fx168api.com/app/qh/api/stock/trend"
headers = {
"Content-Type": "application/json;charset=UTF-8",
"_pcc": "SGkj5avwu2h8Rs8/41r2LUwDHeEbaMKWe06+hWcEOO/uAQVbckWBHbwAvFbEI1eBBSvmTNqyjHKfFAn/kCpZ"
"IU7QNDvTrL2xGkQyuu+EVMU6RnZb/drmVGJRR6VhoHYMmzJvDuR6d43LnY219r44mGeL5x8qSUdh+cHjs0dm0AI=",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/58.0.3029.110 Safari/537.3",
"referer": "https://www.99qh.com",
}
exchange = exchange_map[exchange]
temp_symbol_code_map = dict(zip(data_name[exchange], data_code[exchange]))
symbol = temp_symbol_code_map[symbol]
out_exchange_name = {
"1": "上海期货交易所",
"2": "郑州商品交易所",
"3": "大连商品交易所",
"4": "LME",
"5": "NYMEX",
"6": "CBOT",
"7": "NYBOT",
"8": "TOCOM",
"10": "上海国际能源交易中心",
"11": "OSE",
params = {
"productId": symbol_map[symbol],
"type": "1",
"pageNo": "1",
"pageSize": "4000",
"startDate": "",
"endDate": f"{datetime.now().date().isoformat()}",
"appCategory": "web",
}
name_temp_dict = {}
code_temp_dict = {}
for num in data_code.keys():
name_temp_dict[out_exchange_name[num]] = dict(
zip(data_code[num], data_name[num])
)
code_temp_dict[num] = dict(zip(data_code[num], data_name[num]))
n = 10
while n != 0:
try:
n -= 1
session = requests.Session()
url = "http://service.99qh.com/Storage/Storage.aspx"
params = {"page": "99qh"}
r = session.post(url, params=params, headers=sample_headers)
cookie = r.cookies.get_dict()
url = "http://service.99qh.com/Storage/Storage.aspx"
params = {"page": "99qh"}
r = requests.post(
url, params=params, headers=sample_headers, cookies=cookie
)
soup = BeautifulSoup(r.text, "lxml")
view_state = soup.find_all(attrs={"id": "__VIEWSTATE"})[0]["value"]
even_validation = soup.find_all(attrs={"id": "__EVENTVALIDATION"})[
0
]["value"]
payload = {
"__EVENTTARGET": "ddlExchName",
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"__VIEWSTATE": view_state,
"__VIEWSTATEGENERATOR": "6EAC22FA",
"__EVENTVALIDATION": even_validation,
"ddlExchName": int(exchange),
"ddlGoodsName": 1,
}
res = requests.post(
url,
params={"page": "99qh"},
data=payload,
headers=qh_headers,
cookies=cookie,
)
soup = BeautifulSoup(res.text, "lxml")
view_state = soup.find_all(attrs={"id": "__VIEWSTATE"})[0]["value"]
even_validation = soup.find_all(attrs={"id": "__EVENTVALIDATION"})[
0
]["value"]
payload = {
"__EVENTTARGET": "ddlGoodsName",
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"__VIEWSTATE": view_state,
"__VIEWSTATEGENERATOR": "6EAC22FA",
"__EVENTVALIDATION": even_validation,
"ddlExchName": int(exchange),
"ddlGoodsName": int(symbol),
}
res = requests.post(
url,
params=params,
data=payload,
headers=qh_headers,
cookies=cookie,
)
data_df = pd.read_html(StringIO(res.text))[-1].T
data_df.columns = data_df.iloc[0, :]
data_df = data_df.iloc[1:, :]
data_df.reset_index(inplace=True, drop=True)
data_df.columns.name = None
data_df["日期"] = pd.to_datetime(data_df["日期"]).dt.date
data_df["库存"] = pd.to_numeric(data_df["库存"])
data_df["增减"] = pd.to_numeric(data_df["增减"])
data_df.sort_values("日期", inplace=True)
data_df.reset_index(inplace=True, drop=True)
return data_df
except:
continue
r = requests.get(url, params, headers=headers)
data_json = r.json()
temp_df = pd.DataFrame(data_json["data"]["list"])
temp_df.columns = ["日期", "收盘价", "库存"]
temp_df.sort_values(by=["日期"], ignore_index=True, inplace=True)
temp_df["日期"] = pd.to_datetime(temp_df["日期"], errors="coerce").dt.date
temp_df["收盘价"] = pd.to_numeric(temp_df["收盘价"], errors="coerce")
temp_df["库存"] = pd.to_numeric(temp_df["库存"], errors="coerce")
return temp_df


if __name__ == "__main__":
futures_inventory_99_df = futures_inventory_99(
exchange="郑州商品交易所", symbol="菜籽油"
)
futures_inventory_99_df = futures_inventory_99(symbol="豆一")
print(futures_inventory_99_df)
Loading

0 comments on commit 93fc8ac

Please sign in to comment.