diff --git a/pom.xml b/pom.xml index 6911d946..6ce314d4 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.github.kingschan1204 istock jar - 1.5.7 + 1.6.3 pritice Maven Webapp http://maven.apache.org @@ -110,7 +110,22 @@ org.springframework.boot spring-boot-starter-data-mongodb - + + + org.springframework.boot + spring-boot-starter-cache + + + + net.sf.ehcache + ehcache + + + + org.quartz-scheduler + quartz + 2.3.0 + diff --git a/readme-res/stock-list.png b/readme-res/stock-list.png index d3bd7723..e9d3aea8 100644 Binary files a/readme-res/stock-list.png and b/readme-res/stock-list.png differ diff --git a/src/main/java/io.github.kingschan1204.istock/Application.java b/src/main/java/io.github.kingschan1204.istock/Application.java index e9291d5c..cdb5e6f8 100644 --- a/src/main/java/io.github.kingschan1204.istock/Application.java +++ b/src/main/java/io.github.kingschan1204.istock/Application.java @@ -1,12 +1,16 @@ package io.github.kingschan1204.istock; +import io.github.kingschan1204.istock.module.startup.InitQuartzTaskRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; @Controller +@EnableCaching @SpringBootApplication public class Application { @@ -16,6 +20,11 @@ public String index() { return "index"; } + @Bean + public InitQuartzTaskRunner startupRunner() { + return new InitQuartzTaskRunner(); + } + public static void main(String[] args) { SpringApplication.run(Application.class, args); } diff --git a/src/main/java/io.github.kingschan1204.istock/common/conf/MongoConfig.java b/src/main/java/io.github.kingschan1204.istock/common/conf/MongoConfig.java new file mode 100644 index 00000000..78ea6cf7 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/conf/MongoConfig.java @@ -0,0 +1,35 @@ +package io.github.kingschan1204.istock.common.conf; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.core.convert.*; +import org.springframework.data.mongodb.core.mapping.MongoMappingContext; + +/** + * 去掉mongo 保存_class字段 + * @author chenguoxiang + * @create 2018-07-09 16:54 + **/ +@Configuration +public class MongoConfig { + private static Logger log = LoggerFactory.getLogger(MongoConfig.class); + @Bean + public MappingMongoConverter mappingMongoConverter(MongoDbFactory factory, MongoMappingContext context, BeanFactory beanFactory) { + DbRefResolver dbRefResolver = new DefaultDbRefResolver(factory); + MappingMongoConverter mappingConverter = new MappingMongoConverter(dbRefResolver, context); + try { + mappingConverter.setCustomConversions(beanFactory.getBean(CustomConversions.class)); + } catch (Exception ex) { + ex.printStackTrace(); + log.error("{}",ex); + } + + // Don't save _class to mongo + mappingConverter.setTypeMapper(new DefaultMongoTypeMapper(null)); + return mappingConverter; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/common/conf/ScheduleConfig.java b/src/main/java/io.github.kingschan1204.istock/common/conf/ScheduleConfig.java deleted file mode 100644 index e6760e83..00000000 --- a/src/main/java/io.github.kingschan1204.istock/common/conf/ScheduleConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -package io.github.kingschan1204.istock.common.conf; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.scheduling.annotation.SchedulingConfigurer; -import org.springframework.scheduling.config.ScheduledTaskRegistrar; - -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -/** - * 并行执行（多任务同时） - * @author chenguoxiang - * @create 2018-03-30 10:24 - **/ -@Configuration -@EnableScheduling -public class ScheduleConfig implements SchedulingConfigurer { - @Override - public void configureTasks(ScheduledTaskRegistrar scheduledTaskRegistrar) { - scheduledTaskRegistrar.setScheduler(taskExecutor()); - } - - @Bean(destroyMethod = "shutdown") - public Executor taskExecutor() { - return Executors.newScheduledThreadPool(100); - } -} diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/cache/EhcacheUtil.java b/src/main/java/io.github.kingschan1204.istock/common/util/cache/EhcacheUtil.java new file mode 100644 index 00000000..30f5ccf2 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/util/cache/EhcacheUtil.java @@ -0,0 +1,54 @@ +package io.github.kingschan1204.istock.common.util.cache; + +import net.sf.ehcache.Cache; +import net.sf.ehcache.Element; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.ehcache.EhCacheCacheManager; +import org.springframework.stereotype.Component; + +/** + * + * @author chenguoxiang + * @create 2018-07-10 15:02 + **/ +@Component +public class EhcacheUtil { + + @Autowired + EhCacheCacheManager ehCacheCacheManager; + + /** + * 得到一个缓存的值 + * @param cacheName 缓存名字 + * @param key + * @return + */ + public Object getKey(String cacheName,String key){ + Element em =ehCacheCacheManager.getCacheManager().getCache(cacheName).get(key); + return null==em?null:em.getObjectValue(); + } + + /** + * 向缓存中写入一个key + * @param cacheName + * @param key + * @param value + */ + public void addKey(String cacheName,String key,Object value){ + ehCacheCacheManager.getCacheManager().getCache(cacheName).put(new Element(key,value)); + } + + /** + * 得到一个缓存里key 的数量 + * @param cacheName + * @return + */ + public int getCacheTotalKeys(String cacheName){ + Cache cache=ehCacheCacheManager.getCacheManager().getCache(cacheName); + return null==cache?0:cache.getKeys().size(); + } + + + + +} diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/file/FileCommonOperactionTool.java b/src/main/java/io.github.kingschan1204.istock/common/util/file/FileCommonOperactionTool.java index 2739ab60..9285e226 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/file/FileCommonOperactionTool.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/file/FileCommonOperactionTool.java @@ -19,17 +19,23 @@ public class FileCommonOperactionTool { /** * 通过指定的文件下载URL以及下载目录下载文件 * @param url 下载url路径 + * @referrer 来源 * @param dir 存放目录 * @param filename 文件名 * @throws Exception */ - public static String downloadFile(String url, String dir, String filename) throws Exception { + public static String downloadFile(String url,String referrer, String dir, String filename) throws Exception { log.info("start download file :{}",url); //Open a URL Stream Connection.Response resultResponse = Jsoup.connect(url) .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3346.9 Safari/537.36") + .referrer(referrer) .ignoreContentType(true).execute(); String defaultFileName=""; + if(resultResponse.statusCode()!=200){ + log.error("文件下载失败：{}",url); + throw new Exception(String.format("文件下载失败：%s 返回码:%s",url,resultResponse.statusCode())); + } if(resultResponse.contentType().contains("name")){ String[] list =resultResponse.contentType().split(";"); defaultFileName = Arrays.stream(list) @@ -43,10 +49,12 @@ public static String downloadFile(String url, String dir, String filename) throw out.write(resultResponse.bodyAsBytes()); }catch (Exception ex){ log.error("{}",ex); + log.error("文件下载失败：{}",url); ex.printStackTrace(); }finally { out.close(); } return path; } + } diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java new file mode 100644 index 00000000..967b6588 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/QuartzManager.java @@ -0,0 +1,175 @@ +package io.github.kingschan1204.istock.common.util.quartz; + +import org.quartz.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; +import org.springframework.stereotype.Component; + +/** + * quartz 任务管理工具类 + * + * @author chenguoxiang + * @create 2018-07-13 14:46 + **/ +@Component +public class QuartzManager { + private static Logger log = LoggerFactory.getLogger(QuartzManager.class); + @Autowired + SchedulerFactoryBean schedulerFactoryBean; + + /** + * @param jobName 任务名 + * @param jobGroupName 任务组名 + * @param triggerName 触发器名 + * @param triggerGroupName 触发器组名 + * @param jobClass 任务 + * @param cron 时间设置，参考quartz说明文档 + * @Description: 添加一个定时任务 + */ + public void addJob(String jobName, String jobGroupName, + String triggerName, String triggerGroupName, Class jobClass, String cron) { + try { + Scheduler sched = schedulerFactoryBean.getScheduler();//schedulerFactory.getScheduler(); + + // 任务名，任务组，任务执行类 + JobDetail jobDetail = JobBuilder.newJob(jobClass).withIdentity(jobName, jobGroupName).build(); + // 触发器 + TriggerBuilder triggerBuilder = TriggerBuilder.newTrigger(); + // 触发器名,触发器组 + triggerBuilder.withIdentity(triggerName, triggerGroupName); + triggerBuilder.startNow(); + // 触发器时间设定 + triggerBuilder.withSchedule(CronScheduleBuilder.cronSchedule(cron)); + // 创建Trigger对象 + CronTrigger trigger = (CronTrigger) triggerBuilder.build(); + // 调度容器设置JobDetail和Trigger + sched.scheduleJob(jobDetail, trigger); + // 启动 + if (!sched.isShutdown()) { + log.info("添加定时任务{} , class:{} 执行频率：{}", jobName, jobClass.getName(), cron); + sched.start(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 修改一个任务的触发时间 + * + * @param jobName + * @param jobGroupName + * @param triggerName 触发器名 + * @param triggerGroupName 触发器组名 + * @param cron 时间设置，参考quartz说明文档 + * @Description: 修改一个任务的触发时间 + */ + public void modifyJobTime(String jobName, + String jobGroupName, String triggerName, String triggerGroupName, String cron) { + try { + Scheduler sched = schedulerFactoryBean.getScheduler(); + TriggerKey triggerKey = TriggerKey.triggerKey(triggerName, triggerGroupName); + CronTrigger trigger = (CronTrigger) sched.getTrigger(triggerKey); + if (trigger == null) { + return; + } + + String oldTime = trigger.getCronExpression(); + if (!oldTime.equalsIgnoreCase(cron)) { + /** 方式一 ：调用 rescheduleJob 开始 */ + // 触发器 + TriggerBuilder triggerBuilder = TriggerBuilder.newTrigger(); + // 触发器名,触发器组 + triggerBuilder.withIdentity(triggerName, triggerGroupName); + triggerBuilder.startNow(); + // 触发器时间设定 + triggerBuilder.withSchedule(CronScheduleBuilder.cronSchedule(cron)); + // 创建Trigger对象 + trigger = (CronTrigger) triggerBuilder.build(); + // 方式一 ：修改一个任务的触发时间 + sched.rescheduleJob(triggerKey, trigger); + log.info("添加定时任务{} , 执行频率：{}", jobName, cron); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 删除一个job + * + * @param jobName + * @param jobGroupName + * @param triggerName + * @param triggerGroupName + * @Description: 移除一个任务 + */ + public void removeJob(String jobName, String jobGroupName, + String triggerName, String triggerGroupName) { + try { + Scheduler sched = schedulerFactoryBean.getScheduler(); + TriggerKey triggerKey = TriggerKey.triggerKey(triggerName, triggerGroupName); + // 停止触发器 + sched.pauseTrigger(triggerKey); + // 移除触发器 + sched.unscheduleJob(triggerKey); + // 删除任务 + sched.deleteJob(JobKey.jobKey(jobName, jobGroupName)); + log.info("删除定时任务{}", jobName); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + /** + * @Description:启动所有定时任务 + */ + public void startJobs() { + try { + Scheduler sched = schedulerFactoryBean.getScheduler(); + sched.start(); + log.info("启动所有任务"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * @Description:关闭所有定时任务 + */ + public void shutdownJobs() { + try { + Scheduler sched = schedulerFactoryBean.getScheduler(); + if (!sched.isShutdown()) { + sched.shutdown(); + log.info("关闭所有任务"); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 暂停全部任务 + * + * @throws SchedulerException + */ + public void pauseAll() throws Exception { + schedulerFactoryBean.getScheduler().pauseAll(); + log.info("暂停所有任务"); + } + + + /** + * 恢复所有任务 + * + * @throws Exception + */ + public void resumeAll() throws Exception { + schedulerFactoryBean.getScheduler().resumeAll(); + log.info("恢复所有任务"); + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/quartz/SchedulerConfig.java b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/SchedulerConfig.java new file mode 100644 index 00000000..8bf1288a --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/SchedulerConfig.java @@ -0,0 +1,25 @@ +package io.github.kingschan1204.istock.common.util.quartz; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; + +/** + * + * @author chenguoxiang + * @create 2018-07-13 15:42 + **/ +@Configuration +public class SchedulerConfig { + @Autowired + private SpringJobFactory springJobFactory; + + @Bean + public SchedulerFactoryBean schedulerFactoryBean() { + SchedulerFactoryBean schedulerFactoryBean = new SchedulerFactoryBean(); + schedulerFactoryBean.setJobFactory(springJobFactory); + return schedulerFactoryBean; + } + +} \ No newline at end of file diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/quartz/SpringJobFactory.java b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/SpringJobFactory.java new file mode 100644 index 00000000..4d033bd7 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/util/quartz/SpringJobFactory.java @@ -0,0 +1,32 @@ +package io.github.kingschan1204.istock.common.util.quartz; + +import org.quartz.spi.TriggerFiredBundle; +import org.springframework.beans.factory.config.AutowireCapableBeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.scheduling.quartz.SpringBeanJobFactory; +import org.springframework.stereotype.Component; + +/** + * + * @author chenguoxiang + * @create 2018-07-13 15:40 + **/ +@Component +public class SpringJobFactory extends SpringBeanJobFactory implements + ApplicationContextAware { + private transient AutowireCapableBeanFactory beanFactory; + @Override + public void setApplicationContext(final ApplicationContext context) { + beanFactory = context.getAutowireCapableBeanFactory(); + } + @Override + protected Object createJobInstance(final TriggerFiredBundle bundle) throws Exception { + final Object job = super.createJobInstance(bundle); + beanFactory.autowireBean(job); + return job; + } + + +} + diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java index 236b90cf..54ea43c8 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockDateUtil.java @@ -16,6 +16,16 @@ public class StockDateUtil { private static Logger log = LoggerFactory.getLogger(StockDateUtil.class); + /** + * 得到当前系统的年月日时间 + * + * @return YYYY-MM-DD + */ + public static String getCurrentDate() { + return new SimpleDateFormat("yyyy-MM-dd").format(new Date(System + .currentTimeMillis())); + } + /** * 返回当前日期星期几 * @@ -61,6 +71,7 @@ public static boolean stockOpenTime() { /** * 传入年数返回具体的每个月日期 + * * @param years 年数 * @return */ @@ -85,15 +96,15 @@ public static String[] getTimeLine(int years) { * * @return 10位的数字 */ - public static long getCurrentDate() { + public static long getCurrentTimestamp() { long date = System.currentTimeMillis() / 1000; return date; } - /** * 输入一个时间,获取该时间的时间戳 + * * @param dateString 日期字符串(yyyy-MM-dd HH:mm:ss) * @return long类型的10位数字 * @throws ParseException @@ -108,8 +119,7 @@ public static long dateToUnixTime(String dateString) throws ParseException { /** * 格式化unix 时间戳成日期字符串(yyyy-MM-dd HH:mm:ss) * - * @param time - * 10位 + * @param time 10位 * @return yyyy-MM-dd HH:mm:ss的字符串 */ public static String formatUnixDate(long time) { @@ -120,12 +130,63 @@ public static String formatUnixDate(long time) { /** * 得到当前系统的年月日数字 * - * @return YYYYMMDD + * @return YYYYMMDD */ public static Integer getCurrentDateNumber() { - String date=new SimpleDateFormat("yyyyMMdd").format(new Date(System + String date = new SimpleDateFormat("yyyyMMdd").format(new Date(System .currentTimeMillis())); return Integer.valueOf(date); } + /** + * 得到当前时间的Long数字 + * @return + */ + public static Long getCurrentDateTimeNumber() { + String date = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System + .currentTimeMillis())); + return Long.valueOf(date); + } + + /** + * 获取某年第一天日期 + * + * @param year 年份 + * @return String 传入年份的第一天 + */ + public static String getCurrYearFirstDay(int year) { + Calendar calendar = Calendar.getInstance(); + calendar.clear(); + calendar.set(Calendar.YEAR, year); + Date currYearFirst = calendar.getTime(); + return new SimpleDateFormat("yyyy-MM-dd").format(currYearFirst); + } + + /** + * 获取某年最后一天日期 + * + * @param year 年份 + * @return String 传入年份的最后一天 + */ + public static String getCurrYearLastDay(int year) { + Calendar calendar = Calendar.getInstance(); + calendar.clear(); + calendar.set(Calendar.YEAR, year); + calendar.roll(Calendar.DAY_OF_YEAR, -1); + Date currYearLast = calendar.getTime(); + return new SimpleDateFormat("yyyy-MM-dd").format(currYearLast); + } + + /** + * 得到当前年份 + * + * @return + */ + public static int getCurrentYear() { + String year = new SimpleDateFormat("yyyy").format(new Date(System + .currentTimeMillis())); + return Integer.parseInt(year); + } + + } diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java index 0d6907c6..85e6067e 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/StockSpider.java @@ -1,44 +1,50 @@ package io.github.kingschan1204.istock.common.util.stock; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import org.jsoup.Jsoup; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import java.io.IOException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * 爬虫接口定义 + * * @author chenguoxiang * @create 2018-01-30 18:00 **/ public interface StockSpider { - static final String regexNumber = "^[-+]?([0]{1}(\\.[0-9]+)?|[1-9]{1}\\d*(\\.[0-9]+)?)";//"^[-+]?[0-9]+(\\.[0-9]+)?$"; + static final String REGEX_NUMBER = "^[-+]?([0]{1}(\\.[0-9]+)?|[1-9]{1}\\d*(\\.[0-9]+)?)";//"^[-+]?[0-9]+(\\.[0-9]+)?$"; /** * 将股票代码转换成新浪接口的格式http://hq.sinajs.cn/list= * sh 上海 sz 深圳 + * * @param code * @return */ - static String formatStockCode(String code) { - if(code.matches("^(sz|sh)\\d{6}$")){ - return code; - } + static String formatStockCode(String code) { + if (code.matches("^(sz|sh)\\d{6}$")) { + return code; + } //5开头，沪市基金或权证 60开头上证 - else if (code.matches("^60.*|^5.*")) { + else if (code.matches("^60.*|^5.*")) { return String.format("sh%s", code); } //1开头的，是深市基金 00开头是深圳 - else if(code.matches("^1.*|^00.*|^300...")){ + else if (code.matches("^1.*|^00.*|^300...")) { return String.format("sz%s", code); } return null; @@ -46,11 +52,12 @@ else if(code.matches("^1.*|^00.*|^300...")){ /** * 提取文本中匹配正则的字符串 + * * @param text * @param regx 正则 - * @return 多个结果已,分隔 + * @return 多个结果已, 分隔 */ - static String findStrByRegx(String text,String regx){ + static String findStrByRegx(String text, String regx) { StringBuffer bf = new StringBuffer(64); try { Pattern pattern = Pattern.compile(regx); @@ -69,16 +76,54 @@ static String findStrByRegx(String text,String regx){ } } + + /** + * 是否工作日 + * @param date + * @return + */ + static boolean isWorkDay(String date) throws IOException { + String api =String.format("http://api.goseek.cn/Tools/holiday?date=%s",date); + String result = Jsoup.connect(api).timeout(3000).ignoreContentType(true).get().text(); + //{"code":10001,"data":2} 工作日对应结果为 0, 休息日对应结果为 1, 节假日对应的结果为 2 + JSONObject json = JSON.parseObject(result); + return json.getIntValue("data")==0; + } + + + /** + * 提取文本中匹配正则的字符串 + * + * @param text 源字符串 + * @param regx 正则 + * @return + */ + static List findStringByRegx(String text, String regx) { + List list = new ArrayList(); + try { + Pattern pattern = Pattern.compile(regx); + Matcher matcher = pattern.matcher(text); + while (matcher.find()) { + list.add(matcher.group()); + } + + return list; + } catch (Exception e) { + e.printStackTrace(); + return list; + } + + } + /** * 格式化数据，如果不是数字全部返回-1 * * @param value * @return */ - static Double mathFormat(String value) { - String v =value.replaceAll("[^0-9|\\.|\\-]",""); - //value.replaceAll("\\%", "").replace("亿", ""); - if (v.matches(regexNumber)) { + static Double mathFormat(String value) { + String v = value.replaceAll("[^0-9|\\.|\\-]", ""); + if (v.matches(REGEX_NUMBER)) { return Double.valueOf(v); } return -1D; @@ -86,14 +131,22 @@ static Double mathFormat(String value) { /** * 启用ssl + * * @throws KeyManagementException * @throws NoSuchAlgorithmException */ - static void enableSSLSocket() throws KeyManagementException, NoSuchAlgorithmException { - TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager(){ - public X509Certificate[] getAcceptedIssuers(){return new X509Certificate[0];} - public void checkClientTrusted(X509Certificate[] certs, String authType){} - public void checkServerTrusted(X509Certificate[] certs, String authType){} + static void enableSSLSocket() throws KeyManagementException, NoSuchAlgorithmException { + TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + @Override + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } }}; SSLContext sc = SSLContext.getInstance("TLS"); sc.init(null, trustAllCerts, new SecureRandom()); @@ -103,31 +156,35 @@ public void checkServerTrusted(X509Certificate[] certs, String authType){} /** * 得到指定代码的价格 + * * @param stockCode * @return */ - JSONArray getStockPrice(String[] stockCode)throws Exception; + JSONArray getStockPrice(String[] stockCode) throws Exception; /** * 得到指定代码的基本信息 + * * @param code * @return * @throws Exception */ - JSONObject getStockInfo(String code) throws Exception; + JSONObject getStockInfo(String code) throws Exception; /** * 得到指定代码历史分红 + * * @param code * @return * @throws Exception */ - JSONArray getHistoryDividendRate(String code) throws Exception; + JSONArray getHistoryDividendRate(String code) throws Exception; /** * 得到历史ROE + * * @param code * @return * @throws Exception @@ -135,42 +192,27 @@ public void checkServerTrusted(X509Certificate[] certs, String authType){} JSONArray getHistoryROE(String code) throws Exception; - /** - * 得到历史PE - * @param code - * @return - * @throws Exception - */ - JSONArray getHistoryPE(String code) throws Exception; - - - /** - * 得到历史pb - * @param code - * @return - * @throws Exception - */ - JSONArray getHistoryPB(String code) throws Exception; - - /** * 得到所有代码 + * * @return * @throws Exception */ - List getAllStockCode()throws Exception; + List getAllStockCode() throws Exception; /** * 得到股息 - * @param page 页码 + * + * @param page 页码 * @return * @throws Exception */ - JSONObject getDy(int page)throws Exception; + JSONObject getDy(int page) throws Exception; /** * 得到上海所有代码 + * * @return * @throws Exception */ @@ -179,9 +221,11 @@ public void checkServerTrusted(X509Certificate[] certs, String authType){} /** * 得到深圳所有代码 + * * @return * @throws Exception */ List getStockCodeBySZ() throws Exception; + } diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java index e2352a45..7bfe3485 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/DefaultSpiderImpl.java @@ -21,12 +21,13 @@ import java.io.File; import java.math.RoundingMode; import java.net.SocketTimeoutException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; import java.text.NumberFormat; import java.util.*; /** * 默认爬虫 - * * @author chenguoxiang * @create 2018-01-31 14:02 **/ @@ -75,8 +76,8 @@ public JSONArray getStockPrice(String[] stockCode) throws Exception { double xj = StockSpider.mathFormat(data[4]); double zs = StockSpider.mathFormat(data[3]); double zf = (xj - zs) / zs * 100; - double today_max = StockSpider.mathFormat(data[5]); - double today_min = StockSpider.mathFormat(data[6]); + double todayMax = StockSpider.mathFormat(data[5]); + double todayMin = StockSpider.mathFormat(data[6]); json = new JSONObject(); if (xj == 0) { //一般这种是停牌的 json.put("fluctuate", 0);//波动 @@ -91,12 +92,12 @@ public JSONArray getStockPrice(String[] stockCode) throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai")); json.put("code", data[0]);//代码 json.put("type", StockSpider.formatStockCode(data[0]).replaceAll("\\d", "")); - json.put("name", data[1].replaceAll("\\s",""));//名称 + json.put("name", data[1].replaceAll("\\s", ""));//名称 json.put("price", xj);//现价 - json.put("todayMax", today_max);//今日最高价 - json.put("todayMin", today_min);//今日最低价 + json.put("todayMax", todayMax);//今日最高价 + json.put("todayMin", todayMin);//今日最低价 json.put("yesterdayPrice", zs);//昨收 - json.put("priceDate", new Date()); + json.put("priceDate", StockDateUtil.getCurrentDateTimeNumber()); rows.add(json); } return rows; @@ -151,20 +152,24 @@ public JSONArray getHistoryDividendRate(String code) throws Exception { JSONArray jsons = new JSONArray(); JSONObject json; for (int i = 1; i < rows.size(); i++) { - String[] data = rows.get(i).select("td").text().split(" "); - log.debug("报告期:{},A股除权除息日:{},实施日期:{},分红方案说明:{},分红率:{}", data[0], data[6], data[3], data[4], data[9]); - //String year, String date, String plan, Double percent - double value = -1; - if (null != data[9]) { - value = StockSpider.mathFormat(data[9]); + String rowtext = rows.get(i).select("td").text(); + String[] data = rowtext.split(" "); + if ("--".equals(data[6]) || "--".equals(data[9])) { + continue; } + log.debug("报告期:{},A股除权除息日:{},实施日期:{},分红方案说明:{},分红率:{}", data[0], data[6], data[3], data[4], data[9]); json = new JSONObject(); json.put("code", stockCode); - json.put("title", data[0]); - json.put("date", data[6]); - json.put("percent", value); - json.put("executeDate", data[3]); - json.put("remark", data[4]); + json.put("title", data[0]);//报告期 + json.put("releaseDate", data[1]);//披露时间 董事会日期 + json.put("plan", data[4]);//分红方案 + json.put("sgbl", 0);//送股比例 + json.put("zgbl", 0);//转股比例 + json.put("percent", StockSpider.mathFormat(data[9]));//分红率 + json.put("gqdjr", data[5]);//股权登记日 + json.put("cxcqr", data[6]);//除息除权日 + json.put("progress", data[7]);//方案进度 + json.put("from", "ths");//来源 jsons.add(json); } return jsons; @@ -176,12 +181,15 @@ public JSONArray getHistoryDividendRate(String code) throws Exception { public JSONArray getHistoryROE(String code) throws Exception { String url = String.format("http://basic.10jqka.com.cn/api/stock/export.php?export=main&type=year&code=%s", code); String path = String.format("./data/%s_main_year.xls", code); + String referrer=String.format("http://basic.10jqka.com.cn/%s/finance.html",code); if (!new File(path).exists()) { //下载 - FileCommonOperactionTool.downloadFile(url, "./data/", null); + path= FileCommonOperactionTool.downloadFile(url, referrer,"./data/", code+".xls"); + }else{ + log.info("文件存在，直接读取：{}",path); } //读取excel数据 - List list = ExcelOperactionTool.readExcelData(String.format("./data/%s_main_year.xls", code)); + List list = ExcelOperactionTool.readExcelData(path); Object[] year = list.get(1); Object[] roe = list.get(10); Object[] roeTb = list.get(11); @@ -199,65 +207,6 @@ public JSONArray getHistoryROE(String code) throws Exception { return jsons; } - @Override - public JSONArray getHistoryPE(String code) throws Exception { - String url = String.format("https://androidinvest.com/Stock/History/%s", StockSpider.formatStockCode(code).toUpperCase()); - String regex = "\'|\\[|\\]"; - log.info("craw history pe :{}", url); - StockSpider.enableSSLSocket(); - JSONArray jsons = new JSONArray(); - Document doc = null; - try { - doc = Jsoup.connect(url).userAgent(useAgent).timeout(timeout).get(); - Element div = doc.getElementById("chart2"); - String data[] = div.text().split("@");//日期@市盈率@股价 - String date[] = data[0].replaceAll(regex, "").split(","); - String pe[] = data[1].replaceAll(regex, "").split(","); - String price[] = data[2].replaceAll(regex, "").split(","); - JSONObject json; - for (int i = 0; i < date.length; i++) { - json = new JSONObject(); - json.put("code", code.replaceAll("\\D", "")); - json.put("date", date[i].trim()); - json.put("pe", pe[i].trim()); - json.put("price", price[i].trim()); - jsons.add(json); - } - } catch (Exception e) { - e.printStackTrace(); - } - return jsons; - } - - @Override - public JSONArray getHistoryPB(String code) throws Exception { - String url = String.format("https://androidinvest.com/Stock/HistoryPB/%s", StockSpider.formatStockCode(code).toUpperCase()); - log.info("craw history pb :{}", url); - StockSpider.enableSSLSocket(); - JSONArray jsons = new JSONArray(); - Document doc = null; - try { - doc = Jsoup.connect(url).userAgent(useAgent).timeout(timeout).get(); - Element div = doc.getElementById("chart4"); - String data[] = div.text().split("@");//日期@市净率@股价 - String date[] = data[0].replaceAll("\'|\\[|\\]", "").split(","); - String pb[] = data[1].replaceAll("\'|\\[|\\]", "").split(","); - String price[] = data[2].replaceAll("\'|\\[|\\]", "").split(","); - JSONObject json; - for (int i = 0; i < date.length; i++) { - json = new JSONObject(); - json.put("code", code.replaceAll("\\D", "")); - json.put("date", date[i].trim()); - json.put("pb", pb[i].trim()); - json.put("price", price[i].trim()); - jsons.add(json); - } - - } catch (Exception e) { - e.printStackTrace(); - } - return jsons; - } @Override public List getAllStockCode() throws Exception { @@ -286,6 +235,7 @@ public List getAllStockCode() throws Exception { * @return * @throws Exception */ + @Override public JSONObject getDy(int page) throws Exception { String url = "https://xueqiu.com/stock/screener/screen.json?category=SH&exchange=&areacode=&indcode=&orderby=symbol&order=desc¤t=ALL&pct=ALL&page=%s&dy=0_19.31&size=100"; url = String.format(url, page); @@ -314,6 +264,7 @@ public JSONObject getDy(int page) throws Exception { @Override public List getStockCodeBySH() throws Exception { + List list = new ArrayList<>(); String url = "http://www.sse.com.cn/js/common/ssesuggestdata.js"; log.info("craw sh codes :{}", url); Document infoDoc = Jsoup.connect(url).userAgent(useAgent) @@ -322,7 +273,10 @@ public List getStockCodeBySH() throws Exception { .get(); String result = StockSpider.findStrByRegx(infoDoc.html(), "60\\d{4}"); String[] codes = result.split(","); - return Arrays.asList(codes); + Arrays.stream(codes).forEach(code ->{ + list.add(StockSpider.formatStockCode(code)); + }); + return list; } @Override @@ -334,13 +288,13 @@ public List getStockCodeBySZ() throws Exception { String path = String.format("./data/%s", filename); if (!new File(path).exists()) { //下载 - FileCommonOperactionTool.downloadFile(url, "./data/", filename); + FileCommonOperactionTool.downloadFile(url, "","./data/", filename); } //读取excel数据 List list = ExcelOperactionTool.readExcelData(path); for (Object[] row : list) { if (row[0].toString().trim().matches("^00\\d{4}")) { - codes.add(row[0].toString().trim()); + codes.add(StockSpider.formatStockCode(row[0].toString().trim())); } } return codes; diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java index aa512610..2f95d160 100644 --- a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/EastmoneySpider.java @@ -23,7 +23,7 @@ public class EastmoneySpider extends DefaultSpiderImpl { private Logger log = LoggerFactory.getLogger(EastmoneySpider.class); @Value("${eastmoney.token}") - private String token ; + private String token; /** * ReportingPeriod:报告期 @@ -46,38 +46,47 @@ public JSONArray getHistoryDividendRate(String code) throws Exception { String regex = "T.*"; String apiUrl = String.format("http://dcfm.eastmoney.com/EM_MutiSvcExpandInterface/api/js/get?type=DCSOBS&token=%s&p=1&ps=50&sr=-1&st=ReportingPeriod&filter=&cmd=%s", token, code); String referer = String.format("http://data.eastmoney.com/yjfp/detail/%s.html", code); - Document doc=null; - try{ - doc = Jsoup.connect(apiUrl).userAgent(useAgent) + Document doc = null; + try { + doc = Jsoup.connect(apiUrl).userAgent(useAgent) .timeout(10000) .ignoreContentType(true) .referrer(referer).get(); - }catch (SocketTimeoutException e){ - log.error("抓取超时：{}",apiUrl); + } catch (SocketTimeoutException e) { + log.error("抓取超时：{}", apiUrl); return new JSONArray(); } JSONArray data = JSONArray.parseArray(doc.text()); - if(null==data)return new JSONArray(); - + if (null == data) { + return new JSONArray(); + } JSONArray jsons = new JSONArray(); JSONObject temp; for (int i = 0; i < data.size(); i++) { JSONObject item = data.getJSONObject(i); temp = new JSONObject(); temp.put("code", code); - temp.put("title", item.getString("ReportingPeriod").replaceAll(regex, ""));//报告期 + String title = item.getString("ReportingPeriod").replaceAll(regex, ""); + if (title.matches("^\\d{4}\\-12-31$")) { + title = title.replaceAll("\\-.*", "") + "年报"; + } + if (title.matches("^\\d{4}\\-06-30$")) { + title = title.replaceAll("\\-.*", "") + "中报"; + } + temp.put("title", title);//报告期 temp.put("releaseDate", item.getString("ResultsbyDate").replaceAll(regex, ""));//披露时间 temp.put("plan", item.getString("AllocationPlan"));//分配预案 temp.put("sgbl", intFormart(item.getString("SGBL")));//送股比例 temp.put("zgbl", intFormart(item.getString("ZGBL")));//转股比例 - temp.put("percent",doubleFormat(item.getString("GXL"),true));//股息率 + temp.put("percent", doubleFormat(item.getString("GXL"), true));//股息率 temp.put("gqdjr", item.getString("GQDJR").replaceAll(regex, ""));//股权登记日 temp.put("cxcqr", item.getString("CQCXR").replaceAll(regex, ""));//除息除权日 temp.put("progress", item.getString("ProjectProgress"));//进度 + temp.put("from", "east");//来源 jsons.add(temp); } - log.info("{}:抓取成功!",code); + log.info("{}:抓取成功!", code); return jsons; } @@ -94,17 +103,17 @@ public int intFormart(String data) { * @param math * @return */ - public double doubleFormat(String math,boolean percent) { - String regex_number="^[-+]?([0]{1}(\\.[0-9]+)?|[1-9]{1}\\d*(\\.[0-9]+)?)"; - if(math.matches(regex_number)){ + public double doubleFormat(String math, boolean percent) { + String regexNumber = "^[-+]?([0]{1}(\\.[0-9]+)?|[1-9]{1}\\d*(\\.[0-9]+)?)"; + if (math.matches(regexNumber)) { Double d = Double.parseDouble(math); - if(percent){ - d=d*100; + if (percent) { + d = d * 100; } BigDecimal b = new BigDecimal(d); return b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue(); } - return 0d; + return 0d; } diff --git a/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java new file mode 100644 index 00000000..72c2fb7f --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/common/util/stock/impl/JisiluSpilder.java @@ -0,0 +1,95 @@ +package io.github.kingschan1204.istock.common.util.stock.impl; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import io.github.kingschan1204.istock.common.util.stock.StockSpider; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * jisilu爬虫 + * + * @author chenguoxiang + * @create 2018-07-03 11:22 + **/ +@Component("JisiluSpilder") +public class JisiluSpilder extends DefaultSpiderImpl { + + private static Logger log = LoggerFactory.getLogger(JisiluSpilder.class); + + /** + * 得到ipo上市日期，最近公开报表，历史数据 + * + * @param code + * @return + */ + public JSONObject crawHisPbPePriceAndReports(String code) throws Exception { +// String useAgent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3346.9 Safari/537.36"; + String referrer = "https://www.jisilu.cn/data/stock/dividend_rate/#cn"; +// int timeout=8000; + JSONObject result = new JSONObject(); + String url = String.format("https://www.jisilu.cn/data/stock/%s", code); + log.info("craw jisilu page :{}", url); + StockSpider.enableSSLSocket(); + JSONArray jsons = new JSONArray(); + Document doc = null; + try { + doc = Jsoup.connect(url).userAgent(useAgent).referrer(referrer).timeout(timeout).get(); + // 得到ipo上市日期 + Elements spans = doc.getElementsByAttributeValue("style", "display: inline-block;width: 80px;color: #0088cb;font-size:12px;"); + spans.stream().forEach(span -> { + if (span.text().matches("\\d{4}\\-\\d{2}\\-\\d{2}")) { + result.put("ipoDate", span.html()); + } + }); + //拿定期公告 + Element table = doc.getElementById("tbl_periodicalreport"); + Elements tr = table.getElementsByTag("tr"); + JSONArray reportsJsons = new JSONArray(); + tr.stream().forEach(row -> { + Elements a = row.getElementsByTag("a"); + String releaseDay = row.getElementsByTag("td").get(1).text(); + String link = a.get(0).attr("href"); + String title = a.get(0).text(); + JSONObject temp = new JSONObject(); + temp.put("releaseDay", releaseDay); + temp.put("link", link); + temp.put("title", title); + reportsJsons.add(temp); + }); + result.put("reports", reportsJsons); + + //解析历史数据 + Elements js = doc.getElementsByTag("script").eq(21); + List list = StockSpider.findStringByRegx(js.html(), "\\[.*\\]"); + //依次顺序 0:日期 1:PRICE 2:PB 3:PE + String replaceRegex = "\'|\\[|\\]"; + String dates[] = list.get(0).replaceAll(replaceRegex, "").split(",");//日期 + String prices[] = list.get(1).replaceAll(replaceRegex, "").split(",");//价格 + String pbs[] = list.get(2).replaceAll(replaceRegex, "").split(",");//市净率 + String pes[] = list.get(3).replaceAll(replaceRegex, "").split(",");//市盈率 + JSONArray hisJson = new JSONArray(); + for (int i = 0; i < dates.length; i++) { + JSONObject temp = new JSONObject(); + temp.put("date", dates[i]); + temp.put("price", Double.parseDouble(prices[i])); + temp.put("pb", Double.parseDouble(pbs[i])); + temp.put("pe", Double.parseDouble(pes[i])); + hisJson.add(temp); + } + result.put("hisdata", hisJson); + + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java index 3e7f8877..8b1a0760 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockCtrl.java @@ -1,14 +1,25 @@ package io.github.kingschan1204.istock.module.maindata.ctrl; +import com.mongodb.BasicDBObject; +import com.mongodb.WriteResult; +import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; import io.github.kingschan1204.istock.module.maindata.services.StockCodeService; import io.github.kingschan1204.istock.module.maindata.services.StockService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.mapreduce.MapReduceOptions; +import org.springframework.data.mongodb.core.mapreduce.MapReduceResults; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; +import java.util.Iterator; + /** * @author chenguoxiang * @create 2018-03-27 11:14 @@ -24,22 +35,89 @@ public class StockCtrl { private StockSpider spider; @RequestMapping("/stock/q") - public String queryStock(Integer page, Integer rows, String code,String type,String pb,String dy, String sidx, String sord) { + public String queryStock(Integer page, Integer rows, String code, String type, String pb, String dy, String sidx, String sord) { String order = (null == sord || sord.isEmpty()) ? "asc" : sord; String field = (null == sidx || sidx.isEmpty()) ? "_id" : sidx; - return service.queryStock(page, rows, code,type,pb,dy, field, order); + return service.queryStock(page, rows, code, type, pb, dy, field, order); + } + + + @ResponseBody + @RequestMapping(value = "/stock/init_code", method = RequestMethod.POST) + public String initCode() { + try { + stockCodeService.saveAllStockCode(); + return "success"; + } catch (Exception e) { + e.printStackTrace(); + return e.getMessage(); + } + } + + @Autowired + private MongoTemplate template; + + @ResponseBody + @RequestMapping(value = "/stock/mapReduce/5years_dy") + public String fiveYearsDy() { + int year = StockDateUtil.getCurrentYear(); + int fiveYearAgo = year - 5; + String startDate = StockDateUtil.getCurrYearLastDay(fiveYearAgo).toString(); + String endDate = StockDateUtil.getCurrentDate(); + Query query = new Query(); + query.addCriteria(Criteria.where("releaseDate").gte(startDate).lte(endDate)); + MapReduceResults result = template.mapReduce(query, "stock_dividend", + "classpath:/mapreduce/5years_dy/dy5years_map.js", "classpath:/mapreduce/5years_dy/dy5years_reduce.js", + new MapReduceOptions().outputCollection("stock_dy_statistics"), BasicDBObject.class); + Iterator iter = result.iterator(); + while (iter.hasNext()) { + BasicDBObject item = iter.next(); + String code = item.getString("_id"); + BasicDBObject value = (BasicDBObject) item.get("value"); + if (value.containsKey("size") && value.getInt("size") > 4) { + double percent = Double.parseDouble(value.getString("percent")); + WriteResult wr = template.upsert( + new Query(Criteria.where("_id").is(code)), + new Update() + .set("_id", code) + .set("fiveYearDy", percent), + "stock" + ); + } + + } + return String.format("success:%s - %s", startDate, endDate); } - @ResponseBody - @RequestMapping(value = "/stock/init_code",method = RequestMethod.POST) - public String initCode(){ - try { - stockCodeService.saveAllStockCode(); - return "success"; - } catch (Exception e) { - e.printStackTrace(); - return e.getMessage(); - } - } + @ResponseBody + @RequestMapping(value = "/stock/mapReduce/5years_roe") + public String fiveYearsRoe() { + int endYear = StockDateUtil.getCurrentYear(); + int startYear = endYear - 5; + Query query = new Query(); + query.addCriteria(Criteria.where("year").gte(startYear).lte(endYear)); + MapReduceResults result = template.mapReduce(query, "stock_his_roe", + "classpath:/mapreduce/5years_roe/map.js", "classpath:/mapreduce/5years_roe/reduce.js", + new MapReduceOptions().outputCollection("stock_hisroe_statistics"), BasicDBObject.class); + Iterator iter = result.iterator(); + while (iter.hasNext()) { + BasicDBObject item = iter.next(); + String code = item.getString("_id"); + BasicDBObject value = (BasicDBObject) item.get("value"); + if (value.containsKey("size") && value.getInt("size") > 4) { + double percent = Double.parseDouble(value.getString("percent")); + WriteResult wr = template.upsert( + new Query(Criteria.where("_id").is(code)), + new Update() + .set("_id", code) + .set("fiveYearRoe", percent), + "stock" + ); + } + + } + return String.format("success:%s - %s", startYear, endYear); + } + } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java index a37bf136..6e4c0872 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/ctrl/StockHisPageCtrl.java @@ -1,20 +1,31 @@ package io.github.kingschan1204.istock.module.maindata.ctrl; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.mongodb.BasicDBObject; +import com.mongodb.DBCollection; +import com.mongodb.DBObject; +import io.github.kingschan1204.istock.common.util.stock.impl.JisiluSpilder; import io.github.kingschan1204.istock.module.maindata.po.*; import io.github.kingschan1204.istock.module.maindata.repository.StockRepository; import io.github.kingschan1204.istock.module.maindata.services.StockHisRoeService; import io.github.kingschan1204.istock.module.maindata.services.StockService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.servlet.ModelAndView; +import java.util.ArrayList; import java.util.List; /** - *代码历史数据控制器 + * 代码历史数据控制器 + * * @author chenguoxiang * @create 2018-03-27 15:19 **/ @@ -27,157 +38,148 @@ public class StockHisPageCtrl { private StockService stockService; @Autowired private StockHisRoeService stockHisRoeService; - private final String template_path="/stock/his/"; - - @RequestMapping("/stock/his_dy/{code}") - public ModelAndView hisdata(@PathVariable String code, Model model){ - ModelAndView mav = new ModelAndView(template_path+"his_dy"); - mav.addObject("year","''"); - mav.addObject("percent","0"); - Stock stock =stockRepository.findOne(code); - if(null==stock){ - mav.addObject("msg",String.format("代码:%s %s",code,"不存在，或者非A股代码!")); - return mav; - } - mav.addObject("stock",stock); - //历年分红 - List list =stockService.getStockDividend(code); - if(null!=list&&list.size()>0){ - StringBuffer year = new StringBuffer(); - StringBuffer percent = new StringBuffer(); - list.stream().forEach(item ->{ - if(item.getPercent()>0){ - percent.append(item.getPercent()).append(","); - year.append("'").append(item.getTitle()).append("',"); - } - - }); - String data= String.format("%s|%s",year.toString().replaceAll("\\,$",""), - percent.toString().replaceAll("\\,$","") - ); - String item[]=data.split("\\|"); - if(item.length==2){ - mav.addObject("year",item[0]); - mav.addObject("percent",item[1]); - mav.addObject("rows",list); - } - }else{ - mav.addObject("msg","该股票没有分红信息!"); - } - return mav; - } + private final String template_path = "/stock/his/"; - @RequestMapping("/stock/his_roe/{code}") - public ModelAndView getStockHisRoe(@PathVariable String code) { - ModelAndView mav = new ModelAndView(template_path+"his_roe"); - mav.addObject("roe_year","''"); - mav.addObject("roe_percent","0"); - Stock stock =stockRepository.findOne(code); - if(null==stock){ - mav.addObject("msg",String.format("代码:%s %s",code,"不存在，或者非A股代码!")); - return mav; - } - mav.addObject("stock",stock); - List list=stockService.getStockHisRoe(code); - if(null==list||list.size()==0){ - try { - list= stockHisRoeService.addStockHisRoe(code); - } catch (Exception e) { - e.printStackTrace(); - } - } - mav.addObject("rows",list); + @RequestMapping("/stock/his_dy/{code}") + public ModelAndView hisdata(@PathVariable String code, Model model) { + ModelAndView mav = new ModelAndView(template_path + "his_dy"); + mav.addObject("year", "''"); + mav.addObject("percent", "0"); + Stock stock = stockRepository.findOne(code); + if (null == stock) { + mav.addObject("msg", String.format("代码:%s %s", code, "不存在，或者非A股代码!")); + return mav; + } + mav.addObject("stock", stock); + //历年分红 + List list = stockService.getStockDividend(code); + if (null != list && list.size() > 0) { StringBuffer year = new StringBuffer(); StringBuffer percent = new StringBuffer(); - list.stream().forEach(item ->{ - if(item.getRoe()>0){ - percent.append(item.getRoe()).append(","); - year.append("'").append(item.getYear()).append("',"); + list.stream().forEach(item -> { + if (item.getPercent() > 0) { + percent.append(item.getPercent()).append(","); + year.append("'").append(item.getTitle()).append("',"); } }); - String data= String.format("%s|%s",year.toString().replaceAll("\\,$",""), - percent.toString().replaceAll("\\,$","") + String data = String.format("%s|%s", year.toString().replaceAll("\\,$", ""), + percent.toString().replaceAll("\\,$", "") ); - String roeItem[]=data.split("\\|"); - mav.addObject("roe_year",roeItem[0]); - mav.addObject("roe_percent",roeItem[1]); - return mav; + String item[] = data.split("\\|"); + if (item.length == 2) { + mav.addObject("year", item[0]); + mav.addObject("percent", item[1]); + mav.addObject("rows", list); + } + } else { + mav.addObject("msg", "该股票没有分红信息!"); } + return mav; + } - - @RequestMapping("/stock/his_pe/{code}") - public ModelAndView getStockHisPe(@PathVariable String code) { - ModelAndView mav = new ModelAndView(template_path+"his_pe"); - mav.addObject("pe_date","''"); - mav.addObject("pe_value","0"); - Stock stock =stockRepository.findOne(code); - if(null==stock){ - mav.addObject("msg",String.format("代码:%s %s",code,"不存在，或者非A股代码!")); + @RequestMapping("/stock/his_roe/{code}") + public ModelAndView getStockHisRoe(@PathVariable String code) { + ModelAndView mav = new ModelAndView(template_path + "his_roe"); + mav.addObject("roe_year", "''"); + mav.addObject("roe_percent", "0"); + Stock stock = stockRepository.findOne(code); + if (null == stock) { + mav.addObject("msg", String.format("代码:%s %s", code, "不存在，或者非A股代码!")); return mav; } - mav.addObject("stock",stock); - List list=stockService.getStockHisPe(code); - if(null==list||list.size()==0){ + mav.addObject("stock", stock); + List list = stockService.getStockHisRoe(code); + if (null == list || list.size() == 0) { try { - list= stockService.addStockHisPe(code); + list = stockHisRoeService.addStockHisRoe(code); } catch (Exception e) { e.printStackTrace(); } } - mav.addObject("rows",list); + mav.addObject("rows", list); StringBuffer year = new StringBuffer(); - StringBuffer pe = new StringBuffer(); - list.stream().forEach(item ->{ - if(item.getPe()>0){ - pe.append(item.getPe()).append(","); - year.append("'").append(item.getDate()).append("',"); + StringBuffer percent = new StringBuffer(); + list.stream().forEach(item -> { + if (item.getRoe() > 0) { + percent.append(item.getRoe()).append(","); + year.append("'").append(item.getYear()).append("',"); } + }); - String data= String.format("%s|%s",year.toString().replaceAll("\\,$",""), - pe.toString().replaceAll("\\,$","") + String data = String.format("%s|%s", year.toString().replaceAll("\\,$", ""), + percent.toString().replaceAll("\\,$", "") ); - String roeItem[]=data.split("\\|"); - mav.addObject("pe_date",roeItem[0]); - mav.addObject("pe_value",roeItem[1]); + String roeItem[] = data.split("\\|"); + mav.addObject("roe_year", roeItem[0]); + mav.addObject("roe_percent", roeItem[1]); return mav; } - @RequestMapping("/stock/his_pb/{code}") - public ModelAndView getStockHisPb(@PathVariable String code) { - ModelAndView mav = new ModelAndView(template_path+"his_pb"); - mav.addObject("pb_date","''"); - mav.addObject("pb_value","0"); - Stock stock =stockRepository.findOne(code); - if(null==stock){ - mav.addObject("msg",String.format("代码:%s %s",code,"不存在，或者非A股代码!")); - return mav; - } - mav.addObject("stock",stock); - List list=stockService.getStockHisPb(code); - if(null==list||list.size()==0){ - try { - list= stockService.addStockHisPb(code); - } catch (Exception e) { - e.printStackTrace(); + + + @Autowired + private MongoTemplate template; + + @RequestMapping("/stock/info/{code}") + public ModelAndView getStockInfo(@PathVariable String code) throws Exception { + ModelAndView mav = new ModelAndView(template_path + "his_pbpe"); + StringBuilder date = new StringBuilder(); + StringBuilder pb = new StringBuilder(); + StringBuilder pe = new StringBuilder(); + StringBuilder price = new StringBuilder(); + + + Query query = new Query(); + query.addCriteria(Criteria.where("code").is(code)); + List lis = template.find(query, StockHisPbPe.class); + if (null == lis || lis.size() == 0) { + List data = stockService.crawAndSaveHisPbPe(code); + //价格》日期》pb》pe + price.append(data.get(0)); + date.append(data.get(1)); + pb.append(data.get(2)); + pe.append(data.get(3)); + + } else { + for (int i = 0; i < lis.size(); i++) { + StockHisPbPe item = lis.get(i); + date.append("'").append(item.getDate()).append("'"); + pb.append(item.getPb()); + pe.append(item.getPe()); + price.append(item.getPrice()); + if (i != lis.size() - 1) { + date.append(","); + pb.append(","); + pe.append(","); + price.append(","); + } + } } - mav.addObject("rows",list); - StringBuffer year = new StringBuffer(); - StringBuffer pe = new StringBuffer(); - list.stream().forEach(item ->{ - if(item.getPb()>0){ - pe.append(item.getPb()).append(","); - year.append("'").append(item.getDate()).append("',"); - } - }); - String data= String.format("%s|%s",year.toString().replaceAll("\\,$",""), - pe.toString().replaceAll("\\,$","") - ); - String roeItem[]=data.split("\\|"); - mav.addObject("pb_date",roeItem[0]); - mav.addObject("pb_value",roeItem[1]); + mav.addObject("pb", pb.toString()); + mav.addObject("pe", pe.toString()); + mav.addObject("dates", date.toString()); + mav.addObject("price", price.toString()); + mav.addObject("code", code); return mav; } + @RequestMapping("/stock/report/{code}") + public ModelAndView getStockReport(@PathVariable String code) throws Exception { + ModelAndView mav = new ModelAndView(template_path + "report"); + Query query = new Query(); + query.addCriteria(Criteria.where("code").is(code)); + List lis = template.find(query, StockReport.class); + if (null==lis||lis.size()==0){ + List data = stockService.crawAndSaveHisPbPe(code); + JSONArray jsons =JSONArray.parseArray(data.get(4)); + lis=jsons.toJavaList(StockReport.class); + } + + mav.addObject("reports", lis); + return mav; + + } + } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java index 9796bfdd..5c34342a 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/Stock.java @@ -3,8 +3,6 @@ import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; -import java.util.Date; - /** * stock maindata * @@ -23,7 +21,7 @@ public class Stock { private Double fluctuate; private Double todayMax; private Double todayMin; - private Date priceDate; + private Long priceDate; private String industry; private String mainBusiness; private Double totalValue; @@ -33,11 +31,13 @@ public class Stock { private Double pes; private Double ped; private Integer infoDate; - private String dividendDate; + private String dividendDate;//静态分红日期 private Double dividend; - private Integer dividendUpdateDay; + private Integer dividendUpdateDay;//静态分红更新时间 private Double dy;//实时股息率 private Integer dyDate;//实时股息更新时间 + private Double fiveYearDy;//5年平均股息 + private Double fiveYearRoe;//5年平均Roe public String getCode() { @@ -104,11 +104,11 @@ public void setTodayMin(Double todayMin) { this.todayMin = todayMin; } - public Date getPriceDate() { + public Long getPriceDate() { return priceDate; } - public void setPriceDate(Date priceDate) { + public void setPriceDate(Long priceDate) { this.priceDate = priceDate; } @@ -216,6 +216,7 @@ public Integer getDyDate() { public void setDyDate(Integer dyDate) { this.dyDate = dyDate; } + public Integer getInfoDate() { return infoDate; } @@ -223,4 +224,20 @@ public Integer getInfoDate() { public void setInfoDate(Integer infoDate) { this.infoDate = infoDate; } + + public Double getFiveYearDy() { + return fiveYearDy; + } + + public void setFiveYearDy(Double fiveYearDy) { + this.fiveYearDy = fiveYearDy; + } + + public Double getFiveYearRoe() { + return fiveYearRoe; + } + + public void setFiveYearRoe(Double fiveYearRoe) { + this.fiveYearRoe = fiveYearRoe; + } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java index 188cd851..6f35a346 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockCode.java @@ -13,9 +13,15 @@ public class StockCode { @Id private String code; + private Integer hrdud;//history report data update date 历史年度报表更新时间 + private Integer xlsError;//下载xls错误 public StockCode(){} - public StockCode(String code){this.code=code;} + public StockCode(String code){ + this.code=code; + this.hrdud=0; + this.xlsError=0; + } public String getCode() { @@ -26,4 +32,19 @@ public void setCode(String code) { this.code = code; } + public Integer getHrdud() { + return hrdud; + } + + public void setHrdud(Integer hrdud) { + this.hrdud = hrdud; + } + + public Integer getXlsError() { + return xlsError; + } + + public void setXlsError(Integer xlsError) { + this.xlsError = xlsError; + } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockDividend.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockDividend.java index e4c59cf8..b350a83c 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockDividend.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockDividend.java @@ -33,6 +33,7 @@ public class StockDividend { private String gqdjr; private String cxcqr; private String progress; + private String from; public String getId() { return id; @@ -121,4 +122,12 @@ public String getProgress() { public void setProgress(String progress) { this.progress = progress; } + + public String getFrom() { + return from; + } + + public void setFrom(String from) { + this.from = from; + } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java index 7bc712e5..daa571cd 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisDividend.java @@ -14,19 +14,19 @@ public class StockHisDividend { //{"date":"2017-07-07","code":"600519","year":"2016年报","executeDate":"2017-07-01","remark":"10派67.87元(含税)","percent":1.44} @Id - private String _id; + private String id; private String code; private String title; private String executeDate; private String remark; private Double percent; - public String get_id() { - return _id; + public String getId() { + return id; } - public void set_id(String _id) { - this._id = _id; + public void setId(String id) { + this.id = id; } public String getCode() { diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPb.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPbPe.java similarity index 59% rename from src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPb.java rename to src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPbPe.java index 23406f42..ffe2ad54 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPb.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPbPe.java @@ -3,30 +3,38 @@ import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.mapping.Document; -import java.util.Date; - /** - * stock maindata + * ${DESCRIPTION} * * @author chenguoxiang - * @create 2018-03-27 10:15 + * @create 2018-07-03 17:56 **/ -@Document(collection = "stock_his_pb") -public class StockHisPb { +@Document(collection = "stock_his_pe_pb") +public class StockHisPbPe { + /** + * "_id" : ObjectId("5b3b33799d35576680835a60"), + "code" : "601088", + "date" : "2007-10-09", + "pb" : 19.83924, + "pe" : 140.56197, + "price" : 45.3 + * */ @Id - private String _id; + private String id; private String code; - private String date; private Double pb; + private Double pe; private Double price; + private String date; - public String get_id() { - return _id; + + public String getId() { + return id; } - public void set_id(String _id) { - this._id = _id; + public void setId(String id) { + this.id = id; } public String getCode() { @@ -37,14 +45,6 @@ public void setCode(String code) { this.code = code; } - public String getDate() { - return date; - } - - public void setDate(String date) { - this.date = date; - } - public Double getPb() { return pb; } @@ -53,6 +53,14 @@ public void setPb(Double pb) { this.pb = pb; } + public Double getPe() { + return pe; + } + + public void setPe(Double pe) { + this.pe = pe; + } + public Double getPrice() { return price; } @@ -60,4 +68,12 @@ public Double getPrice() { public void setPrice(Double price) { this.price = price; } + + public String getDate() { + return date; + } + + public void setDate(String date) { + this.date = date; + } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPe.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPe.java deleted file mode 100644 index ff312f51..00000000 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisPe.java +++ /dev/null @@ -1,61 +0,0 @@ -package io.github.kingschan1204.istock.module.maindata.po; - -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; - -/** - * stock maindata - * - * @author chenguoxiang - * @create 2018-03-27 10:15 - **/ -@Document(collection = "stock_his_pe") -public class StockHisPe { - - @Id - private String _id; - private String code; - private String date; - private Double pe; - private Double price; - - public String get_id() { - return _id; - } - - public void set_id(String _id) { - this._id = _id; - } - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public String getDate() { - return date; - } - - public void setDate(String date) { - this.date = date; - } - - public Double getPe() { - return pe; - } - - public void setPe(Double pe) { - this.pe = pe; - } - - public Double getPrice() { - return price; - } - - public void setPrice(Double price) { - this.price = price; - } -} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisRoe.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisRoe.java index a619256b..3b1a9d4d 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisRoe.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockHisRoe.java @@ -16,19 +16,19 @@ public class StockHisRoe { @Id - private String _id; + private String id; private String code; private Integer year; private Double roe; private Double roetb; private Date date; - public String get_id() { - return _id; + public String getId() { + return id; } - public void set_id(String _id) { - this._id = _id; + public void setId(String id) { + this.id = id; } public String getCode() { diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java new file mode 100644 index 00000000..ef4d66b7 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/po/StockReport.java @@ -0,0 +1,58 @@ +package io.github.kingschan1204.istock.module.maindata.po; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +/** + * Created by kingschan on 2018/7/6. + */ +@Document(collection = "stock_report") +public class StockReport { + + @Id + private String id; + private String code; + private String releaseDay; + private String link; + private String title; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getReleaseDay() { + return releaseDay; + } + + public void setReleaseDay(String releaseDay) { + this.releaseDay = releaseDay; + } + + public String getLink() { + return link; + } + + public void setLink(String link) { + this.link = link; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/repository/StockHisPbRepository.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/repository/StockHisPbRepository.java deleted file mode 100644 index d8ad98d7..00000000 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/repository/StockHisPbRepository.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.github.kingschan1204.istock.module.maindata.repository; - -import io.github.kingschan1204.istock.module.maindata.po.StockHisPb; -import io.github.kingschan1204.istock.module.maindata.po.StockHisPe; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.stereotype.Repository; - -/** - * - * - * @author chenguoxiang - * @create 2018-03-09 14:55 - **/ -@Repository -public interface StockHisPbRepository extends MongoRepository { - -} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/repository/StockHisPeRepository.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/repository/StockHisPeRepository.java deleted file mode 100644 index 9a73326a..00000000 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/repository/StockHisPeRepository.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.github.kingschan1204.istock.module.maindata.repository; - -import io.github.kingschan1204.istock.module.maindata.po.StockHisPe; -import io.github.kingschan1204.istock.module.maindata.po.StockHisRoe; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.stereotype.Repository; - -/** - * - * - * @author chenguoxiang - * @create 2018-03-09 14:55 - **/ -@Repository -public interface StockHisPeRepository extends MongoRepository { - -} diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java index a7557f75..65409e11 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockCodeService.java @@ -6,9 +6,12 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.util.HashSet; import java.util.List; /** @@ -28,31 +31,37 @@ public class StockCodeService { /** * 保存所有代码 + * * @throws Exception */ public void saveAllStockCode() throws Exception { - List szCodes=stockSpider.getStockCodeBySZ(); - List shCodes=stockSpider.getStockCodeBySH(); - List allcodes= stockSpider.getAllStockCode(); - szCodes.stream().forEach(code-> - mongoTemplate.save(new StockCode(StockSpider.formatStockCode(code))) - ); - shCodes.stream().forEach(code-> - mongoTemplate.save(new StockCode(StockSpider.formatStockCode(code))) - ); - allcodes.stream().forEach(code-> - mongoTemplate.save(new StockCode(code)) + List szCodes = stockSpider.getStockCodeBySZ(); + List shCodes = stockSpider.getStockCodeBySH(); + List allcodes = stockSpider.getAllStockCode(); + HashSet codes = new HashSet(); + codes.addAll(szCodes); + codes.addAll(shCodes); + codes.addAll(allcodes); + codes.stream().forEach(code -> { + mongoTemplate.upsert( + new Query(Criteria.where("_id").is(code)), + new Update().set("_id", code).set("hrdud",0).set("xlsError",0), + "stock_code" + ); + } ); + } /** * 返回所有代码 + * * @return */ - public List getAllStockCodes(){ - return mongoTemplate.find(new Query(),StockCode.class); + public List getAllStockCodes() { + return mongoTemplate.find(new Query(), StockCode.class); } diff --git a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java index 936d260b..58afaded 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java +++ b/src/main/java/io.github.kingschan1204.istock/module/maindata/services/StockService.java @@ -4,12 +4,15 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.mongodb.BasicDBObject; +import com.mongodb.DBCollection; import com.mongodb.DBObject; import io.github.kingschan1204.istock.common.util.stock.StockSpider; -import io.github.kingschan1204.istock.module.maindata.po.*; +import io.github.kingschan1204.istock.common.util.stock.impl.JisiluSpilder; +import io.github.kingschan1204.istock.module.maindata.po.Stock; +import io.github.kingschan1204.istock.module.maindata.po.StockDividend; +import io.github.kingschan1204.istock.module.maindata.po.StockHisDividend; +import io.github.kingschan1204.istock.module.maindata.po.StockHisRoe; import io.github.kingschan1204.istock.module.maindata.repository.StockHisDividendRepository; -import io.github.kingschan1204.istock.module.maindata.repository.StockHisPbRepository; -import io.github.kingschan1204.istock.module.maindata.repository.StockHisPeRepository; import io.github.kingschan1204.istock.module.maindata.repository.StockRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Sort; @@ -37,13 +40,11 @@ public class StockService { @Autowired private StockHisRoeService stockHisRoeService; @Autowired - private StockHisPbRepository stockHisPbRepository; - @Autowired - private StockHisPeRepository stockHisPeRepository; - @Autowired private StockSpider spider; @Autowired private MongoTemplate template; + @Autowired + private JisiluSpilder jisiluSpilder; /** * add stock code @@ -73,49 +74,18 @@ public void addStock(String... codes) throws Exception { //save dividend List stockHisDividendList = JSONArray.parseArray(dividends.toJSONString(),StockHisDividend.class); template.remove(new Query(Criteria.where("code").is(scode)),StockHisDividend.class); -// stockHisDividendRepository.save(stockHisDividendList); } json.put("dividend",percent); json.put("dividendDate",date); json.putAll(info); // his roe stockHisRoeService.addStockHisRoe(scode); - //his pb - addStockHisPb(scode); - //his pe - addStockHisPe(scode); } List list = JSON.parseArray(jsons.toJSONString(), Stock.class); stockRepository.save(list); } - /** - * 增加历史pe - * @param code - * @throws Exception - */ - public List addStockHisPe(String code)throws Exception{ - JSONArray jsons=spider.getHistoryPE(StockSpider.formatStockCode(code)); - List lis = JSON.parseArray(jsons.toJSONString(),StockHisPe.class); - template.remove(new Query(Criteria.where("code").is(code)),StockHisPe.class); - stockHisPeRepository.save(lis); - return lis; - } - - /** - * 增加历史pb - * @param code - * @throws Exception - */ - public List addStockHisPb(String code)throws Exception{ - JSONArray jsons=spider.getHistoryPB(StockSpider.formatStockCode(code)); - List lis = JSON.parseArray(jsons.toJSONString(),StockHisPb.class); - template.remove(new Query(Criteria.where("code").is(code)),StockHisPb.class); - stockHisPbRepository.save(lis); - return lis; - - } public String queryStock(int pageindex, int pagesize, final String pcode,final String type,String pb,String dy, String orderfidld, String psort){ @@ -148,10 +118,11 @@ public String queryStock(int pageindex, int pagesize, final String pcode,final S double d = Double.parseDouble(pb.split("-")[1]); query.addCriteria(Criteria.where("pb").gte(s).lte(d)); } - if(null!=dy&&dy.matches("\\d+(\\.\\d+)?\\-\\d+(\\.\\d+)?")){ - double s = Double.parseDouble(dy.split("-")[0]); - double d = Double.parseDouble(dy.split("-")[1]); - query.addCriteria(Criteria.where("dy").gte(s).lte(d)); + if(null!=dy&&dy.matches("(dy|dividend|fiveYearDy)\\-\\d+(\\.\\d+)?\\-\\d+(\\.\\d+)?")){ + String field=dy.split("-")[0]; + double s = Double.parseDouble(dy.split("-")[1]); + double d = Double.parseDouble(dy.split("-")[2]); + query.addCriteria(Criteria.where(field).gte(s).lte(d)); } //记录总数 Long total=template.count(query,Stock.class); @@ -160,7 +131,7 @@ public String queryStock(int pageindex, int pagesize, final String pcode,final S //排序 List orders = new ArrayList(); //排序 orders.add(new Sort.Order( - psort.equalsIgnoreCase("asc")?Sort.Direction.ASC:Sort.Direction.DESC + "asc".equalsIgnoreCase(psort) ?Sort.Direction.ASC:Sort.Direction.DESC ,orderfidld)); Sort sort = new Sort(orders); query.with(sort); @@ -172,21 +143,29 @@ public String queryStock(int pageindex, int pagesize, final String pcode,final S temp =jsons.getJSONObject(i); temp.put("fluctuate",temp.getString("fluctuate")+"%"); - if(temp.containsKey("roe")&&temp.getDouble("roe")!=-1){ - temp.put("roe",temp.getString("roe")+"%"); + if(temp.containsKey("roe")){ + if(temp.getDouble("roe")!=-1){ + temp.put("roe",temp.getString("roe")+"%"); + }else { + temp.put("roe","--"); + } + } if(temp.containsKey("totalValue")&&temp.getDouble("totalValue")!=-1){ temp.put("totalValue",temp.getString("totalValue")+"亿"); } - if(temp.containsKey("dividend")&&temp.getDouble("pb")==-1){ + if(temp.containsKey("pb")&&temp.getDouble("pb")==-1){ temp.put("pb","--"); } - if(temp.containsKey("dividend")&&temp.getDouble("pes")==-1){ + if(temp.containsKey("pes")&&temp.getDouble("pes")==-1){ temp.put("pes","--"); } if(temp.containsKey("ped")&&temp.getDouble("ped")==-1){ temp.put("ped","--"); } + if(temp.containsKey("bvps")&&temp.getDouble("bvps")==-1){ + temp.put("bvps","--"); + } if(temp.containsKey("dividend")){ if(temp.getDouble("dividend")!=-1){ temp.put("dividend",temp.getString("dividend")+"%"); @@ -197,6 +176,12 @@ public String queryStock(int pageindex, int pagesize, final String pcode,final S if(temp.containsKey("dy")){ temp.put("dy",temp.getString("dy")+"%"); } + if(temp.containsKey("fiveYearDy")){ + temp.put("fiveYearDy",temp.getString("fiveYearDy")+"%"); + } + if(temp.containsKey("fiveYearRoe")){ + temp.put("fiveYearRoe",temp.getString("fiveYearRoe")+"%"); + } } @@ -243,33 +228,77 @@ public List getStockHisRoe(String code){ } - public List getStockHisPb(String code){ - Query query = new Query(); - query.addCriteria(Criteria.where("code").is(code)); - //排序 - List orders = new ArrayList(); //排序 - orders.add(new Sort.Order(Sort.Direction.ASC,"date")); - Sort sort = new Sort(orders); - query.with(sort); - //code - List list =template.find(query,StockHisPb.class); - return list; - } + /** + * 抓取历史数据 + * @param code + * @return + * @throws Exception + */ + public List crawAndSaveHisPbPe(String code)throws Exception{ - public List getStockHisPe(String code){ - Query query = new Query(); - query.addCriteria(Criteria.where("code").is(code)); - //排序 - List orders = new ArrayList(); //排序 - orders.add(new Sort.Order(Sort.Direction.ASC,"date")); - Sort sort = new Sort(orders); - query.with(sort); - //code - List list =template.find(query,StockHisPe.class); - return list; - } + StringBuilder price = new StringBuilder(); + StringBuilder pe = new StringBuilder(); + StringBuilder pb = new StringBuilder(); + StringBuilder date = new StringBuilder(); + JSONObject data= jisiluSpilder.crawHisPbPePriceAndReports(code); + List list = new ArrayList<>(); + DBCollection hisdata = template.getCollection("stock_his_pe_pb"); + DBCollection report = template.getCollection("stock_report"); + JSONArray hisdataJsons=data.getJSONArray("hisdata"); + for (int i = 0; i (); + for (int i = 0; i result = new ArrayList<>(); + result.add(price.toString()); + result.add(date.toString()); + result.add(pb.toString()); + result.add(pe.toString()); + result.add(reportJsons.toJSONString()); + return result; + } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java b/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java new file mode 100644 index 00000000..0180195b --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/startup/InitQuartzTaskRunner.java @@ -0,0 +1,70 @@ +package io.github.kingschan1204.istock.module.startup; + +import io.github.kingschan1204.istock.common.util.quartz.QuartzManager; +import io.github.kingschan1204.istock.module.task.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.core.Ordered; + +/** + * + * @author chenguoxiang + * @create 2018-07-13 15:12 + **/ +public class InitQuartzTaskRunner implements ApplicationRunner, Ordered { + + @Autowired + private QuartzManager quartzManager; + + @Override + public void run(ApplicationArguments applicationArguments) throws Exception { + quartzManager.addJob("sinaPriceTask", + "sinaPriceTask-group", + "sinaPriceTask-trigger", + "sinaPriceTask-trigger-group", + SinaStockPriceTask.class, + "0 0/2 * * * ?"); + + quartzManager.addJob("stockCodeTask", + "stockCodeTask-group", + "stockCodeTask-trigger", + "stockCodeTask-trigger-group", + StockCodeTask.class, + "0 0 0 * * ?"); + + quartzManager.addJob("stockDividendTask", + "stockDividendTask-group", + "stockDividendTask-trigger", + "stockDividendTask-trigger-group", + StockDividendTask.class, + "*/6 * * * * ?"); + + + quartzManager.addJob("hisRepoartTask", + "hisRepoartTask-group", + "hisRepoartTask-trigger", + "hisRepoartTask-trigger-group", + ThsHisYearReportTask.class, + "*/6 * * * * ?"); + + quartzManager.addJob("stockInfoTask", + "stockInfoTask-group", + "stockInfoTask-trigger", + "stockInfoTask-trigger-group", + ThsStockInfoTask.class, + "*/6 * * * * ?"); + + quartzManager.addJob("xueqiuDyTask", + "xueqiuDyTask-group", + "xueqiuDyTask-trigger", + "xueqiuDyTask-trigger-group", + XueQiuStockDyTask.class, + "0 0/1 * * * ?"); + } + + @Override + public int getOrder() { + return 1; + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java index 7041fb84..239663c7 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/SinaStockPriceTask.java @@ -7,6 +7,9 @@ import io.github.kingschan1204.istock.module.maindata.po.Stock; import io.github.kingschan1204.istock.module.maindata.po.StockCode; import io.github.kingschan1204.istock.module.maindata.services.StockCodeService; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,7 +30,7 @@ * @create 2018-03-29 14:50 **/ @Component -public class SinaStockPriceTask { +public class SinaStockPriceTask implements Job { private Logger log = LoggerFactory.getLogger(SinaStockPriceTask.class); @@ -38,8 +41,12 @@ public class SinaStockPriceTask { @Autowired private StockCodeService stockCodeService; - @Scheduled(cron = "0 0/2 * * * ?") - public void stockPriceExecute() throws Exception { +// @Scheduled(cron = "0 0/2 * * * ?") + /* public void stockPriceExecute() throws Exception { + }*/ + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { if (!StockDateUtil.stockOpenTime()) { return; } @@ -61,11 +68,8 @@ public void stockPriceExecute() throws Exception { } } - - } - public void updateStockPrice(List codes) throws Exception { JSONArray jsons = spider.getStockPrice(codes.toArray(new String[]{})); List stocks = JSON.parseArray(jsons.toJSONString(), Stock.class); @@ -86,4 +90,6 @@ public void updateStockPrice(List codes) throws Exception { ); }); } + + } diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java index a22f102c..ee5d94e2 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/StockCodeTask.java @@ -2,6 +2,9 @@ import io.github.kingschan1204.istock.common.util.stock.StockSpider; import io.github.kingschan1204.istock.module.maindata.services.StockCodeService; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -13,7 +16,7 @@ * */ @Component -public class StockCodeTask { +public class StockCodeTask implements Job{ private Logger log = LoggerFactory.getLogger(StockCodeTask.class); @@ -24,10 +27,20 @@ public class StockCodeTask { @Autowired private StockCodeService stockCodeService; - @Scheduled(cron = "0 0 0 * * ?") +/*// @Scheduled(cron = "0 0 0 * * ?") public void stockCodeExecute()throws Exception{ + + }*/ + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { Long start =System.currentTimeMillis(); - stockCodeService.saveAllStockCode(); + try { + stockCodeService.saveAllStockCode(); + } catch (Exception e) { + log.error("代码更新错误：{}",e); + e.printStackTrace(); + } log.info(String.format("更新代码共耗时：%s ms",(System.currentTimeMillis()-start))); } } diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java index 4ea135eb..f155a6cf 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/StockDividendTask.java @@ -3,10 +3,14 @@ import com.alibaba.fastjson.JSONArray; import com.mongodb.WriteResult; import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; +import io.github.kingschan1204.istock.common.util.stock.impl.DefaultSpiderImpl; import io.github.kingschan1204.istock.common.util.stock.impl.EastmoneySpider; import io.github.kingschan1204.istock.module.maindata.po.Stock; import io.github.kingschan1204.istock.module.maindata.po.StockDividend; import io.github.kingschan1204.istock.module.maindata.repository.StockHisDividendRepository; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -18,9 +22,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; /** * 定时更新分红情况 @@ -29,80 +31,101 @@ * @create 2018-03-29 14:50 **/ @Component -public class StockDividendTask { +public class StockDividendTask implements Job { private Logger log = LoggerFactory.getLogger(StockDividendTask.class); @Resource(name = "EastmoneySpider") - private EastmoneySpider spider; + private EastmoneySpider eastmoneySpider; + @Autowired + private DefaultSpiderImpl defaultSpider; @Autowired private MongoTemplate template; @Autowired private StockHisDividendRepository stockHisDividendRepository; - @Scheduled(cron = "*/6 * * * * ?") - public void stockDividendExecute() throws Exception { - /*if (!StockDateUtil.stockOpenTime()) { - return; - }*/ +// @Scheduled(cron = "*/6 * * * * ?") + /* public void stockDividendExecute() throws Exception { + + }*/ - Integer dateNumber = StockDateUtil.getCurrentDateNumber()-5; - Criteria cr = new Criteria(); - Criteria c1 = Criteria.where("dividendUpdateDay").lt(dateNumber);//小于 （5天更新一遍） - Criteria c2 = Criteria.where("dividendUpdateDay").exists(false); - Query query = new Query(cr.orOperator(c1, c2)); - query.limit(3); - List list = template.find(query, Stock.class); - for (Stock stock:list) { - Long start =System.currentTimeMillis();//记录开始时间 - int affected=0;//分红更新记录条数 - JSONArray dividends=new JSONArray(); - String date=""; - Double percent=0D; - try { - dividends=spider.getHistoryDividendRate(stock.getCode()); - if(null!=dividends&÷nds.size()>0){ - for (int j = 0; j < dividends.size(); j++) { - if(dividends.getJSONObject(j).getDouble("percent")>0){ - percent=dividends.getJSONObject(j).getDoubleValue("percent"); - date=dividends.getJSONObject(j).getString("cxcqr"); - break; + /** + * 同花顺和东方财富的并集 + * @param code + * @return + * @throws Exception + */ + public JSONArray combineHisDy(String code)throws Exception{ + JSONArray ths =defaultSpider.getHistoryDividendRate(code); + JSONArray east =eastmoneySpider.getHistoryDividendRate(code); + if(null==east){ + east=new JSONArray(); + } + if(null!=ths){ + east.addAll(ths); + } + JSONArray ret=new JSONArray(); + Set titles=new HashSet(); + for (int i = 0; i < east.size(); i++) { + String title =east.getJSONObject(i).getString("title"); + if(!titles.contains(title)){ + ret.add(east.getJSONObject(i)); + titles.add(title); + } + } + return ret; + } + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + Integer dateNumber = StockDateUtil.getCurrentDateNumber()-3; + Criteria cr = new Criteria(); + Criteria c1 = Criteria.where("dividendUpdateDay").lt(dateNumber);//小于 （3天更新一遍） + Criteria c2 = Criteria.where("dividendUpdateDay").exists(false); + Query query = new Query(cr.orOperator(c1, c2)); + query.limit(3); + List list = template.find(query, Stock.class); + for (Stock stock:list) { + Long start =System.currentTimeMillis();//记录开始时间 + int affected=0;//分红更新记录条数 + JSONArray dividends=new JSONArray(); + String date=""; + Double percent=0D; + try { + dividends=combineHisDy(stock.getCode()); + if(null!=dividends&÷nds.size()>0){ + for (int j = 0; j < dividends.size(); j++) { + double tempPercent=dividends.getJSONObject(j).getDouble("percent"); + String tempDate=dividends.getJSONObject(j).getString("cxcqr"); + if(tempPercent>=0&&!"-".equals(tempDate)){ + percent=tempPercent; + date=tempDate; + break; + } } + //save dividend + List stockDividendList = JSONArray.parseArray(dividends.toJSONString(),StockDividend.class); + template.remove(new Query(Criteria.where("code").is(stock.getCode())),StockDividend.class); + stockHisDividendRepository.save(stockDividendList); + affected=stockDividendList.size(); } - //save dividend - List stockDividendList = JSONArray.parseArray(dividends.toJSONString(),StockDividend.class); - template.remove(new Query(Criteria.where("code").is(stock.getCode())),StockDividend.class); - stockHisDividendRepository.save(stockDividendList); - affected=stockDividendList.size(); + } catch (Exception e) { + log.error("error:{}",e); + e.printStackTrace(); } - } catch (Exception e) { - log.error("error:{}",e); - e.printStackTrace(); + WriteResult wr =template.upsert( + new Query(Criteria.where("_id").is(stock.getCode())), + new Update() + .set("_id", stock.getCode()) + .set("dividendDate", date) + .set("dividend", percent) + .set("dividendUpdateDay", dateNumber), + "stock" + ); + affected+=wr.getN(); + log.info("{}分红抓取,耗时{},获得{}行数据",stock.getCode(),(System.currentTimeMillis()-start),affected); } - WriteResult wr =template.upsert( - new Query(Criteria.where("_id").is(stock.getCode())), - new Update() - .set("_id", stock.getCode()) - .set("dividendDate", date) - .set("dividend", percent) - .set("dividendUpdateDay", dateNumber), - "stock" - ); - affected+=wr.getN(); - log.info("{}分红抓取,耗时{},获得{}行数据",stock.getCode(),(System.currentTimeMillis()-start),affected); - } - - } - public static void main(String[] args) { - List lis = null;//new ArrayList<>(); - if (Optional.of(lis).isPresent()) - { - System.out.println("ok"); - }else { - System.out.println("ng"); - } } - } diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java new file mode 100644 index 00000000..393bd131 --- /dev/null +++ b/src/main/java/io.github.kingschan1204.istock/module/task/ThsHisYearReportTask.java @@ -0,0 +1,113 @@ +package io.github.kingschan1204.istock.module.task; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import io.github.kingschan1204.istock.common.util.cache.EhcacheUtil; +import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; +import io.github.kingschan1204.istock.common.util.stock.StockSpider; +import io.github.kingschan1204.istock.module.maindata.po.StockCode; +import io.github.kingschan1204.istock.module.maindata.po.StockHisRoe; +import io.github.kingschan1204.istock.module.maindata.repository.StockHisRoeRepository; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.List; + +/** + * 爬取历史roe + * @author chenguoxiang + * @create 2018-07-09 17:28 + **/ +@Component +public class ThsHisYearReportTask implements Job{ + private Logger log = LoggerFactory.getLogger(ThsHisYearReportTask.class); + + @Autowired + private StockSpider spider; + @Autowired + private MongoTemplate template; + @Autowired + private StockHisRoeRepository stockHisRoeRepository; + @Autowired + EhcacheUtil ehcacheUtil; + final String cacheName="ThsHisYearReportTask"; + + /** + * 是否错误次数过多，停止任务 + * @return + */ + boolean stopTask(){ + return getErrorTotal()>3; + } + + /** + * 得到执行错误的记录 + * @return + */ + int getErrorTotal(){ + Object value =ehcacheUtil.getKey(cacheName,"error"); + int val=null==value?0:(int)value; + return val; + } + + +// @Scheduled(cron = "*/6 * * * * ?") + /* public void execute() throws Exception { + }*/ + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + Long start = System.currentTimeMillis(); + if(stopTask()){ + log.info("错误次数过多，不执行任务!"); + return; + } + Integer dateNumber = StockDateUtil.getCurrentDateNumber(); + Criteria cr = new Criteria(); + //3天更新一把 + Criteria c1 = Criteria.where("hrdud").lt(dateNumber-3); + Criteria c2 = Criteria.where("xlsError").is(0); + Query query = new Query(cr.andOperator(c1,c2)); + query.limit(2); + List list = template.find(query, StockCode.class); + if(null==list||list.size()==0){ + log.info("his year report data 当前已全部更新完!"); + return ; + } + list.stream().forEach(code ->{ + JSONArray jsons= null; + String codestr=code.getCode().replaceAll("\\D",""); + try { + jsons = spider.getHistoryROE(codestr); + List lis = JSON.parseArray(jsons.toJSONString(),StockHisRoe.class); + template.remove(new Query(Criteria.where("code").is(codestr)),StockHisRoe.class); + stockHisRoeRepository.save(lis); + template.upsert( + new Query(Criteria.where("_id").is(code.getCode())), + new Update().set("hrdud", dateNumber), + "stock_code" + ); + } catch (Exception e) { + e.printStackTrace(); + ehcacheUtil.addKey(cacheName,"error",getErrorTotal()+1); + template.upsert( + new Query(Criteria.where("_id").is(code.getCode())),new Update().set("xlsError", 1), + "stock_code" + ); + } + }); + log.info(String.format("download xls and update data use ：%s ms ", (System.currentTimeMillis() - start))); + //end + } +} diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java index 916dbb3a..eb96ca37 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/ThsStockInfoTask.java @@ -6,6 +6,9 @@ import io.github.kingschan1204.istock.common.util.stock.StockSpider; import io.github.kingschan1204.istock.module.maindata.po.Stock; import io.github.kingschan1204.istock.module.maindata.repository.StockHisDividendRepository; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -25,7 +28,7 @@ * @create 2018-03-29 14:50 **/ @Component -public class ThsStockInfoTask { +public class ThsStockInfoTask implements Job{ private Logger log = LoggerFactory.getLogger(ThsStockInfoTask.class); @@ -36,8 +39,13 @@ public class ThsStockInfoTask { @Autowired private StockHisDividendRepository stockHisDividendRepository; - @Scheduled(cron = "*/6 * * * * ?") - public void stockInfoExecute() throws Exception { +// @Scheduled(cron = "*/6 * * * * ?") + /*public void stockInfoExecute() throws Exception { + + }*/ + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { if (!StockDateUtil.stockOpenTime()) { return; } @@ -59,7 +67,7 @@ public void stockInfoExecute() throws Exception { try { JSONObject info = spider.getStockInfo(stock.getCode()); item = info.toJavaObject(Stock.class); - if (null == item) return; + if (null == item) {return;} WriteResult wr = template.upsert( new Query(Criteria.where("_id").is(stock.getCode())), new Update() @@ -82,7 +90,6 @@ public void stockInfoExecute() throws Exception { ; } } - log.info(String.format("info更新耗时：%s ms ,影响行: %s", (System.currentTimeMillis() - start),affected)); + log.info(String.format("craw stock info and update data use ：%s ms ,affected rows : %s", (System.currentTimeMillis() - start),affected)); } - } diff --git a/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java b/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java index 3b793206..a1997b6a 100644 --- a/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java +++ b/src/main/java/io.github.kingschan1204.istock/module/task/XueQiuStockDyTask.java @@ -3,11 +3,15 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.mongodb.WriteResult; +import io.github.kingschan1204.istock.common.util.cache.EhcacheUtil; import io.github.kingschan1204.istock.common.util.stock.StockDateUtil; import io.github.kingschan1204.istock.common.util.stock.StockSpider; import io.github.kingschan1204.istock.module.maindata.po.Stock; import io.github.kingschan1204.istock.module.maindata.po.StockDyQueue; import io.github.kingschan1204.istock.module.maindata.repository.StockDyQueueRepository; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,7 +31,7 @@ * @create 2018-03-29 14:50 **/ @Component -public class XueQiuStockDyTask { +public class XueQiuStockDyTask implements Job{ private Logger log = LoggerFactory.getLogger(XueQiuStockDyTask.class); @@ -37,14 +41,62 @@ public class XueQiuStockDyTask { private MongoTemplate template; @Autowired private StockDyQueueRepository stockDyQueueRepository; + @Autowired + EhcacheUtil ehcacheUtil; + final String cacheName="XueQiuStockDyTask"; + + /** + * 是否错误次数过多，停止任务 + * @return + */ + boolean stopTask(){ + return getErrorTotal()>5; + } + + /** + * 得到执行错误的记录 + * @return + */ + int getErrorTotal(){ + Object value =ehcacheUtil.getKey(cacheName,"error"); + int val=null==value?0:(int)value; + return val; + } + +// @Scheduled(cron = "0 0/1 * * * ?") + /* public void stockDividendExecute() throws Exception { + }*/ + + public void uptateDy(JSONObject data) { + int affected = 0;//受影响行 + Integer dateNumber = StockDateUtil.getCurrentDateNumber(); + JSONArray rows = data.getJSONArray("list"); + List list = rows.toJavaList(Stock.class); + for (Stock stock : list) { + WriteResult wr = template.updateFirst( + new Query(Criteria.where("_id").is(stock.getCode())), + new Update() + .set("dy", stock.getDy()) + .set("dyDate", dateNumber), + "stock" + ); + affected += wr.getN(); + } + log.info("dy 批处理：共{}条，本次更新{}条", list.size(), affected); - @Scheduled(cron = "0 0/1 * * * ?") - public void stockDividendExecute() throws Exception { + } + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { int day = StockDateUtil.getCurrentWeekDay(); if (day == 6 || day == 0) { log.debug("非交易时间不执行操作..."); return; } + if(stopTask()){ + log.error("错误次数过多，不执行任务!"); + return; + } Long start = System.currentTimeMillis(); List list = template.find( new Query(Criteria.where("date").is(StockDateUtil.getCurrentDateNumber())), StockDyQueue.class @@ -71,28 +123,9 @@ public void stockDividendExecute() throws Exception { template.save(stockDyQueue, "stock_dy_queue"); log.info("dy更新第{}页,共{}页 耗时：{} ms", pageindex, totalpage, (System.currentTimeMillis() - start)); } catch (Exception ex) { + ehcacheUtil.addKey(cacheName,"error",getErrorTotal()+1); log.error("dy 出错了:{}", ex); ex.printStackTrace(); } - - } - - public void uptateDy(JSONObject data) { - int affected = 0;//受影响行 - Integer dateNumber = StockDateUtil.getCurrentDateNumber(); - JSONArray rows = data.getJSONArray("list"); - List list = rows.toJavaList(Stock.class); - for (Stock stock : list) { - WriteResult wr = template.updateFirst( - new Query(Criteria.where("_id").is(stock.getCode())), - new Update() - .set("dy", stock.getDy()) - .set("dyDate", dateNumber), - "stock" - ); - affected += wr.getN(); - } - log.info("dy 批处理：共{}条，本次更新{}条", list.size(), affected); - } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4bfeaed4..040a655a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -16,20 +16,22 @@ spring.jackson.serialization.indent_output=true spring.mvc.favicon.enabled=false spring.data.mongodb.database=istock -#192.168.10.240 +#192.168.10.175 #127.0.0.1 -spring.data.mongodb.host=192.168.10.240 +spring.data.mongodb.host=127.0.0.1 spring.data.mongodb.port=27017 #spider spider.timeout=8000 spider.useagent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3346.9 Safari/537.36 ## token -xueqiu.token=229a3a53d49b5d0078125899e528279b0e54b5fe +xueqiu.token= eastmoney.token=70f12f2f4f091e459a279469fe49eca5 #security management.context-path=/admin management.security.enabled=false management.security.roles=SUPERUSER -security.user.name=kingschan -security.user.password=password \ No newline at end of file +security.user.name=root +security.user.password= +###ehcache +spring.cache.ehcache.config=classpath:ehcache.xml diff --git a/src/main/resources/ehcache.xml b/src/main/resources/ehcache.xml new file mode 100644 index 00000000..06f61fa3 --- /dev/null +++ b/src/main/resources/ehcache.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + diff --git a/src/main/resources/mapreduce/5years_dy/dy5years_map.js b/src/main/resources/mapreduce/5years_dy/dy5years_map.js new file mode 100644 index 00000000..4b325b54 --- /dev/null +++ b/src/main/resources/mapreduce/5years_dy/dy5years_map.js @@ -0,0 +1,3 @@ +function () { + emit(this.code, {percent: this.percent, title: this.title}) +} \ No newline at end of file diff --git a/src/main/resources/mapreduce/5years_dy/dy5years_reduce.js b/src/main/resources/mapreduce/5years_dy/dy5years_reduce.js new file mode 100644 index 00000000..13ee00e2 --- /dev/null +++ b/src/main/resources/mapreduce/5years_dy/dy5years_reduce.js @@ -0,0 +1,15 @@ +function(key, values) { + var total=0; + var index=0; + var years=""; + for (var i = 0; i < values.length; i++) { + var title =values[i].title; + total+=values[i].percent; + index++; + years+=title+","; + } + if(index<5){ + total=0; + } + return {percent:(total/index).toFixed(2),years:years,size:index}; +} \ No newline at end of file diff --git a/src/main/resources/mapreduce/5years_roe/map.js b/src/main/resources/mapreduce/5years_roe/map.js new file mode 100644 index 00000000..2e9d5ecb --- /dev/null +++ b/src/main/resources/mapreduce/5years_roe/map.js @@ -0,0 +1,3 @@ +function () { + emit(this.code, {percent: this.roe, title: this.year}) +} \ No newline at end of file diff --git a/src/main/resources/mapreduce/5years_roe/reduce.js b/src/main/resources/mapreduce/5years_roe/reduce.js new file mode 100644 index 00000000..1e660bc8 --- /dev/null +++ b/src/main/resources/mapreduce/5years_roe/reduce.js @@ -0,0 +1,16 @@ +function(key, values) { + var total=0; + var index=0; + var years=""; + for (var i = 0; i < values.length; i++) { + var title =values[i].title; + total+=values[i].percent; + index++; + years+=title+","; + } + if(index<5){ + total=0; + } + var percent=(total/index).toFixed(2); + return {percent:parseFloat(percent),years:years,size:index}; +} \ No newline at end of file diff --git a/src/main/resources/templates/index.html b/src/main/resources/templates/index.html index b189124b..ce03de80 100644 --- a/src/main/resources/templates/index.html +++ b/src/main/resources/templates/index.html @@ -1,229 +1,245 @@ <#compress> - - - - - 股票筛选 - - - - - - - - - -
-
-
-
-
-
$
- + + + + + 股票筛选 + + + + + +
+
+ +
+
+
$
+ +
-
-
-
- +
+
+ +
-
-
-
-
PB
- +
+
+
PB
+ +
-
-
-
-
股息(动)
- +
+
+ +
-
- - +
+
+ +
+
+ + - -
- - -
- + +
+ + +
+ +
-
-
-
+
+
- - - + + + + + + \ No newline at end of file diff --git a/src/main/resources/templates/stock/his/his_dy.html b/src/main/resources/templates/stock/his/his_dy.html index bbb4d23b..8834f8f9 100644 --- a/src/main/resources/templates/stock/his/his_dy.html +++ b/src/main/resources/templates/stock/his/his_dy.html @@ -30,6 +30,7 @@

${msg}

股息率 除权除息日 方案进度 + 来源 <#list rows as row> @@ -40,6 +41,7 @@

${msg}

${row.percent}% ${row.cxcqr} ${row.progress} + ${row.from} diff --git a/src/main/resources/templates/stock/his/his_pb.html b/src/main/resources/templates/stock/his/his_pb.html deleted file mode 100644 index 71669969..00000000 --- a/src/main/resources/templates/stock/his/his_pb.html +++ /dev/null @@ -1,88 +0,0 @@ - - - - - 量化分析- - <#if stock??>${stock.name} - <#else>${code!""} - </#if> - 市净率 - - - - - - - -<#if msg??> -

${msg}

- - -
- - - - \ No newline at end of file diff --git a/src/main/resources/templates/stock/his/his_pbpe.html b/src/main/resources/templates/stock/his/his_pbpe.html new file mode 100644 index 00000000..865fba1e --- /dev/null +++ b/src/main/resources/templates/stock/his/his_pbpe.html @@ -0,0 +1,219 @@ + + + + + ${code}历史数据 + + + + + + + +
+
+
+ + + + + + + + \ No newline at end of file diff --git a/src/main/resources/templates/stock/his/his_pe.html b/src/main/resources/templates/stock/his/his_pe.html deleted file mode 100644 index b43a55bf..00000000 --- a/src/main/resources/templates/stock/his/his_pe.html +++ /dev/null @@ -1,91 +0,0 @@ - - - - - 量化分析- - <#if stock??>${stock.name} - <#else>${code!""} - </#if> - 市盈率 - - - - - - - -<#if msg??> -

${msg}

- - -
- - - - - - - \ No newline at end of file diff --git a/src/main/resources/templates/stock/his/report.html b/src/main/resources/templates/stock/his/report.html new file mode 100644 index 00000000..fe48253b --- /dev/null +++ b/src/main/resources/templates/stock/his/report.html @@ -0,0 +1,28 @@ + + + + + 报表 + + + + + + +
+ + + + + + + <#list reports as row> + + + + + +
报告期披露日期
${row.title}${row.releaseDay}
+
+ + \ No newline at end of file diff --git a/src/test/java/io/github/kingschan1204/istock/test/DefaultSpiderImplTest.java b/src/test/java/io/github/kingschan1204/istock/test/DefaultSpiderImplTest.java index d82228f7..9ae973c9 100644 --- a/src/test/java/io/github/kingschan1204/istock/test/DefaultSpiderImplTest.java +++ b/src/test/java/io/github/kingschan1204/istock/test/DefaultSpiderImplTest.java @@ -68,16 +68,6 @@ public void getHistoryROE() throws Exception { System.out.println(jsons.getJSONObject(0)); System.out.println(jsons); } - @Ignore - @Test - public void getHistoryPE() throws Exception { - System.out.println(spider.getHistoryPE(code).toJSONString()); - } - @Ignore - @Test - public void getHistoryPB() throws Exception { - System.out.println(spider.getHistoryPB(code).toJSONString()); - } @Ignore @Test