diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4bc1c04..b98e168 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -13,6 +13,12 @@ jobs: build_deploy: runs-on: ubuntu-latest + env: + TEST_BUCKET_NAME: ${{ secrets.TEST_BUCKET_NAME }} + TEST_KEY_OPTION_CSV: ${{ secrets.TEST_KEY_OPTION_CSV }} + TEST_KEY_FEAR_GREED_JSON: ${{ secrets.TEST_KEY_FEAR_GREED_JSON }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - name: Set Timezone diff --git a/.gitignore b/.gitignore index cf2cf55..8698b03 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ target/ .sts4-cache ### IntelliJ IDEA ### +.run .idea *.iws *.iml @@ -36,4 +37,7 @@ build/ ### Local DBs ### *.db -venv \ No newline at end of file +venv + +src/main/resources/data/fear-greed-files/ +fear-greed-json/ \ No newline at end of file diff --git a/pom.xml b/pom.xml index 263a77f..1f23d12 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.5.6 + 2.6.1 com.dpgrandslam @@ -15,15 +15,28 @@ MicroService for getting stock and options data 11 - 11.7 - 1.14.3 + 11.8 + 1.15.3 1.9.3 - 3.0.4 + 3.0.6 2.0.1.Final 3.10.2 7.0.0 0.80 + 2.15.0 + 2.2.6.RELEASE + + + + com.amazonaws + aws-java-sdk-bom + 1.12.186 + pom + import + + + org.springframework.boot @@ -45,6 +58,10 @@ org.springframework.boot spring-boot-starter-web + + org.springframework.batch + spring-batch-core + org.liquibase liquibase-core @@ -172,9 +189,14 @@ org.ehcache ehcache - 3.9.6 + 3.10.0 runtime + + com.amazonaws + aws-java-sdk-s3 + + @@ -201,6 +223,9 @@ check + + com/dpgrandslam/stockdataservice/domain/util/* + PACKAGE @@ -216,6 +241,7 @@ **/util* **/tiingo* + **/error* @@ -260,17 +286,17 @@ org.springframework spring-beans - 5.3.13 + 5.3.18 org.springframework.data spring-data-jpa - 2.6.0 + 2.6.3 org.yaml snakeyaml - 1.29 + 1.33 javax.validation @@ -291,7 +317,7 @@ org.apache.maven.plugins maven-surefire-report-plugin - 3.0.0-M5 + 3.0.0-M7 diff --git a/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/JobController.java b/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/JobController.java new file mode 100644 index 0000000..133e302 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/JobController.java @@ -0,0 +1,64 @@ +package com.dpgrandslam.stockdataservice.adapter.api; + +import com.dpgrandslam.stockdataservice.domain.model.JobRunRequest; +import com.dpgrandslam.stockdataservice.domain.model.JobRunResponse; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.*; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; +import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; +import org.springframework.batch.core.repository.JobRestartException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +@Slf4j +@Controller +@RequestMapping("/job") +public class JobController { + + @Autowired + private JobLauncher jobLauncher; + + @Autowired + private List batchJobs; + + @Autowired + private JobExplorer jobExplorer; + + @PostMapping("/run") + public ResponseEntity runOptionCSVLoadJob(@RequestBody JobRunRequest runRequest) throws JobInstanceAlreadyCompleteException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException { + Map jobParameterMap = runRequest.getJobParams().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, x -> new JobParameter(x.getValue()))); + Optional jobToRun = batchJobs.stream().filter(job -> job.getName().equals(runRequest.getJobName())).findFirst(); + if (jobToRun.isEmpty()) { + return ResponseEntity.badRequest().build(); + } + log.info("Batch job {} started through API call.", runRequest.getJobName()); + JobExecution jobExecution = jobLauncher.run(jobToRun.get(), new JobParameters(jobParameterMap)); + JobRunResponse jobRunResponse = new JobRunResponse(); + jobRunResponse.setJobId(jobExecution.getJobId()); + jobRunResponse.setJobExecutionId(jobExecution.getId()); + jobRunResponse.setJobStatus(jobExecution.getStatus().name()); + return ResponseEntity.ok(jobRunResponse); + } + + @GetMapping("/status") + public ResponseEntity getJobStatus(@RequestParam Long executionId) { + JobRunResponse jobRunResponse = new JobRunResponse(); + JobExecution jobExecution = jobExplorer.getJobExecution(executionId); + jobRunResponse.setJobStatus(jobExecution.getStatus().name()); + String message = jobExecution.getAllFailureExceptions().stream().findFirst().map(Throwable::getMessage).orElse(null); + jobRunResponse.setMessage(message); + jobRunResponse.setJobId(jobExecution.getJobId()); + jobRunResponse.setJobExecutionId(jobExecution.getId()); + return ResponseEntity.ok(jobRunResponse); + } + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/StockDataServiceController.java b/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/StockDataServiceController.java index b1905db..96b7962 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/StockDataServiceController.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/adapter/api/StockDataServiceController.java @@ -1,15 +1,14 @@ package com.dpgrandslam.stockdataservice.adapter.api; import com.dpgrandslam.stockdataservice.domain.error.OptionsChainLoadException; +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.model.options.Option; import com.dpgrandslam.stockdataservice.domain.model.options.OptionsChain; import com.dpgrandslam.stockdataservice.domain.model.stock.*; -import com.dpgrandslam.stockdataservice.domain.service.OptionsChainLoadService; -import com.dpgrandslam.stockdataservice.domain.service.StockDataLoadService; -import com.dpgrandslam.stockdataservice.domain.service.TenYearTreasuryYieldService; -import com.dpgrandslam.stockdataservice.domain.service.TrackedStockService; +import com.dpgrandslam.stockdataservice.domain.service.*; import lombok.extern.slf4j.Slf4j; -import org.apache.tomcat.jni.Local; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -19,7 +18,9 @@ import java.time.LocalDate; import java.util.*; +import java.util.stream.Collectors; +@CrossOrigin(origins = "http://localhost:3000") @Controller @RequestMapping("/data") @Slf4j @@ -39,6 +40,13 @@ public class StockDataServiceController { @Autowired private TenYearTreasuryYieldService treasuryYieldService; + @Autowired + @Qualifier("CNNFearGreedDataLoadAPIService") + private FearGreedDataLoadService fearGreedDataLoadService; + + @Autowired + private VIXLoadService vixLoadService; + @GetMapping("/option/{ticker}") public ResponseEntity> getOptionsChain(@PathVariable(name = "ticker") String ticker, @RequestParam(name = "expirationDate") Optional expirationDate, @@ -125,8 +133,30 @@ public ResponseEntity addTrackedStocks(@RequestBody List tickers) { } @GetMapping("/treasury-yield") - public ResponseEntity getTreasuryYield(@RequestParam Optional date) { - return ResponseEntity.ok(treasuryYieldService.getTreasuryYieldForDate(date.map(LocalDate::parse) - .orElse(LocalDate.now()))); + public ResponseEntity> getTreasuryYield(@RequestParam String startDate, @RequestParam Optional endDate) { + return ResponseEntity.ok(treasuryYieldService.getTreasuryYieldForDate(LocalDate.parse(startDate), + endDate.map(LocalDate::parse).orElse(LocalDate.now()))); + } + + @GetMapping("/fear-greed") + public ResponseEntity> getFearGreedIndexBetweenDates(@RequestParam Optional startDate, + @RequestParam Optional endDate) { + LocalDate sd = startDate.map(LocalDate::parse).orElse(LocalDate.now()); + LocalDate ed = endDate.map(LocalDate::parse).orElse(LocalDate.now()); + + if (sd.equals(LocalDate.now()) && ed.equals(LocalDate.now())) { + return ResponseEntity.ok(fearGreedDataLoadService.loadCurrentFearGreedIndex().stream() + .sorted(Comparator.comparing(FearGreedIndex::getTradeDate)) + .collect(Collectors.toList())); + } else if (sd.equals(ed)) { + Optional fgIndex = fearGreedDataLoadService.getFearGreedIndexOfDay(sd); + return fgIndex.map(fearGreedIndex -> ResponseEntity.ok(Collections.singletonList(fearGreedIndex))).orElseGet(() -> ResponseEntity.notFound().build()); + } + return ResponseEntity.ok(fearGreedDataLoadService.loadFearGreedDataBetweenDates(sd, ed)); + } + + @GetMapping("/vix") + public ResponseEntity> getVixForDates(@RequestParam String startDate, @RequestParam Optional endDate) { + return ResponseEntity.ok(vixLoadService.loadVIXBetweenDates(LocalDate.parse(startDate), endDate.map(LocalDate::parse).orElse(LocalDate.parse(startDate)))); } } diff --git a/src/main/java/com/dpgrandslam/stockdataservice/adapter/apiclient/CNNFearGreedClient.java b/src/main/java/com/dpgrandslam/stockdataservice/adapter/apiclient/CNNFearGreedClient.java new file mode 100644 index 0000000..b400c23 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/adapter/apiclient/CNNFearGreedClient.java @@ -0,0 +1,16 @@ +package com.dpgrandslam.stockdataservice.adapter.apiclient; + +import com.dpgrandslam.stockdataservice.domain.model.CNNFearGreedResponse; +import feign.Headers; +import feign.Param; +import feign.RequestLine; + +import java.time.LocalDate; + +@Headers({"Accept: application/json", "if-none-match: W/2005698896447632232"}) +public interface CNNFearGreedClient { + + @RequestLine("GET /index/fearandgreed/graphdata/{date}") + CNNFearGreedResponse getFearGreedData(@Param("date") String date); + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/FearGreedIndexRepository.java b/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/FearGreedIndexRepository.java new file mode 100644 index 0000000..fb7325c --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/FearGreedIndexRepository.java @@ -0,0 +1,18 @@ +package com.dpgrandslam.stockdataservice.adapter.repository; + +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; + +public interface FearGreedIndexRepository extends JpaRepository { + + Optional findFearGreedIndexByTradeDate(LocalDate tradeDate); + + List findFearGreedIndexByTradeDateBetween(LocalDate startDate, LocalDate endDate); + + List findFearGreedIndicesByTradeDateGreaterThanEqual(LocalDate tradeDate); + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/TrackedStocksRepository.java b/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/TrackedStocksRepository.java index de7e58d..8360509 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/TrackedStocksRepository.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/adapter/repository/TrackedStocksRepository.java @@ -2,9 +2,11 @@ import com.dpgrandslam.stockdataservice.domain.model.stock.TrackedStock; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; import java.util.List; +@Repository public interface TrackedStocksRepository extends JpaRepository { List findAllByActiveIsTrue(); diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/APISecurityConfig.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/APISecurityConfig.java new file mode 100644 index 0000000..5731b01 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/APISecurityConfig.java @@ -0,0 +1,44 @@ +package com.dpgrandslam.stockdataservice.domain.config; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.HandlerInterceptor; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * A simple security configuration for the api without having to use spring security + */ +@Configuration +public class APISecurityConfig implements WebMvcConfigurer { + + @Bean + @ConfigurationProperties(prefix = "api.security") + public APISecurityConfigurationProperties apiSecurityConfigurationProperties() { + return new APISecurityConfigurationProperties(); + } + + @Override + public void addInterceptors(InterceptorRegistry registry) { + WebMvcConfigurer.super.addInterceptors(registry); + registry.addInterceptor(new HandlerInterceptor() { + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + APISecurityConfigurationProperties securityConfigurationProperties = apiSecurityConfigurationProperties(); + if (securityConfigurationProperties.getEnabled() + && (StringUtils.isBlank(securityConfigurationProperties.getPassword()) + || !securityConfigurationProperties.getPassword().equals(request.getHeader("stock-data-password")))) { + response.setStatus(401); + return false; + } + return HandlerInterceptor.super.preHandle(request, response, handler); + } + }); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/APISecurityConfigurationProperties.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/APISecurityConfigurationProperties.java new file mode 100644 index 0000000..7f26fac --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/APISecurityConfigurationProperties.java @@ -0,0 +1,11 @@ +package com.dpgrandslam.stockdataservice.domain.config; + +import lombok.Data; + +@Data +public class APISecurityConfigurationProperties { + + private Boolean enabled = false; + private String password = ""; + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/ApiClientsConfiguration.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/ApiClientsConfiguration.java index 3236e35..03cb1b9 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/ApiClientsConfiguration.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/ApiClientsConfiguration.java @@ -1,7 +1,10 @@ package com.dpgrandslam.stockdataservice.domain.config; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.dpgrandslam.stockdataservice.adapter.apiclient.CNNFearGreedClient; import com.dpgrandslam.stockdataservice.adapter.apiclient.tiingo.TiingoApiClient; -import com.dpgrandslam.stockdataservice.domain.config.ApiClientConfigurationProperties; import com.dpgrandslam.stockdataservice.domain.util.BasicAuthorizationTarget; import feign.Feign; import feign.gson.GsonDecoder; @@ -38,4 +41,31 @@ public TiingoApiClient tiingoApiClient( .client(new OkHttpClient()) .target(new BasicAuthorizationTarget<>(TiingoApiClient.class, configurationProperties)); } + + @Bean("CNNClientConfigurationProperties") + @ConfigurationProperties(prefix = "api.client.cnn") + public ApiClientConfigurationProperties cnnClientConfigurationProperties() { + return new ApiClientConfigurationProperties(); + } + + @Bean("FearGreedClientConfigurationProperties") + @ConfigurationProperties(prefix = "api.client.fear-greed") + public ApiClientConfigurationProperties fearGreedClientConfigurationProperties() { + return new ApiClientConfigurationProperties(); + } + + @Bean + public CNNFearGreedClient cnnFearGreedClient(@Qualifier("FearGreedClientConfigurationProperties") ApiClientConfigurationProperties configurationProperties) { + return Feign.builder() + .decoder(new GsonDecoder()) + .encoder(new GsonEncoder()) + .logger(new Slf4jLogger(CNNFearGreedClient.class)) + .client(new OkHttpClient()) + .target(new BasicAuthorizationTarget<>(CNNFearGreedClient.class, configurationProperties)); + } + + @Bean + public AmazonS3 amazonS3() { + return AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build(); + } } diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/CacheConfiguration.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/CacheConfiguration.java index bf6b166..95ffcd9 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/CacheConfiguration.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/CacheConfiguration.java @@ -1,13 +1,15 @@ package com.dpgrandslam.stockdataservice.domain.config; +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; import com.dpgrandslam.stockdataservice.domain.model.options.HistoricalOption; import com.dpgrandslam.stockdataservice.domain.model.stock.EndOfDayStockData; -import com.dpgrandslam.stockdataservice.domain.model.stock.YahooFinanceTenYearTreasuryYield; +import com.dpgrandslam.stockdataservice.domain.model.stock.YahooFinanceQuote; import com.dpgrandslam.stockdataservice.domain.model.tiingo.TiingoStockSearchResponse; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import lombok.Data; import lombok.NonNull; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -38,8 +40,8 @@ public Cache> historical .build(); } - @Bean - public Cache treasuryYieldCache() { + @Bean("TreasuryYieldCache") + public Cache, List> treasuryYieldCache() { return Caffeine.newBuilder() .expireAfterWrite(10, TimeUnit.DAYS) .maximumSize(1000) @@ -54,6 +56,22 @@ public Cache> endOfDayStock .build(); } + @Bean + public Cache, List> fearGreedBetweenDatesCache() { + return Caffeine.newBuilder() + .expireAfterWrite(2, TimeUnit.HOURS) + .maximumSize(2000) + .build(); + } + + @Bean("VIXCache") + public Cache, List> vixCache() { + return Caffeine.newBuilder() + .expireAfterWrite(2, TimeUnit.DAYS) + .maximumSize(1000) + .build(); + } + @Data public static class HistoricOptionsDataCacheKey { diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/FearGreedJSONLoadJobConfig.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/FearGreedJSONLoadJobConfig.java new file mode 100644 index 0000000..b55a896 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/FearGreedJSONLoadJobConfig.java @@ -0,0 +1,104 @@ +package com.dpgrandslam.stockdataservice.domain.config; + +import com.amazonaws.services.s3.AmazonS3; +import com.dpgrandslam.stockdataservice.domain.jobs.feargreedbatch.FearGreedJSONFile; +import com.dpgrandslam.stockdataservice.domain.jobs.feargreedbatch.FearGreedJSONItemProcessor; +import com.dpgrandslam.stockdataservice.domain.jobs.feargreedbatch.FearGreedJSONItemWriter; +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.util.AWSS3ItemReader; +import com.dpgrandslam.stockdataservice.domain.util.SingleJacksonJsonObjectReader; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParametersValidator; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.DefaultJobParametersValidator; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.json.JacksonJsonObjectReader; +import org.springframework.batch.item.json.JsonItemReader; +import org.springframework.batch.item.json.JsonObjectReader; +import org.springframework.batch.item.support.SynchronizedItemStreamReader; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.TaskExecutor; + +import java.util.Set; + +@Configuration +@EnableBatchProcessing +@Slf4j +@RequiredArgsConstructor +public class FearGreedJSONLoadJobConfig { + + public static final String JOB_NAME = "fear-greed-json-load-job"; + public static final String STEP_NAME = "fear-greed-json-load-step"; + + + @Bean("fearGreedJSONLoadJobStep") + public Step fearGreedJSONLoadJobStep(StepBuilderFactory stepBuilderFactory, + ItemReader itemReader, + FearGreedJSONItemProcessor itemProcessor, + FearGreedJSONItemWriter itemWriter) { + return stepBuilderFactory.get(STEP_NAME) + .>chunk(5) + .reader(itemReader) + .processor(itemProcessor) + .writer(itemWriter) + .faultTolerant() + .build(); + } + + @Bean("fearGreedJSONLoadJob") + public Job fearGreedJSONLoadJob(JobBuilderFactory jobBuilderFactory, @Qualifier("fearGreedJSONLoadJobStep") Step fearGreedJSONLoadJobStep) { + return jobBuilderFactory.get(JOB_NAME) + .incrementer(new RunIdIncrementer()) + .validator(jobParametersValidator()) + .start(fearGreedJSONLoadJobStep) + .build(); + } + + private JobParametersValidator jobParametersValidator() { + return new DefaultJobParametersValidator( + new String[] {"bucket", "keyPrefix"}, + new String[0] + ); + } + + @Bean("fearGreedJSONFileItemStreamReader") + @StepScope + public ItemStreamReader fearGreedJSONFileItemStreamReader(@Value("#{jobParameters['bucket']}") String bucket, + @Value("#{jobParameters['keyPrefix']}") String keyPrefix, + AmazonS3 amazonS3) { + JsonItemReader jsonFileReader = new JsonItemReader<>(); + jsonFileReader.setName("fearGreedJSONReader"); + jsonFileReader.setStrict(false); + jsonFileReader.setJsonObjectReader(jsonObjectReader()); + + AWSS3ItemReader awss3ItemReader = new AWSS3ItemReader<>(amazonS3); + awss3ItemReader.setBucket(bucket); + awss3ItemReader.setKeyPrefix(keyPrefix); + awss3ItemReader.setDelegate(jsonFileReader); + + SynchronizedItemStreamReader synchronizedItemStreamReader = new SynchronizedItemStreamReader<>(); + synchronizedItemStreamReader.setDelegate(awss3ItemReader); + + return synchronizedItemStreamReader; + } + + private JsonObjectReader jsonObjectReader() { + return new SingleJacksonJsonObjectReader<>(FearGreedJSONFile.class); + } + + + + + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/config/OptionCSVLoadJobConfig.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/OptionCSVLoadJobConfig.java new file mode 100644 index 0000000..3ace410 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/config/OptionCSVLoadJobConfig.java @@ -0,0 +1,168 @@ +package com.dpgrandslam.stockdataservice.domain.config; + +import com.amazonaws.services.s3.AmazonS3; +import com.dpgrandslam.stockdataservice.domain.jobs.optioncsv.OptionCSVItemProcessor; +import com.dpgrandslam.stockdataservice.domain.jobs.optioncsv.OptionCSVFile; +import com.dpgrandslam.stockdataservice.domain.model.options.HistoricalOption; +import com.dpgrandslam.stockdataservice.domain.util.AWSS3ItemReader; +import lombok.AllArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParametersValidator; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.*; +import org.springframework.batch.core.job.DefaultJobParametersValidator; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.batch.core.launch.support.SimpleJobLauncher; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.file.FlatFileItemReader; +import org.springframework.batch.item.file.LineMapper; +import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; +import org.springframework.batch.item.file.mapping.DefaultLineMapper; +import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; +import org.springframework.batch.item.support.SynchronizedItemStreamReader; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; +import java.io.IOException; +import java.time.format.DateTimeParseException; +import java.util.concurrent.ThreadPoolExecutor; + +@Configuration +@EnableBatchProcessing +@Slf4j +@AllArgsConstructor +public class OptionCSVLoadJobConfig { + + public static final String JOB_NAME = "option-csv-load-job"; + public static final String STEP_NAME = "option-csv-load-step"; + + @Bean("optionCsvLoadJobStep") + public Step optionCsvLoadJobStep(StepBuilderFactory stepBuilderFactory, + ItemReader itemReader, + OptionCSVItemProcessor itemProcessor, + ItemWriter itemWriter) { + return stepBuilderFactory.get(STEP_NAME) + .chunk(50) + .reader(itemReader) + .processor(itemProcessor) + .writer(itemWriter) + .faultTolerant() + .skipLimit(1000000) + .skip(DateTimeParseException.class) + .taskExecutor(taskExecutor()) + .build(); + } + + @Bean + public TaskExecutor taskExecutor() { + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + taskExecutor.setCorePoolSize(10); + taskExecutor.setMaxPoolSize(15); + taskExecutor.setQueueCapacity(10); + taskExecutor.setThreadNamePrefix("MultiThreaded-"); + taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + return taskExecutor; + } + + @Bean("optionCSVLoadJob") + public Job optionCSVLoadJob(JobBuilderFactory jobBuilderFactory, @Qualifier("optionCsvLoadJobStep") Step optionCsvLoadJobStep) { + return jobBuilderFactory.get(JOB_NAME) + .incrementer(new RunIdIncrementer()) + .validator(jobParametersValidator()) + .start(optionCsvLoadJobStep) + .build(); + } + + private JobParametersValidator jobParametersValidator() { + return new DefaultJobParametersValidator( + new String[] {"bucket", "keyPrefix"}, + new String[0] + ); + } + + + @Bean("optionCSVItemReader") + @StepScope + public ItemStreamReader optionCSVItemReader(@Value("#{jobParameters['bucket']}") String bucket, + @Value("#{jobParameters['keyPrefix']}") String keyPrefix, + AmazonS3 amazonS3) throws IOException { + FlatFileItemReader csvReader = new FlatFileItemReader<>(); + csvReader.setLineMapper(lineMapper()); + csvReader.setName("optionCSVReader"); + csvReader.setLinesToSkip(1); + + AWSS3ItemReader awss3ItemReader = new AWSS3ItemReader<>(amazonS3); + awss3ItemReader.setBucket(bucket); + awss3ItemReader.setKeyPrefix(keyPrefix); + awss3ItemReader.setDelegate(csvReader); + + SynchronizedItemStreamReader synchronizedItemStreamReader = new SynchronizedItemStreamReader<>(); + synchronizedItemStreamReader.setDelegate(awss3ItemReader); + + return synchronizedItemStreamReader; + } + + @Bean + public BatchConfigurer batchConfigurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) { + return new DefaultBatchConfigurer(dataSource) { + + @SneakyThrows + @Override + public JobLauncher getJobLauncher() { + SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); + jobLauncher.setJobRepository(getJobRepository()); + jobLauncher.setTaskExecutor(new SimpleAsyncTaskExecutor()); + jobLauncher.afterPropertiesSet(); + return jobLauncher; + } + + @SneakyThrows + @Override + public JobRepository getJobRepository() { + JobRepositoryFactoryBean jobRepositoryFactoryBean = new JobRepositoryFactoryBean(); + jobRepositoryFactoryBean.setDataSource(dataSource); + jobRepositoryFactoryBean.setTransactionManager(getTransactionManager()); + // set other properties + return jobRepositoryFactoryBean.getObject(); + } + + @Override + public PlatformTransactionManager getTransactionManager() { + return new JpaTransactionManager(entityManagerFactory); + } + }; + } + + private LineMapper lineMapper() { + DefaultLineMapper defaultLineMapper = new DefaultLineMapper<>(); + DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer(); + + lineTokenizer.setDelimiter(","); + lineTokenizer.setStrict(false); + lineTokenizer.setNames("optionKey", "symbol", "expirationDate", "askPrice", "askSize", "bidPrice", "bidSize", + "lastPrice", "putCall", "strikePrice", "volume", "openInterest", "underlyingPrice", "dataDate"); + BeanWrapperFieldSetMapper fieldSetMapper = new BeanWrapperFieldSetMapper<>(); + fieldSetMapper.setTargetType(OptionCSVFile.class); + + defaultLineMapper.setLineTokenizer(lineTokenizer); + defaultLineMapper.setFieldSetMapper(fieldSetMapper); + + return defaultLineMapper; + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/error/TreasuryYieldLoadException.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/error/TreasuryYieldLoadException.java deleted file mode 100644 index cf889e7..0000000 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/error/TreasuryYieldLoadException.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.dpgrandslam.stockdataservice.domain.error; - -import java.time.DayOfWeek; -import java.time.LocalDate; - -public class TreasuryYieldLoadException extends RuntimeException { - - private final LocalDate date; - - public TreasuryYieldLoadException(LocalDate date, String message) { - super(message); - this.date = date; - } - - public TreasuryYieldLoadException(LocalDate date) { - super(); - this.date = date; - } - - public TreasuryYieldLoadException(LocalDate date, String message, Throwable e) { - super(message, e); - this.date = date; - } - - @Override - public String getMessage() { - String defaultMessage = "Could not load 10 yr treasury yield for date " + date.toString() + "."; - if (date.getDayOfWeek() == DayOfWeek.SATURDAY || date.getDayOfWeek() == DayOfWeek.SUNDAY) { - defaultMessage += " Failure most likely due to the date being a weekend."; - } - return defaultMessage + (super.getMessage() != null ? " " + super.getMessage() : ""); - } -} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/error/YahooFinanceQuoteLoadException.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/error/YahooFinanceQuoteLoadException.java new file mode 100644 index 0000000..267d22e --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/error/YahooFinanceQuoteLoadException.java @@ -0,0 +1,49 @@ +package com.dpgrandslam.stockdataservice.domain.error; + +import java.time.DayOfWeek; +import java.time.LocalDate; + +public class YahooFinanceQuoteLoadException extends RuntimeException { + + private final LocalDate startDate; + private final LocalDate endDate; + private final String ticker; + + public YahooFinanceQuoteLoadException(String ticker, LocalDate startDate, LocalDate endDate, String message) { + super(message); + this.startDate = startDate; + this.endDate = endDate; + this.ticker = ticker; + } + + public YahooFinanceQuoteLoadException(String ticker, LocalDate startDate, LocalDate endDate) { + super(); + this.startDate = startDate; + this.endDate = endDate; + this.ticker = ticker; + + } + + public YahooFinanceQuoteLoadException(String ticker, LocalDate startDate, LocalDate endDate, Throwable e) { + super(e); + this.startDate = startDate; + this.endDate = endDate; + this.ticker = ticker; + } + + public YahooFinanceQuoteLoadException(String ticker, LocalDate startDate, LocalDate endDate, String message, Throwable e) { + super(message, e); + this.startDate = startDate; + this.endDate = endDate; + this.ticker = ticker; + } + + @Override + public String getMessage() { + String defaultMessage = "Could not load yahoo finance quote for ticker " + ticker +" for date " + startDate.toString() + "."; + if (startDate.getDayOfWeek() == DayOfWeek.SATURDAY || startDate.getDayOfWeek() == DayOfWeek.SUNDAY) { + defaultMessage += " Failure most likely due to the date being a weekend."; + } + return defaultMessage + (super.getMessage() != null ? " " + super.getMessage() : ""); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayFearGreedLoaderJob.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayFearGreedLoaderJob.java new file mode 100644 index 0000000..5578601 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayFearGreedLoaderJob.java @@ -0,0 +1,51 @@ +package com.dpgrandslam.stockdataservice.domain.jobs; + +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.service.FearGreedDataLoadService; +import com.dpgrandslam.stockdataservice.domain.util.TimeUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.LocalDate; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class EndOfDayFearGreedLoaderJob { + + @Autowired + @Qualifier("CNNFearGreedDataLoadAPIService") + private FearGreedDataLoadService fearGreedDataLoadService; + + @Autowired + private TimeUtils timeUtils; + + @Scheduled(cron = "50 59 15-23 * * MON-FRI") + public void runJob() { + LocalDate tradeDate = timeUtils.getCurrentOrLastTradeDate(); + Optional existing = fearGreedDataLoadService.getFearGreedIndexOfDay(tradeDate); + if (existing.isEmpty() && !timeUtils.isStockMarketHoliday(tradeDate)) { + try { + log.info("Loading fear greed data for day {}...", tradeDate); + Set fearGreedIndices = fearGreedDataLoadService.loadCurrentFearGreedIndex().stream() + .filter(x -> !timeUtils.isStockMarketHoliday(x.getTradeDate()) && fearGreedDataLoadService.getFearGreedIndexOfDay(x.getTradeDate()).isEmpty()) + .collect(Collectors.toSet()); + log.info("Found fear greed data for day {}: {}", tradeDate, fearGreedIndices); + fearGreedDataLoadService.saveFearGreedData(fearGreedIndices); + log.info("Saved fear greed data to the database. Job Complete!"); + } catch (Exception e) { + log.error("Error loading fear greed data for day {}.", tradeDate, e); + } + } else if (existing.isEmpty()) { + log.info("Existing fear greed data for day {} already exists... skipping", tradeDate); + } else if (!timeUtils.isStockMarketHoliday(tradeDate)) { + log.info("Today is stock market holiday, no fear greed data will be loaded."); + } + + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayOptionsLoaderJob.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayOptionsLoaderJob.java index 47a8b29..5b5be92 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayOptionsLoaderJob.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/EndOfDayOptionsLoaderJob.java @@ -86,7 +86,7 @@ private void reset() { trackedStocks = new ConcurrentLinkedQueue<>(); // Put all tracked stock tickers into queue for processing by batch trackedStocks.addAll(trackedStockService.getAllTrackedStocks(true).stream() - .filter(trackedStock -> trackedStock.getLastOptionsHistoricDataUpdate() == null || trackedStock.getLastOptionsHistoricDataUpdate().isBefore(timeUtils.getLastTradeDate())) + .filter(trackedStock -> trackedStock.getLastOptionsHistoricDataUpdate() == null || trackedStock.getLastOptionsHistoricDataUpdate().isBefore(timeUtils.getCurrentOrLastTradeDate())) .map(TrackedStock::getTicker) .collect(Collectors.toList())); failCountMap = new HashMap<>(); @@ -144,7 +144,7 @@ public void runRetryAfterMidnight() { * A retry job that trys again for any failed reads. */ private void retryQueueJob() { - LocalDate tradeDate = timeUtils.getLastTradeDate(); + LocalDate tradeDate = timeUtils.getCurrentOrLastTradeDate(); log.info("Getting options in retry table for trade date {}.", tradeDate); Set retrySet = optionRetryService.getAllWithTradeDate(tradeDate); log.info("Found {} options in retry table for trade date {}.", retrySet.size(),tradeDate); @@ -192,7 +192,7 @@ private void storeOptionsChainEndOfDayData() { TrackedStock current = trackedStockService.findByTicker(trackedStocks.poll()); // If the current is null or not active or already updated today, do nothing. if (current != null && mainJobStatus.isRunning() && current.isActive() - && (current.getLastOptionsHistoricDataUpdate() == null || !current.getLastOptionsHistoricDataUpdate().equals(timeUtils.getLastTradeDate()))) { + && (current.getLastOptionsHistoricDataUpdate() == null || !current.getLastOptionsHistoricDataUpdate().equals(timeUtils.getCurrentOrLastTradeDate()))) { // Do options data load and put into database log.info("Executing update for {}", current); TimerUtil timerUtil = TimerUtil.startTimer(); @@ -239,7 +239,7 @@ private void startJob() { private void completeJob(int job) { if (job == RETRY_JOB && retryJobStatus.isRunning()) { - if (optionRetryService.getAllWithTradeDate(timeUtils.getLastTradeDate()).isEmpty()) { + if (optionRetryService.getAllWithTradeDate(timeUtils.getCurrentOrLastTradeDate()).isEmpty()) { retryJobStatus = JobStatus.COMPLETE; } else { retryJobStatus = JobStatus.COMPLETE_WITH_FAILURES; @@ -247,15 +247,15 @@ private void completeJob(int job) { log.info("Retry job finished with status: {}", retryJobStatus.name()); } else if (job == MAIN_JOB && mainJobStatus.isRunning()) { log.info("Main job finished."); - if (optionRetryService.getAllWithTradeDate(timeUtils.getLastTradeDate()).isEmpty()) { + if (optionRetryService.getAllWithTradeDate(timeUtils.getCurrentOrLastTradeDate()).isEmpty()) { mainJobStatus = JobStatus.COMPLETE; } else { mainJobStatus = JobStatus.COMPLETE_WITH_FAILURES; } log.info("Main job finished with status: {}", mainJobStatus.name()); } - long optionsLoadedCount = historicOptionsDataService.countOptionsLoadedOnTradeDate(timeUtils.getLastTradeDate()); - log.info("Loaded {} options for date {}", optionsLoadedCount, timeUtils.getLastTradeDate()); + long optionsLoadedCount = historicOptionsDataService.countOptionsLoadedOnTradeDate(timeUtils.getCurrentOrLastTradeDate()); + log.info("Loaded {} options for date {}", optionsLoadedCount, timeUtils.getCurrentOrLastTradeDate()); } @EventListener(TrackedStockAddedEvent.class) diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONFile.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONFile.java new file mode 100644 index 0000000..6c66e2b --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONFile.java @@ -0,0 +1,24 @@ +package com.dpgrandslam.stockdataservice.domain.jobs.feargreedbatch; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.Set; + +@Data +public class FearGreedJSONFile { + + private Set data; + + @Data + public static class FearGreedJSONData { + + @JsonProperty("x") + private Double timestamp; + + @JsonProperty("y") + private Double value; + + private String rating; + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONItemProcessor.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONItemProcessor.java new file mode 100644 index 0000000..7bf8b41 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONItemProcessor.java @@ -0,0 +1,49 @@ +package com.dpgrandslam.stockdataservice.domain.jobs.feargreedbatch; + +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.service.FearGreedDataLoadService; +import com.dpgrandslam.stockdataservice.domain.util.TimeUtils; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +@Component +@Slf4j +public class FearGreedJSONItemProcessor implements ItemProcessor> { + + @Autowired + @Qualifier("CNNFearGreedDataLoadAPIService") + private FearGreedDataLoadService fearGreedDataLoadService; + + @Autowired + private TimeUtils timeUtils; + + @Override + public Set process(FearGreedJSONFile fearGreedJSONFile) throws Exception { + Set fearGreedIndices = new HashSet<>(); + fearGreedJSONFile.getData().forEach(data -> { + Instant instant = Instant.ofEpochMilli(data.getTimestamp().longValue()); + LocalDate tradeDate = LocalDate.ofInstant(instant, ZoneOffset.UTC); + if (fearGreedDataLoadService.getFearGreedIndexOfDay(tradeDate).isEmpty() && timeUtils.isTradingOpenOnDay(tradeDate)) { + FearGreedIndex fearGreedIndex = new FearGreedIndex(); + fearGreedIndex.setTradeDate(tradeDate); + fearGreedIndex.setValue(data.getValue().intValue()); + fearGreedIndices.add(fearGreedIndex); + } + }); + Set filteredFearGreedIndices = fearGreedIndices.stream().filter(Objects::nonNull).collect(Collectors.toSet()); + log.info("Processed {} fear-greed record successfully.", filteredFearGreedIndices.size()); + return filteredFearGreedIndices; + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONItemWriter.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONItemWriter.java new file mode 100644 index 0000000..6fcafe0 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/feargreedbatch/FearGreedJSONItemWriter.java @@ -0,0 +1,25 @@ +package com.dpgrandslam.stockdataservice.domain.jobs.feargreedbatch; + +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.service.FearGreedDataLoadService; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +@Component +public class FearGreedJSONItemWriter implements ItemWriter> { + + @Autowired + @Qualifier("CNNFearGreedDataLoadAPIService") + private FearGreedDataLoadService fearGreedDataLoadService; + + @Override + public void write(List> list) throws Exception { + list.forEach(fearGreedDataLoadService::saveFearGreedData); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVFile.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVFile.java new file mode 100644 index 0000000..5cefdc8 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVFile.java @@ -0,0 +1,23 @@ +package com.dpgrandslam.stockdataservice.domain.jobs.optioncsv; + +import lombok.Data; + +@Data +public class OptionCSVFile { + + private String optionKey; + private String symbol; + private String expirationDate; + private String askPrice; + private String askSize; + private String bidPrice; + private String bidSize; + private String lastPrice; + private String putCall; + private String strikePrice; + private String volume; + private String openInterest; + private String underlyingPrice; + private String dataDate; + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVItemProcessor.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVItemProcessor.java new file mode 100644 index 0000000..646e9ab --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVItemProcessor.java @@ -0,0 +1,111 @@ +package com.dpgrandslam.stockdataservice.domain.jobs.optioncsv; + +import com.dpgrandslam.stockdataservice.domain.model.options.HistoricalOption; +import com.dpgrandslam.stockdataservice.domain.model.options.Option; +import com.dpgrandslam.stockdataservice.domain.model.options.OptionPriceData; +import com.dpgrandslam.stockdataservice.domain.model.stock.TrackedStock; +import com.dpgrandslam.stockdataservice.domain.service.HistoricOptionsDataService; +import com.dpgrandslam.stockdataservice.domain.service.TrackedStockService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.stereotype.Component; + +import javax.persistence.EntityNotFoundException; +import javax.transaction.Transactional; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.stream.Collectors; + +@Component +@RequiredArgsConstructor +@Slf4j +public class OptionCSVItemProcessor implements ItemProcessor { + + private final HistoricOptionsDataService historicOptionsDataService; + private final TrackedStockService trackedStockService; + + private Map tracked = new HashMap<>(); + private boolean trackedStocksLoaded = false; + + @Override + @Transactional + public HistoricalOption process(OptionCSVFile optionCSVFile) throws Exception { + if (!trackedStocksLoaded && tracked.isEmpty()) { + tracked = trackedStockService.getAllTrackedStocks(true).stream() + .collect(Collectors.toMap(TrackedStock::getTicker, TrackedStock::getOptionsHistoricDataStartDate)); + trackedStocksLoaded = true; + } + if (!tracked.containsKey(optionCSVFile.getSymbol())) { + return null; + } + + HistoricalOption historicalOption = HistoricalOption.builder() + .optionType(optionCSVFile.getPutCall().equalsIgnoreCase("call") ? Option.OptionType.CALL : Option.OptionType.PUT) + .strike(Double.parseDouble(optionCSVFile.getStrikePrice())) + .expiration(parseDate(optionCSVFile.getExpirationDate())) + .ticker(optionCSVFile.getSymbol().toUpperCase()) + .build(); + + OptionPriceData optionPriceData = OptionPriceData.builder() + .tradeDate(parseDate(optionCSVFile.getDataDate())) + .openInterest(Integer.parseInt(optionCSVFile.getOpenInterest())) + .bid(Double.parseDouble(optionCSVFile.getBidPrice())) + .ask(Double.parseDouble(optionCSVFile.getAskPrice())) + .volume(Integer.parseInt(optionCSVFile.getVolume())) + .lastTradePrice(Double.parseDouble(optionCSVFile.getLastPrice())) + .dataObtainedDate(Timestamp.from(Instant.now())) + .build(); + + if (optionPriceData.getTradeDate().isAfter(historicalOption.getExpiration())) { + log.warn("Trade date for {} is after expiration: {}. Skipping...", optionPriceData, historicalOption.getExpiration()); + return null; + } + HistoricalOption existing; + try { + existing = historicOptionsDataService.findOption(historicalOption.getTicker(), historicalOption.getExpiration(), + historicalOption.getStrike(), historicalOption.getOptionType()); + if (existing.getOptionPriceData().stream().map(OptionPriceData::getTradeDate) + .collect(Collectors.toSet()).contains(optionPriceData.getTradeDate())) { + log.warn("Option Price data for {} already exists. Skipping...", optionPriceData); + return null; + } + } catch (EntityNotFoundException e) { + existing = null; + log.debug("Option does not exist, creating new one"); + } + if (existing != null) { + existing.getOptionPriceData().add(optionPriceData); + optionPriceData.setOption(existing); + } else { + historicalOption.setOptionPriceData(Collections.singleton(optionPriceData)); + optionPriceData.setOption(historicalOption); + } + if (optionPriceData.getTradeDate().isBefore(tracked.get(historicalOption.getTicker()))) { + try { + TrackedStock trackedStock = trackedStockService.findByTicker(optionCSVFile.getSymbol()); + trackedStock.setOptionsHistoricDataStartDate(optionPriceData.getTradeDate()); + trackedStockService.saveTrackedStock(trackedStock); + tracked.put(historicalOption.getTicker(), optionPriceData.getTradeDate()); + } catch (EntityNotFoundException e) { + log.warn("{}. Skipping update...", e.getMessage()); + return null; + } + } + return existing != null ? existing : historicalOption; + } + + private LocalDate parseDate(String dateString) { + LocalDate date; + try { + date = LocalDate.parse(dateString); + } catch (DateTimeParseException e) { + date = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("M/d/yyyy")); + } + return date; + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVItemWriter.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVItemWriter.java new file mode 100644 index 0000000..bdb8869 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/jobs/optioncsv/OptionCSVItemWriter.java @@ -0,0 +1,24 @@ +package com.dpgrandslam.stockdataservice.domain.jobs.optioncsv; + +import com.dpgrandslam.stockdataservice.domain.model.options.HistoricalOption; +import com.dpgrandslam.stockdataservice.domain.service.HistoricOptionsDataService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.stream.Collectors; + +@Component +@Slf4j +@RequiredArgsConstructor +public class OptionCSVItemWriter implements ItemWriter { + + private final HistoricOptionsDataService dataService; + + @Override + public void write(List list) throws Exception { + dataService.saveOptions(list.stream().map(x -> (HistoricalOption) x).collect(Collectors.toList())); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/CNNFearGreedResponse.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/CNNFearGreedResponse.java new file mode 100644 index 0000000..ad06a2b --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/CNNFearGreedResponse.java @@ -0,0 +1,31 @@ +package com.dpgrandslam.stockdataservice.domain.model; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; + +@Data +public class CNNFearGreedResponse { + + @SerializedName("fear_and_greed") + private FearAndGreed fearAndGreed; + + @Data + public static class FearAndGreed { + + @SerializedName("previous_1_month") + private Double previousOneMonth; + + @SerializedName("previous_1_week") + private Double previousOneWeek; + + @SerializedName("previous_1_year") + private Double previousOneYear; + + @SerializedName("previous_close") + private Double previousClose; + + private String rating; + private Double score; + private String timestamp; + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/FearGreedIndex.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/FearGreedIndex.java new file mode 100644 index 0000000..3070a3f --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/FearGreedIndex.java @@ -0,0 +1,57 @@ +package com.dpgrandslam.stockdataservice.domain.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.*; +import org.hibernate.Hibernate; +import org.hibernate.annotations.CreationTimestamp; + +import javax.persistence.*; +import javax.validation.constraints.NotNull; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.Objects; + +@Entity +@Table(indexes = { + @Index(name = "idx_trade_date", columnList = "trade_date", unique = true) +}) +@ToString +@NoArgsConstructor +public class FearGreedIndex { + + @Id + @GeneratedValue + @JsonIgnore + @Getter + @Setter + private Long id; + + @Getter + @Setter + @NotNull + private Integer value; + + @Getter + @Setter + @Column(name = "trade_date") + @NotNull + private LocalDate tradeDate; + + @Getter + @Setter + @CreationTimestamp + private Timestamp createTime; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + FearGreedIndex that = (FearGreedIndex) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(value, tradeDate); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/Holiday.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/Holiday.java index 2628955..e80e240 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/Holiday.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/Holiday.java @@ -2,6 +2,7 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import java.time.LocalDate; @@ -9,6 +10,7 @@ @Data @NoArgsConstructor @AllArgsConstructor +@EqualsAndHashCode public class Holiday { private String name; diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/JobRunRequest.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/JobRunRequest.java new file mode 100644 index 0000000..3262f6c --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/JobRunRequest.java @@ -0,0 +1,13 @@ +package com.dpgrandslam.stockdataservice.domain.model; + +import lombok.Data; + +import java.util.Map; + +@Data +public class JobRunRequest { + + private String jobName; + private Map jobParams; + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/JobRunResponse.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/JobRunResponse.java new file mode 100644 index 0000000..c071151 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/JobRunResponse.java @@ -0,0 +1,12 @@ +package com.dpgrandslam.stockdataservice.domain.model; + +import lombok.Data; + +@Data +public class JobRunResponse { + + private String jobStatus; + private Long jobId; + private Long jobExecutionId; + private String message; +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/options/OptionPriceData.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/options/OptionPriceData.java index 8972870..ee86f9f 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/options/OptionPriceData.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/options/OptionPriceData.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.*; import org.hibernate.annotations.Cascade; @@ -21,7 +23,8 @@ @JsonIgnoreProperties(ignoreUnknown = true) @Table(indexes = { @Index(name = "idx_optionId_tradeDate", columnList = "option_id, trade_date", unique = true), - @Index(name = "idx_optionId", columnList = "option_id") + @Index(name = "idx_optionId", columnList = "option_id"), + @Index(name = "idx_trade_date", columnList = "trade_date") }) public class OptionPriceData { @@ -47,6 +50,7 @@ public class OptionPriceData { private Integer openInterest; @EqualsAndHashCode.Include + @JsonInclude(JsonInclude.Include.NON_NULL) private Double impliedVolatility; @ManyToOne(fetch = FetchType.LAZY) diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/EndOfDayStockData.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/EndOfDayStockData.java index a46bb98..cacdec1 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/EndOfDayStockData.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/EndOfDayStockData.java @@ -12,8 +12,12 @@ public interface EndOfDayStockData { Double getHigh(); + Double getAdjHigh(); + Double getLow(); + Double getAdjLow(); + Double getClose(); Double getAdjClose(); diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/YahooFinanceTenYearTreasuryYield.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/YahooFinanceQuote.java similarity index 75% rename from src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/YahooFinanceTenYearTreasuryYield.java rename to src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/YahooFinanceQuote.java index e0383fc..961f31a 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/YahooFinanceTenYearTreasuryYield.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/stock/YahooFinanceQuote.java @@ -11,7 +11,7 @@ @Builder @NoArgsConstructor @AllArgsConstructor -public class YahooFinanceTenYearTreasuryYield implements EndOfDayStockData { +public class YahooFinanceQuote implements EndOfDayStockData { private LocalDate date; private Double open; @@ -19,6 +19,7 @@ public class YahooFinanceTenYearTreasuryYield implements EndOfDayStockData { private Double high; private Double low; private Double adjClose; + private String ticker; @Override @@ -36,6 +37,16 @@ public Integer getAdjVolume() { return null; } + @Override + public Double getAdjHigh() { + return null; + } + + @Override + public Double getAdjLow() { + return null; + } + @Override public Double getSplitFactor() { return null; diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/tiingo/TiingoStockEndOfDayResponse.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/tiingo/TiingoStockEndOfDayResponse.java index 3fc631b..578ddec 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/model/tiingo/TiingoStockEndOfDayResponse.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/model/tiingo/TiingoStockEndOfDayResponse.java @@ -21,6 +21,8 @@ public class TiingoStockEndOfDayResponse implements EndOfDayStockData { private Integer volume; private Double adjOpen; private Double adjClose; + private Double adjHigh; + private Double adjLow; private Integer adjVolume; private Double divCash; private Double splitFactor; diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/CNNFearGreedDataLoadAPIService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/CNNFearGreedDataLoadAPIService.java new file mode 100644 index 0000000..53aa402 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/CNNFearGreedDataLoadAPIService.java @@ -0,0 +1,70 @@ +package com.dpgrandslam.stockdataservice.domain.service; + +import com.dpgrandslam.stockdataservice.adapter.apiclient.CNNFearGreedClient; +import com.dpgrandslam.stockdataservice.adapter.repository.FearGreedIndexRepository; +import com.dpgrandslam.stockdataservice.domain.model.CNNFearGreedResponse; +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.util.TimeUtils; +import com.github.benmanes.caffeine.cache.Cache; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Service(value = "CNNFearGreedDataLoadAPIService") +public class CNNFearGreedDataLoadAPIService extends FearGreedDataLoadService { + + public static final String PATH = "/data/fear-and-greed/"; + + private final TimeUtils timeUtils; + private final CNNFearGreedClient cnnFearGreedClient; + + public CNNFearGreedDataLoadAPIService(FearGreedIndexRepository fearGreedIndexRepository, + Cache, List> fearGreedCache, + CNNFearGreedClient cnnFearGreedClient, + TimeUtils timeUtils) { + super(fearGreedIndexRepository, fearGreedCache); + this.cnnFearGreedClient = cnnFearGreedClient; + this.timeUtils = timeUtils; + } + + + @Override + public Set loadCurrentFearGreedIndex() { + Set res = new HashSet<>(); + CNNFearGreedResponse fearGreedResponse = cnnFearGreedClient.getFearGreedData(LocalDate.now().toString()); + + FearGreedIndex fearGreedIndexOneWeek = new FearGreedIndex(); + fearGreedIndexOneWeek.setValue(fearGreedResponse.getFearAndGreed().getPreviousOneWeek().intValue()); + fearGreedIndexOneWeek.setTradeDate(timeUtils.getCurrentOrLastTradeDate(LocalDateTime.now().minusWeeks(1))); + res.add(fearGreedIndexOneWeek); + + FearGreedIndex fearGreedIndexOneMonth = new FearGreedIndex(); + fearGreedIndexOneMonth.setValue(fearGreedResponse.getFearAndGreed().getPreviousOneMonth().intValue()); + fearGreedIndexOneMonth.setTradeDate(timeUtils.getCurrentOrLastTradeDate(LocalDateTime.now().minusMonths(1))); + res.add(fearGreedIndexOneMonth); + + FearGreedIndex fearGreedIndexOneYear = new FearGreedIndex(); + fearGreedIndexOneYear.setValue(fearGreedResponse.getFearAndGreed().getPreviousOneYear().intValue()); + fearGreedIndexOneYear.setTradeDate(timeUtils.getCurrentOrLastTradeDate(LocalDateTime.now().minusYears(1))); + res.add(fearGreedIndexOneYear); + + FearGreedIndex fearGreedIndexPreviousClose = new FearGreedIndex(); + fearGreedIndexPreviousClose.setValue(fearGreedResponse.getFearAndGreed().getPreviousClose().intValue()); + fearGreedIndexPreviousClose.setTradeDate(timeUtils.getCurrentOrLastTradeDate(LocalDateTime.now().minusDays(1))); + res.add(fearGreedIndexPreviousClose); + + FearGreedIndex fearGreedIndexNow = new FearGreedIndex(); + fearGreedIndexNow.setValue(fearGreedResponse.getFearAndGreed().getScore().intValue()); + fearGreedIndexNow.setTradeDate(timeUtils.getCurrentOrLastTradeDate()); + res.add(fearGreedIndexNow); + + return res; + } + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/CNNFearGreedDataLoadWebService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/CNNFearGreedDataLoadWebService.java new file mode 100644 index 0000000..8921657 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/CNNFearGreedDataLoadWebService.java @@ -0,0 +1,41 @@ +package com.dpgrandslam.stockdataservice.domain.service; + +import com.dpgrandslam.stockdataservice.adapter.apiclient.WebpageLoader; +import com.dpgrandslam.stockdataservice.adapter.repository.FearGreedIndexRepository; +import com.dpgrandslam.stockdataservice.domain.config.ApiClientConfigurationProperties; +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.dpgrandslam.stockdataservice.domain.util.TimeUtils; +import com.github.benmanes.caffeine.cache.Cache; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDate; +import java.util.List; +import java.util.Set; + +@Service +public class CNNFearGreedDataLoadWebService extends FearGreedDataLoadService { + + private final WebpageLoader webpageLoader; + private final TimeUtils timeUtils; + private final ApiClientConfigurationProperties apiClientConfigurationProperties; + + + public CNNFearGreedDataLoadWebService(FearGreedIndexRepository fearGreedIndexRepository, + Cache, List> fearGreedBetweenDatesCache, + WebpageLoader webpageLoader, + TimeUtils timeUtils, + @Qualifier("CNNClientConfigurationProperties") ApiClientConfigurationProperties apiClientConfigurationProperties) { + super(fearGreedIndexRepository, fearGreedBetweenDatesCache); + this.webpageLoader = webpageLoader; + this.timeUtils = timeUtils; + this.apiClientConfigurationProperties = apiClientConfigurationProperties; + } + + @Override + public Set loadCurrentFearGreedIndex() { + return null; + } + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/FearGreedDataLoadService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/FearGreedDataLoadService.java new file mode 100644 index 0000000..b35ae84 --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/FearGreedDataLoadService.java @@ -0,0 +1,61 @@ +package com.dpgrandslam.stockdataservice.domain.service; + +import com.dpgrandslam.stockdataservice.adapter.repository.FearGreedIndexRepository; +import com.dpgrandslam.stockdataservice.domain.model.FearGreedIndex; +import com.github.benmanes.caffeine.cache.Cache; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.dao.DataIntegrityViolationException; + +import java.time.LocalDate; +import java.util.*; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Slf4j +public abstract class FearGreedDataLoadService { + + protected final FearGreedIndexRepository fearGreedIndexRepository; + private final Cache, List> fearGreedBetweenDatesCache; + + public abstract Set loadCurrentFearGreedIndex(); + + public List loadFearGreedDataBetweenDates(LocalDate startDate, LocalDate endDate) { + if (endDate.isAfter(LocalDate.now())) { + throw new IllegalArgumentException("endDate cannot be in the future"); + } + if (startDate.equals(LocalDate.now()) && endDate.equals(LocalDate.now())) { + return new ArrayList<>(loadCurrentFearGreedIndex()); + } + return fearGreedBetweenDatesCache.get(Pair.of(startDate, endDate), pair -> fearGreedIndexRepository + .findFearGreedIndexByTradeDateBetween(pair.getLeft(), pair.getRight()).stream() + .sorted(Comparator.comparing(FearGreedIndex::getTradeDate)) + .collect(Collectors.toList())); + } + + public FearGreedIndex saveFearGreedData(FearGreedIndex fearGreedIndex) { + try { + return fearGreedIndexRepository.save(fearGreedIndex); + } catch (DataIntegrityViolationException e) { + log.warn("Tried to save duplicate value for {}. Ignoring save. Message: {}", fearGreedIndex, e.getMessage()); + } + return fearGreedIndex; + } + + public List saveFearGreedData(Collection fearGreedIndices) { + try { + return fearGreedIndexRepository.saveAll(fearGreedIndices); + } catch (DataIntegrityViolationException e) { + return fearGreedIndices.stream() + .map(this::saveFearGreedData) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + } + + public Optional getFearGreedIndexOfDay(LocalDate date) { + return fearGreedIndexRepository.findFearGreedIndexByTradeDate(date); + } + +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/HistoricOptionsDataService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/HistoricOptionsDataService.java index b0182e4..da2d80f 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/HistoricOptionsDataService.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/HistoricOptionsDataService.java @@ -197,7 +197,11 @@ public void removeOption(HistoricalOption option) { } public HistoricalOption saveOption(HistoricalOption historicalOption) { - return historicalOptionRepository.saveAndFlush(historicalOption); + return historicalOptionRepository.save(historicalOption); + } + + public List saveOptions(Collection historicalOptions) { + return historicalOptionRepository.saveAllAndFlush(historicalOptions); } public Long countOptionsLoadedOnTradeDate(LocalDate tradeDate) { diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TenYearTreasuryYieldService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TenYearTreasuryYieldService.java index 3d2648f..2254e1b 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TenYearTreasuryYieldService.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TenYearTreasuryYieldService.java @@ -1,71 +1,34 @@ package com.dpgrandslam.stockdataservice.domain.service; -import com.dpgrandslam.stockdataservice.adapter.apiclient.WebpageLoader; -import com.dpgrandslam.stockdataservice.domain.config.ApiClientConfigurationProperties; -import com.dpgrandslam.stockdataservice.domain.error.TreasuryYieldLoadException; -import com.dpgrandslam.stockdataservice.domain.model.stock.YahooFinanceTenYearTreasuryYield; +import com.dpgrandslam.stockdataservice.domain.error.YahooFinanceQuoteLoadException; +import com.dpgrandslam.stockdataservice.domain.model.stock.YahooFinanceQuote; import com.github.benmanes.caffeine.cache.Cache; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Collectors; @Service -@RequiredArgsConstructor @Slf4j public class TenYearTreasuryYieldService { - private final WebpageLoader basicWebPageLoader; - private final Cache treasuryYieldCache; - @Autowired - @Qualifier("YahooFinanceApiClientConfigurationProperties") - private ApiClientConfigurationProperties clientConfigurationProperties; - - public YahooFinanceTenYearTreasuryYield getTreasuryYieldForDate(LocalDate date) { - final String url = clientConfigurationProperties.getUrlAndPort() + "/quote/%5ETNX/history?period1=" - + convertDate(date) + "&period2=" + convertDate(date.plusDays(1)) - + "&interval=1d&filter=history&frequency=1d&includeAdjustedClose=true"; - try { - return treasuryYieldCache.get(date, (d) -> parseDocument(basicWebPageLoader.parseUrl(url))); - } catch (Exception e) { - log.error("Error parsing document at url {}", url); - throw new TreasuryYieldLoadException(date); - } - } - - private Long convertDate(LocalDate date) { - return date.atStartOfDay().toInstant(ZoneOffset.UTC).getEpochSecond(); - } + private YahooFinanceHistoricStockDataLoadService historicStockDataLoadService; - private YahooFinanceTenYearTreasuryYield parseDocument(Document document) { - Element mainContent = document.body().selectFirst("div#Main"); - Element historicalPricesTable = mainContent.selectFirst("table[data-test='historical-prices']"); - Element firstTableRow = historicalPricesTable.selectFirst("tbody").selectFirst("tr"); - return parseTableRow(firstTableRow); - } - - private YahooFinanceTenYearTreasuryYield parseTableRow(Element tableRow) { - Elements dataPoints = tableRow.select("td"); - return YahooFinanceTenYearTreasuryYield.builder() - .date(parseYahooFinanceDate(dataPoints.get(0))) - .open(Double.parseDouble(dataPoints.get(1).selectFirst("span").text())) - .high(Double.parseDouble(dataPoints.get(2).selectFirst("span").text())) - .low(Double.parseDouble(dataPoints.get(3).selectFirst("span").text())) - .close(Double.parseDouble(dataPoints.get(4).selectFirst("span").text())) - .adjClose(Double.parseDouble(dataPoints.get(5).selectFirst("span").text())) - .build(); - } - - private LocalDate parseYahooFinanceDate(Element dateElement) { - return LocalDate.parse( dateElement.select("span").text(), DateTimeFormatter.ofPattern("MMM dd, yyyy")); + @Autowired + @Qualifier("TreasuryYieldCache") + private Cache, List> treasuryYieldCache; + + public List getTreasuryYieldForDate(LocalDate startDate, LocalDate endDate) { + final String ticker = "^TNX"; + return treasuryYieldCache.get(Pair.of(startDate, endDate), (d) -> historicStockDataLoadService.loadQuoteForDates(ticker, startDate, endDate).stream() + .filter(q -> (q.getDate().equals(startDate) || q.getDate().equals(endDate) || q.getDate().isBefore(endDate) || q.getDate().isAfter(startDate)) && q.getClose() != null) + .collect(Collectors.toList())); } } diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TrackedStockService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TrackedStockService.java index ef1cdb4..d5312e6 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TrackedStockService.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/TrackedStockService.java @@ -11,11 +11,13 @@ import org.springframework.stereotype.Service; import javax.persistence.EntityNotFoundException; +import javax.sound.midi.Track; import java.time.LocalDate; import java.time.ZoneId; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -53,13 +55,15 @@ public void setTrackedStockActive(String ticker, boolean isActive) { public void updateOptionUpdatedTimestamp(String ticker) { TrackedStock trackedStock = findByTicker(ticker); - trackedStock.setLastOptionsHistoricDataUpdate(timeUtils.getLastTradeDate()); + trackedStock.setLastOptionsHistoricDataUpdate(timeUtils.getCurrentOrLastTradeDate()); trackedStocksRepository.save(trackedStock); } public void addTrackedStocks(List tickers) { log.info("Attempting to track tickers {}", tickers); - List added = trackedStocksRepository.saveAll(tickers.stream() + Set existing = trackedStocksRepository.findAll().stream().map(TrackedStock::getTicker).collect(Collectors.toSet()); + List tickersToSave = tickers.stream().filter(x -> !existing.contains(x)).collect(Collectors.toList()); + List added = trackedStocksRepository.saveAll(tickersToSave.stream() .map(ticker -> verifyAndBuildTrackedStock(ticker) .orElseThrow(() -> new IllegalStateException("Ticker: " + ticker + " is not valid. Skipping addition."))) .collect(Collectors.toList())); @@ -75,6 +79,11 @@ public void addTrackedStock(String ticker) { }); } + public TrackedStock saveTrackedStock(TrackedStock trackedStock) { + log.info("Adding tracked stock with ticker: {}", trackedStock.getTicker()); + return trackedStocksRepository.save(trackedStock); + } + private Optional verifyAndBuildTrackedStock(String ticker) { StockMetaData stockMetaData = stockDataLoadService.getStockMetaData(ticker); TrackedStock trackedStock = new TrackedStock(); diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/VIXLoadService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/VIXLoadService.java new file mode 100644 index 0000000..dc47efe --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/VIXLoadService.java @@ -0,0 +1,34 @@ +package com.dpgrandslam.stockdataservice.domain.service; + +import com.dpgrandslam.stockdataservice.domain.model.stock.YahooFinanceQuote; +import com.github.benmanes.caffeine.cache.Cache; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class VIXLoadService { + + @Autowired + @Qualifier("VIXCache") + private Cache, List> vixCache; + + @Autowired + private YahooFinanceHistoricStockDataLoadService historicStockDataLoadService; + + public List loadVIXBetweenDates(LocalDate startDate, LocalDate endDate) { + log.info("Loading VIX data for dates {} - {}", startDate, endDate); + return vixCache.get(Pair.of(startDate, endDate), (pair) -> historicStockDataLoadService + .loadQuoteForDates("^VIX", startDate, endDate)); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceHistoricStockDataLoadService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceHistoricStockDataLoadService.java new file mode 100644 index 0000000..409f73d --- /dev/null +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceHistoricStockDataLoadService.java @@ -0,0 +1,117 @@ +package com.dpgrandslam.stockdataservice.domain.service; + +import com.dpgrandslam.stockdataservice.adapter.apiclient.WebpageLoader; +import com.dpgrandslam.stockdataservice.domain.config.ApiClientConfigurationProperties; +import com.dpgrandslam.stockdataservice.domain.error.YahooFinanceQuoteLoadException; +import com.dpgrandslam.stockdataservice.domain.model.stock.YahooFinanceQuote; +import com.dpgrandslam.stockdataservice.domain.util.TimeUtils; +import com.google.common.base.Charsets; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.net.URLEncoder; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@Slf4j +public class YahooFinanceHistoricStockDataLoadService { + + private final WebpageLoader basicWebPageLoader; + + @Autowired + @Qualifier("YahooFinanceApiClientConfigurationProperties") + private ApiClientConfigurationProperties clientConfigurationProperties; + + @Autowired + private TimeUtils timeUtils; + + + public List loadQuoteForDates(String ticker, LocalDate startDate, LocalDate endDate) { + // Break into 3 month chunks since yahoo finance is weird about long dates + List quotes = new ArrayList<>(); + LocalDate sd = startDate; + LocalDate ed = sd.plusMonths(3); + while (sd.isBefore(endDate)) { + quotes.addAll(doLoad(ticker, sd, ed).stream() + .filter(x -> x.getClose() != null) + .collect(Collectors.toList())); + sd = ed.plusDays(1); + ed = sd.plusMonths(3); + } + return quotes.stream().filter(x -> (x.getDate().isAfter(startDate) || x.getDate().equals(startDate)) + && (x.getDate().isBefore(endDate) || x.getDate().equals(endDate))) + .filter(x -> timeUtils.isTradingOpenOnDay(x.getDate())) + .sorted(Comparator.comparing(YahooFinanceQuote::getDate)) + .collect(Collectors.toList()); + } + + private List doLoad(String ticker, LocalDate startDate, LocalDate endDate) { + StringBuilder sb = new StringBuilder(clientConfigurationProperties.getUrlAndPort()); + sb.append("/quote/"); + sb.append(URLEncoder.encode(ticker, Charsets.UTF_8)); + sb.append("/history?period1="); + sb.append(convertDate(startDate)); + sb.append("&period2="); + Long period2 = convertDate(endDate == null || endDate.equals(startDate) ? startDate.plusDays(1) : endDate.plusDays(1)); + sb.append(period2); + sb.append( "&interval=1d&filter=history&frequency=1d&includeAdjustedClose=true"); + String url = sb.toString(); + List quotes; + try { + quotes = parseDocument(basicWebPageLoader.parseUrl(url)); + } catch (Exception e) { + log.error("Error parsing document at url {}", url, e); + throw new YahooFinanceQuoteLoadException(ticker, startDate, endDate, e); + } + if (quotes.isEmpty()) { + // If no quotes there was probably an error + throw new YahooFinanceQuoteLoadException(ticker, startDate, endDate); + } + quotes.forEach(quote -> quote.setTicker(ticker)); + return quotes; + } + + private Long convertDate(LocalDate date) { + return date.atStartOfDay().toInstant(ZoneOffset.UTC).getEpochSecond(); + } + + private List parseDocument(Document document) { + Element mainContent = document.body().selectFirst("div#Main"); + Element historicalPricesTable = mainContent.selectFirst("table[data-test='historical-prices']"); + List tableRows = historicalPricesTable.selectFirst("tbody").select("tr"); + return tableRows.stream().map(this::parseTableRow).filter(Objects::nonNull).collect(Collectors.toList()); + } + + private YahooFinanceQuote parseTableRow(Element tableRow) { + Elements dataPoints = tableRow.select("td"); + LocalDate d = parseYahooFinanceDate(dataPoints.get(0)); + YahooFinanceQuote.YahooFinanceQuoteBuilder builder = YahooFinanceQuote.builder() + .date(d); + + try { + builder.open(Double.parseDouble(dataPoints.get(1).selectFirst("span").text())) + .high(Double.parseDouble(dataPoints.get(2).selectFirst("span").text())) + .low(Double.parseDouble(dataPoints.get(3).selectFirst("span").text())) + .close(Double.parseDouble(dataPoints.get(4).selectFirst("span").text())) + .adjClose(Double.parseDouble(dataPoints.get(5).selectFirst("span").text())); + } catch (NullPointerException e) { + log.warn("Could not parse row for date {} in chart.", d); + } + return builder.build(); + } + + private LocalDate parseYahooFinanceDate(Element dateElement) { + return LocalDate.parse( dateElement.select("span").text(), DateTimeFormatter.ofPattern("MMM dd, yyyy")); + } +} diff --git a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceOptionsChainLoadService.java b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceOptionsChainLoadService.java index a5c7dac..07617dc 100644 --- a/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceOptionsChainLoadService.java +++ b/src/main/java/com/dpgrandslam/stockdataservice/domain/service/YahooFinanceOptionsChainLoadService.java @@ -10,7 +10,6 @@ import com.dpgrandslam.stockdataservice.domain.model.options.OptionsChain; import com.dpgrandslam.stockdataservice.domain.util.TimeUtils; import lombok.extern.slf4j.Slf4j; -import org.apache.tomcat.jni.Local; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -57,7 +56,7 @@ public OptionsChain loadLiveOptionsChainForClosestExpiration(String ticker) thro expiration = parseDocumentForExpirationDates(document).get(0); return buildOptionsChain(ticker, expiration, document); } catch (Exception e) { - eventPublisher.publishEvent(new OptionChainParseFailedEvent(this, ticker, expiration, timeUtils.getLastTradeDate())); + eventPublisher.publishEvent(new OptionChainParseFailedEvent(this, ticker, expiration, timeUtils.getCurrentOrLastTradeDate())); throw new OptionsChainLoadException(ticker, document.baseUri(), "Options chain load failure most likely due to too many calls.", e); } } @@ -75,7 +74,7 @@ public List loadFullLiveOptionsChain(String ticker) throws Options validateExpirationDates(ticker, expirationDates); } catch (AllOptionsExpirationDatesNotPresentException e) { log.warn("Not all options dates could be loaded from yahoo for ticker {}.", ticker, e); - e.getMissingDates().forEach(date -> eventPublisher.publishEvent(new OptionChainParseFailedEvent(this, ticker, date, timeUtils.getLastTradeDate()))); + e.getMissingDates().forEach(date -> eventPublisher.publishEvent(new OptionChainParseFailedEvent(this, ticker, date, timeUtils.getCurrentOrLastTradeDate()))); } for (LocalDate expiration : expirationDates) { try { @@ -104,7 +103,7 @@ public OptionsChain loadLiveOptionsChainForExpirationDate(String ticker, LocalDa if (document != null) { uri = document.baseUri(); } - eventPublisher.publishEvent(new OptionChainParseFailedEvent(this, ticker, expirationDate, timeUtils.getLastTradeDate())); + eventPublisher.publishEvent(new OptionChainParseFailedEvent(this, ticker, expirationDate, timeUtils.getCurrentOrLastTradeDate())); throw new OptionsChainLoadException(ticker, uri, "Options chain load failure most likely due to too many calls.", e); } return buildOptionsChain(ticker, expirationDate, document); @@ -171,7 +170,7 @@ private List