diff --git a/src/main/java/com/ueby/oss/loteria/crawlers/service/AbstractCrawlerService.java b/src/main/java/com/ueby/oss/loteria/crawlers/service/AbstractCrawlerService.java index ccc8cd2..04bfdbc 100644 --- a/src/main/java/com/ueby/oss/loteria/crawlers/service/AbstractCrawlerService.java +++ b/src/main/java/com/ueby/oss/loteria/crawlers/service/AbstractCrawlerService.java @@ -1,28 +1,61 @@ package com.ueby.oss.loteria.crawlers.service; import com.ueby.oss.loteria.crawlers.exception.CaixaWebCrawlerException; -import org.apache.http.client.fluent.Request; -import org.apache.http.client.fluent.Response; - +import com.ueby.oss.loteria.crawlers.util.SSLHelper; import java.io.IOException; import java.util.concurrent.TimeUnit; +import lombok.extern.log4j.Log4j2; +import org.apache.http.client.fluent.Executor; +import org.apache.http.client.fluent.Request; +import org.apache.http.client.fluent.Response; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; /** * @author Algarves, Khristian */ +@Log4j2 public abstract class AbstractCrawlerService { protected static final Long REQUEST_TIMEOUT = TimeUnit.MILLISECONDS.toSeconds(30); // 30sec - protected Response get(String url) throws CaixaWebCrawlerException { + protected Response get(String url) throws CaixaWebCrawlerException, IOException { + CloseableHttpClient httpClient = createHttpClient(); try { - return Request.Get(url) - .connectTimeout(REQUEST_TIMEOUT.intValue()) - .socketTimeout(REQUEST_TIMEOUT.intValue()) - .execute(); + Executor executor = Executor.newInstance(httpClient); + return executor.execute( + Request.Get(url) + .connectTimeout(REQUEST_TIMEOUT.intValue()) + .socketTimeout(REQUEST_TIMEOUT.intValue()) + ); } catch (IOException ie) { + log.error("An error occurred during the request - {}", url); throw new CaixaWebCrawlerException(ie.getLocalizedMessage(), ie); } } + private static CloseableHttpClient createHttpClient() { + PoolingHttpClientConnectionManager connectionManager = createConnectionManager(); + return HttpClientBuilder.create() + .setConnectionManager(connectionManager) + .build(); + } + + private static PoolingHttpClientConnectionManager createConnectionManager() { + final Registry sfr = RegistryBuilder.create() + .register("https", SSLHelper.socketFactory()) + .build(); + + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager( + sfr); + connectionManager.setDefaultMaxPerRoute(100); + connectionManager.setMaxTotal(200); + connectionManager.setValidateAfterInactivity(1000); + return connectionManager; + } + } diff --git a/src/main/java/com/ueby/oss/loteria/crawlers/service/CaixaWebCrawlerService.java b/src/main/java/com/ueby/oss/loteria/crawlers/service/CaixaWebCrawlerService.java index 64c7b2f..d6f5871 100644 --- a/src/main/java/com/ueby/oss/loteria/crawlers/service/CaixaWebCrawlerService.java +++ b/src/main/java/com/ueby/oss/loteria/crawlers/service/CaixaWebCrawlerService.java @@ -55,8 +55,7 @@ public CaixaWebCrawlerService byContestNumber(final int concourse) { /** * Call whenever the result is invalid for the request. * - * @throws InterruptedException When the process is interrupted. - * @throws CaixaWebCrawlerException If Number of attempts exhausted. + * @throws InterruptedException When the process is interrupted. */ private void recall() throws InterruptedException { if (attempts <= MAX_ATTEMPTS) { @@ -98,7 +97,7 @@ private void call() { responseStatus.getReasonPhrase()); } } catch (CaixaWebCrawlerException e) { - log.error(e.getLocalizedMessage(), e); + log.error(e.getMessage(), e); try { recall(); } catch (InterruptedException ie) { diff --git a/src/main/java/com/ueby/oss/loteria/crawlers/util/SSLHelper.java b/src/main/java/com/ueby/oss/loteria/crawlers/util/SSLHelper.java new file mode 100644 index 0000000..c599bbf --- /dev/null +++ b/src/main/java/com/ueby/oss/loteria/crawlers/util/SSLHelper.java @@ -0,0 +1,35 @@ +package com.ueby.oss.loteria.crawlers.util; + +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; + +public class SSLHelper { + + public static SSLConnectionSocketFactory socketFactory() { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } + } }; + + try { + SSLContext sslContext = SSLContext.getInstance("SSL"); + sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); + + return new SSLConnectionSocketFactory(sslContext); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException("Failed to create a SSL socket factory", e); + } + } +}