diff --git a/curation-module-core/pom.xml b/curation-module-core/pom.xml index 5ecac37e..ccd261c8 100644 --- a/curation-module-core/pom.xml +++ b/curation-module-core/pom.xml @@ -3,7 +3,7 @@ curation-module eu.clarin.cmdi - 3.1 + 3.1.1 4.0.0 curation-module-core @@ -93,7 +93,13 @@ eu.clarin.cmdi linkchecker - 3.0.1 + 4.0 + + + + eu.clarin.cmdi + resource-availability-status-api + 1.6-SNAPSHOT @@ -143,5 +149,5 @@ - 3.1 + 3.1.1 \ No newline at end of file diff --git a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/Configuration.java b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/Configuration.java index 4bdc47a8..b641efed 100644 --- a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/Configuration.java +++ b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/Configuration.java @@ -11,6 +11,11 @@ import java.util.Properties; import java.util.stream.Collectors; +import eu.clarin.cmdi.rasa.helpers.RasaFactory; +import eu.clarin.cmdi.rasa.helpers.impl.ACDHRasaFactory; +import eu.clarin.cmdi.rasa.linkResources.CheckedLinkResource; +import eu.clarin.cmdi.rasa.linkResources.LinkToBeCheckedResource; +import eu.clarin.cmdi.rasa.linkResources.StatisticsResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +35,7 @@ public class Configuration { public static Path OUTPUT_DIRECTORY = null; public static Path CACHE_DIRECTORY = null; public static Path COLLECTION_HTML_DIRECTORY = null; - public static int THREAD_POOL_SIZE=100; + public static int THREAD_POOL_SIZE = 100; public static Collection FACETS = null; public static int REDIRECT_FOLLOW_LIMIT; public static int TIMEOUT; @@ -44,6 +49,10 @@ public class Configuration { public static String BASE_URL; public static String CMD_STORAGE_URL; + public static CheckedLinkResource checkedLinkResource; + public static LinkToBeCheckedResource linkToBeCheckedResource; + public static StatisticsResource statisticsResource; + //this is a boolean that is set by core-module(false) and web-module(true) public static boolean enableProfileLoadTimer = false; @@ -63,6 +72,7 @@ public static void initDefault() throws IOException { config.load(Configuration.class.getResourceAsStream("/config.properties")); readProperties(config); //readProperties(new PropertiesConfiguration("config.properties")); + } private static void readProperties(Properties config) throws IOException { @@ -80,7 +90,7 @@ private static void readProperties(Properties config) throws IOException { } else { TIMEOUT = Integer.parseInt(timeout); } - THREAD_POOL_SIZE = Integer.valueOf(config.getProperty("THREAD_POOL_SIZE","100")); + THREAD_POOL_SIZE = Integer.valueOf(config.getProperty("THREAD_POOL_SIZE", "100")); String[] facets = config.getProperty("FACETS").split(","); FACETS = Arrays.asList(facets).stream().map(f -> f.trim()).collect(Collectors.toList()); @@ -105,10 +115,18 @@ private static void readProperties(Properties config) throws IOException { REDIRECT_FOLLOW_LIMIT = Integer.parseInt(redirectFollowLimit); } + + DATABASE = Boolean.parseBoolean(config.getProperty("DATABASE")); + if (DATABASE) { DATABASE_NAME = config.getProperty("DATABASE_NAME"); DATABASE_URI = config.getProperty("DATABASE_URI"); + + RasaFactory factory = new ACDHRasaFactory(DATABASE_NAME, DATABASE_URI); + checkedLinkResource = factory.getCheckedLinkResource(); + linkToBeCheckedResource = factory.getLinkToBeCheckedResource(); + statisticsResource = factory.getStatisticsResource(); } String vloConfigLocation = config.getProperty("VLO_CONFIG_LOCATION"); diff --git a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CMDInstanceReport.java b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CMDInstanceReport.java index d934c084..521664e0 100644 --- a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CMDInstanceReport.java +++ b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CMDInstanceReport.java @@ -12,6 +12,7 @@ import eu.clarin.cmdi.curation.report.CollectionReport.Record; import eu.clarin.cmdi.curation.utils.TimeUtils; import eu.clarin.cmdi.curation.xml.XMLMarshaller; +import eu.clarin.cmdi.rasa.links.CheckedLink; /** * @@ -115,11 +116,11 @@ public static class URLElement { @XmlAttribute(name = "redirectCount") public int redirectCount; - public URLElement convertFromLinkCheckerURLElement(eu.clarin.cmdi.linkchecker.urlElements.URLElement urlElement) { - url = urlElement.getUrl(); - method = urlElement.getMethod(); - message = urlElement.getMessage(); - status = urlElement.getStatus(); + public URLElement convertFromLinkCheckerURLElement(CheckedLink checkedLink) { + url = checkedLink.getUrl(); + method = checkedLink.getMethod(); + message = checkedLink.getMessage(); + status = checkedLink.getStatus(); //todo put this in its own class (categoryDeterminer) if (status == 200) { category = "Ok"; @@ -128,12 +129,12 @@ public URLElement convertFromLinkCheckerURLElement(eu.clarin.cmdi.linkchecker.ur } else { category = "Broken"; } - contentType = urlElement.getContentType(); - expectedContentType = urlElement.getExpectedMimeType(); - byteSize = urlElement.getByteSize(); - duration = TimeUtils.humanizeToTime(urlElement.getDuration()); - timestamp = TimeUtils.humanizeToDate(urlElement.getTimestamp()); - redirectCount = urlElement.getRedirectCount(); + contentType = checkedLink.getContentType(); + expectedContentType = checkedLink.getExpectedMimeType(); + byteSize = checkedLink.getByteSize(); + duration = TimeUtils.humanizeToTime(checkedLink.getDuration()); + timestamp = TimeUtils.humanizeToDate(checkedLink.getTimestamp()); + redirectCount = checkedLink.getRedirectCount(); return this; } diff --git a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionReport.java b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionReport.java index 97992992..e7ab2b3b 100644 --- a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionReport.java +++ b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionReport.java @@ -1,26 +1,16 @@ package eu.clarin.cmdi.curation.report; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -import javax.xml.bind.annotation.*; - -import com.mongodb.client.*; -import com.mongodb.client.model.Accumulators; -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Filters; import eu.clarin.cmdi.curation.main.Configuration; import eu.clarin.cmdi.curation.utils.TimeUtils; import eu.clarin.cmdi.curation.xml.XMLMarshaller; -import org.bson.Document; -import org.bson.conversions.Bson; +import eu.clarin.cmdi.rasa.filters.impl.ACDHStatisticsFilter; -import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Sorts.ascending; -import static com.mongodb.client.model.Sorts.orderBy; +import javax.xml.bind.annotation.*; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; /** * @@ -279,53 +269,27 @@ public void calculateAverageValues() { //statistics if (Configuration.DATABASE) { - MongoClient mongoClient; - if (Configuration.DATABASE_URI == null || Configuration.DATABASE_URI.isEmpty()) {//if it is empty, try localhost - mongoClient = MongoClients.create(); - } else { - mongoClient = MongoClients.create(Configuration.DATABASE_URI); - } - MongoDatabase database = mongoClient.getDatabase(Configuration.DATABASE_NAME); - - MongoCollection linksChecked = database.getCollection("linksChecked"); - MongoCollection linksToBeChecked = database.getCollection("linksToBeChecked"); - - AggregateIterable iterable = linksChecked.aggregate(Arrays.asList( - Aggregates.match(eq("collection", getName())), - Aggregates.group("$status", - Accumulators.sum("count", 1), - Accumulators.avg("avg_resp", "$duration"), - Accumulators.max("max_resp", "$duration") - ), - Aggregates.sort(orderBy(ascending("_id"))) - )); - - for (Document doc : iterable) { - Statistics statistics = new Statistics(); - statistics.avgRespTime = doc.getDouble("avg_resp"); - statistics.maxRespTime = doc.getLong("max_resp"); - statistics.statusCode = doc.getInteger("_id"); - if (statistics.statusCode == 200) { - statistics.category = "Ok"; - } else if (statistics.statusCode == 401 || statistics.statusCode == 405 || statistics.statusCode == 429) { - statistics.category = "Undetermined"; - } else { - statistics.category = "Broken"; - } - statistics.count = doc.getInteger("count"); - urlReport.status.add(statistics); + List stats = Configuration.statisticsResource.getStatusStatistics(getName()); + for (eu.clarin.cmdi.rasa.links.Statistics statistics : stats) { + Statistics xmlStatistics = new Statistics(); + xmlStatistics.avgRespTime = statistics.getAvgRespTime(); + xmlStatistics.maxRespTime = statistics.getMaxRespTime(); + xmlStatistics.statusCode = statistics.getStatus(); + xmlStatistics.category = statistics.getCategory(); + xmlStatistics.count = statistics.getCount(); + urlReport.status.add(xmlStatistics); } - long numOfCheckedLinks = linksChecked.countDocuments(eq("collection", getName())); + ACDHStatisticsFilter filter = new ACDHStatisticsFilter(getName(), null, false, false); + long numOfCheckedLinks = Configuration.statisticsResource.countLinksChecked(Optional.of(filter)); + filter = new ACDHStatisticsFilter(getName(), null, true, false); + long numOfBrokenLinks = Configuration.statisticsResource.countLinksChecked(Optional.of(filter)); - Bson brokenLinksFilter = Filters.and(Filters.eq("collection", getName()), Filters.not(Filters.in("status", 200, 302, 401, 405, 429))); - long numOfBrokenLinks = linksChecked.countDocuments(brokenLinksFilter); - - Bson undeterminedLinksFilter = Filters.and(Filters.eq("collection", getName()), Filters.in("status", 401, 405, 429)); - urlReport.totNumOfUndeterminedLinks = (int) linksChecked.countDocuments(undeterminedLinksFilter); + filter = new ACDHStatisticsFilter(getName(), null, false, true); + urlReport.totNumOfUndeterminedLinks = (int) Configuration.statisticsResource.countLinksChecked(Optional.of(filter)); urlReport.totNumOfBrokenLinks = (int) numOfBrokenLinks; urlReport.totNumOfCheckedLinks = (int) (numOfCheckedLinks); @@ -336,18 +300,10 @@ public void calculateAverageValues() { //because url validator works on a record basis and not collection basis, so the program //can only know about unique link numbers in a single record and not the whole collection. //thats why some database magic on the whole collection needed. - iterable = linksToBeChecked.aggregate(Arrays.asList( - Aggregates.match(eq("collection", getName())), - Aggregates.lookup("linksChecked", "url", "url", "checked") - )); - int duplicates = 0; - for (Document doc : iterable) { - if (!((List) doc.get("checked")).isEmpty()) { - duplicates++; - } - } + int duplicates = Configuration.statisticsResource.getDuplicateCount(getName()); - int numOfLinksToBeChecked = (int) linksToBeChecked.countDocuments(eq("collection", getName())); + filter = new ACDHStatisticsFilter(getName(), null, false, false); + int numOfLinksToBeChecked = (int) Configuration.statisticsResource.countLinksToBeChecked(Optional.of(filter)); urlReport.totNumOfUniqueLinks = ((int) numOfCheckedLinks) + numOfLinksToBeChecked - duplicates; @@ -355,19 +311,9 @@ public void calculateAverageValues() { urlReport.avgNumOfUniqueLinks = (double) urlReport.totNumOfUniqueLinks / fileReport.numOfFiles; urlReport.avgNumOfBrokenLinks = 1.0 * (double) urlReport.totNumOfBrokenLinks / fileReport.numOfFiles; - AggregateIterable aggregate = linksChecked.aggregate( - Arrays.asList( - Aggregates.match(eq("collection", getName())), - Aggregates.group(null, - Accumulators.avg("avg_resp", "$duration"), - Accumulators.max("max_resp", "$duration") - ))); - Document result = aggregate.first(); - - if(result!=null){ - urlReport.avgRespTime = result.getDouble("avg_resp"); - urlReport.maxRespTime = result.getLong("max_resp"); - } + eu.clarin.cmdi.rasa.links.Statistics statistics = Configuration.statisticsResource.getOverallStatistics(getName()); + urlReport.avgRespTime = statistics.getAvgRespTime(); + urlReport.maxRespTime = statistics.getMaxRespTime(); } diff --git a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionsReport.java b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionsReport.java index 0a4535b3..94416e30 100644 --- a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionsReport.java +++ b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionsReport.java @@ -130,6 +130,7 @@ public Collection(CollectionReport report) { this.numOfResProxies = report.resProxyReport.totNumOfResProxies; this.ratioOfValidRecords = report.xmlValidationReport.ratioOfValidRecords; this.avgNumOfEmptyXMLElements = report.xmlPopulatedReport.avgXMLEmptyElement; + this.avgFacetCoverage = report.facetReport.coverage; report.facetReport.facet.forEach(f -> this.facets.add(new Facet(f.name, f.coverage))); } diff --git a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/LinkCheckerReport.java b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/LinkCheckerReport.java index 59e40ecf..268067da 100644 --- a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/LinkCheckerReport.java +++ b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/LinkCheckerReport.java @@ -1,26 +1,14 @@ package eu.clarin.cmdi.curation.report; -import java.io.OutputStream; -import java.util.*; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; - -import com.mongodb.client.*; -import com.mongodb.client.model.Accumulators; -import com.mongodb.client.model.Aggregates; import eu.clarin.cmdi.curation.main.Configuration; import eu.clarin.cmdi.curation.report.CollectionReport.Statistics; import eu.clarin.cmdi.curation.xml.XMLMarshaller; -import org.bson.Document; -import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Sorts.ascending; -import static com.mongodb.client.model.Sorts.orderBy; +import javax.xml.bind.annotation.*; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; /* * @author Wolfgang Walter SAUER (wowasa) <wolfgang.sauer@oeaw.ac.at> @@ -168,54 +156,24 @@ public Collection getStatistics() { } public Overall() { - MongoClient mongoClient; - if (Configuration.DATABASE_URI == null || Configuration.DATABASE_URI.isEmpty()) {//if it is empty, try localhost - mongoClient = MongoClients.create(); - } else { - mongoClient = MongoClients.create(Configuration.DATABASE_URI); - } - MongoDatabase database = mongoClient.getDatabase(Configuration.DATABASE_NAME); - - MongoCollection linksChecked = database.getCollection("linksChecked"); - - AggregateIterable iterable = linksChecked.aggregate(Arrays.asList( - Aggregates.group("$status", - Accumulators.sum("count", 1), - Accumulators.avg("avg_resp", "$duration"), - Accumulators.max("max_resp", "$duration") - ), - Aggregates.sort(orderBy(ascending("_id"))) - )); - - for (Document doc : iterable) { - Statistics statistics = new Statistics(); - statistics.avgRespTime = doc.getDouble("avg_resp"); - statistics.maxRespTime = doc.getLong("max_resp"); - statistics.statusCode = doc.getInteger("_id"); - if (statistics.statusCode == 200) { - statistics.category = "Ok"; - } else if (statistics.statusCode == 401 || statistics.statusCode == 405 || statistics.statusCode == 429) { - statistics.category = "Undetermined"; - } else { - statistics.category = "Broken"; - } - statistics.count = doc.getInteger("count"); - - this.statistics.add(statistics); + + List stats = Configuration.statisticsResource.getStatusStatistics("Overall"); + + for (eu.clarin.cmdi.rasa.links.Statistics statistics : stats) { + Statistics xmlStatistics = new Statistics(); + xmlStatistics.avgRespTime = statistics.getAvgRespTime(); + xmlStatistics.maxRespTime = statistics.getMaxRespTime(); + xmlStatistics.statusCode = statistics.getStatus(); + xmlStatistics.category = statistics.getCategory(); + xmlStatistics.count = statistics.getCount(); + this.statistics.add(xmlStatistics); } - AggregateIterable aggregate = linksChecked.aggregate( - Arrays.asList( - Aggregates.group(null, - Accumulators.avg("avg_resp", "$duration"), - Accumulators.max("max_resp", "$duration"), - Accumulators.sum("count", 1) - ))); - Document result = aggregate.first(); - this.avgRespTime = result.getDouble("avg_resp"); - this.maxRespTime = result.getLong("max_resp"); - this.count = result.getInteger("count"); + eu.clarin.cmdi.rasa.links.Statistics statistics = Configuration.statisticsResource.getOverallStatistics("Overall"); + this.avgRespTime = statistics.getAvgRespTime(); + this.count = statistics.getCount(); + this.maxRespTime = statistics.getMaxRespTime(); } } } diff --git a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/subprocessor/URLValidator.java b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/subprocessor/URLValidator.java index 0f6fbd71..f30e14fc 100644 --- a/curation-module-core/src/main/java/eu/clarin/cmdi/curation/subprocessor/URLValidator.java +++ b/curation-module-core/src/main/java/eu/clarin/cmdi/curation/subprocessor/URLValidator.java @@ -1,10 +1,6 @@ package eu.clarin.cmdi.curation.subprocessor; -import com.mongodb.MongoException; -import com.mongodb.client.*; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.IndexOptions; import eu.clarin.cmdi.curation.entities.CMDInstance; import eu.clarin.cmdi.curation.main.Configuration; import eu.clarin.cmdi.curation.report.CMDInstanceReport; @@ -13,13 +9,12 @@ import eu.clarin.cmdi.curation.report.Severity; import eu.clarin.cmdi.curation.utils.TimeUtils; import eu.clarin.cmdi.linkchecker.httpLinkChecker.HTTPLinkChecker; -import eu.clarin.cmdi.linkchecker.urlElements.URLElement; -import eu.clarin.cmdi.linkchecker.urlElements.URLElementToBeChecked; +import eu.clarin.cmdi.rasa.filters.impl.ACDHStatisticsFilter; +import eu.clarin.cmdi.rasa.links.CheckedLink; +import eu.clarin.cmdi.rasa.links.LinkToBeChecked; import eu.clarin.cmdi.vlo.importer.CMDIData; import eu.clarin.cmdi.vlo.importer.Resource; import eu.clarin.cmdi.vlo.importer.processor.ValueSet; -import org.bson.Document; -import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,8 +22,6 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; -import static com.mongodb.client.model.Filters.eq; - /** * */ @@ -37,44 +30,6 @@ public class URLValidator extends CMDSubprocessor { private static final Logger _logger = LoggerFactory.getLogger(URLValidator.class); - private static final MongoClient _mongoClient; - private static MongoDatabase database; - private static MongoCollection linksToBeChecked; - private static MongoCollection linksChecked; - - static { //since MongoClient is already a connection pool only one instance should exist in the application - - if (Configuration.DATABASE) { - _logger.info("Connecting to database..."); - if (Configuration.DATABASE_URI == null || Configuration.DATABASE_URI.isEmpty()) {//if it is empty, try localhost - _mongoClient = MongoClients.create(); - } else { - _mongoClient = MongoClients.create(Configuration.DATABASE_URI); - } - - database = _mongoClient.getDatabase(Configuration.DATABASE_NAME); - linksToBeChecked = database.getCollection("linksToBeChecked"); - linksChecked = database.getCollection("linksChecked"); - - _logger.info("Connected to database."); - //Ensure that "url" is a unique index - IndexOptions indexOptions = new IndexOptions().unique(true); -// linksChecked.createIndex(new Document("url", 1), indexOptions); - - //ensure indexes to speed up queries later -// linksChecked.createIndex(Indexes.ascending("record")); -// linksChecked.createIndex(Indexes.ascending("collection")); -// linksChecked.createIndex(Indexes.ascending("status")); -// linksChecked.createIndex(Indexes.ascending("record", "status")); -// linksChecked.createIndex(Indexes.ascending("collection", "status")); - - } else { - - _mongoClient = null; - } - - } - @Override public void process(CMDInstance entity, CMDInstanceReport report) { @@ -117,46 +72,26 @@ public void process(CMDInstance entity, CMDInstanceReport report, String parentN _logger.info("Checking database for url: " + url); - Bson filter = Filters.and(eq("collection", parentName), eq("url", url)); - MongoCursor cursor = linksChecked.find(filter).iterator(); - - //because urls are unique in the database if cursor has next, it found the only one. If not, the url wasn't found. - if (cursor.hasNext()) { - //dont do anything, url is already checked and in the database... -// -// URLElement urlElement = new URLElement(cursor.next()); -// -// addMessageForStatusCode(urlElement.getStatus(), url); - -// CMDInstanceReport.URLElement urlElementReport = new CMDInstanceReport.URLElement().convertFromLinkCheckerURLElement(urlElement); -// report.addURLElement(urlElementReport); - - } else { + CheckedLink checkedLink = Configuration.checkedLinkResource.get(url, parentName); + if (checkedLink == null) { String expectedMimeType = urlMap.get(url).getMimeType(); expectedMimeType = expectedMimeType == null ? "Not Specified" : expectedMimeType; String finalRecord = report.getName(); String finalCollection = parentName != null ? parentName : finalRecord; - URLElementToBeChecked urlElementToBeChecked = new URLElementToBeChecked(url, finalRecord, finalCollection, expectedMimeType); - - try { - linksToBeChecked.insertOne(urlElementToBeChecked.getMongoDocument()); - } catch (MongoException e) { - //duplicate key error - //the url is already in the linksToBeChecked, do nothing - } + LinkToBeChecked linkToBeChecked = new LinkToBeChecked(url, finalRecord, finalCollection, expectedMimeType); + Configuration.linkToBeCheckedResource.save(linkToBeChecked); - } + }//else dont do anything, it is already in linksChecked - cursor.close(); }); - removeOldURLs(urlMap.keySet(), report.getName(), parentName); +// removeOldURLs(urlMap.keySet(), report.getName(), parentName); - report.urlReport = createURLReport(numOfLinks.longValue(), report.getName()); + report.urlReport = createCollectionURLReport(numOfLinks.longValue(), report.getName()); } else { @@ -175,11 +110,11 @@ public void process(CMDInstance entity, CMDInstanceReport report, String parentN try {// check if URL is broken _logger.info("Checking url: " + url); - URLElement urlElement = httpLinkChecker.checkLink(url, 0, 0, url);//redirect follow level is current level, because this is the first request it is set to 0 - urlElement.setExpectedMimeType(expectedMimeType); + CheckedLink checkedLink = httpLinkChecker.checkLink(url, 0, 0, url);//redirect follow level is current level, because this is the first request it is set to 0 + checkedLink.setExpectedMimeType(expectedMimeType); - if (urlElement.getStatus() != 200 && urlElement.getStatus() != 302) { - if (urlElement.getStatus() == 401 || urlElement.getStatus() == 405 || urlElement.getStatus() == 429) { + if (checkedLink.getStatus() != 200 && checkedLink.getStatus() != 302) { + if (checkedLink.getStatus() == 401 || checkedLink.getStatus() == 405 || checkedLink.getStatus() == 429) { numOfUndeterminedLinks++; } else { numOfBrokenLinks++; @@ -187,9 +122,9 @@ public void process(CMDInstance entity, CMDInstanceReport report, String parentN } - addMessageForStatusCode(urlElement.getStatus(), url); + addMessageForStatusCode(checkedLink.getStatus(), url); - CMDInstanceReport.URLElement urlElementReport = new CMDInstanceReport.URLElement().convertFromLinkCheckerURLElement(urlElement); + CMDInstanceReport.URLElement urlElementReport = new CMDInstanceReport.URLElement().convertFromLinkCheckerURLElement(checkedLink); report.addURLElement(urlElementReport); @@ -216,55 +151,17 @@ public void process(CMDInstance entity, CMDInstanceReport report, String parentN } - report.urlReport = createURLReport(numOfLinks.get(), numOfUniqueLinks, numOfBrokenLinks, numOfUndeterminedLinks, report.getName()); + report.urlReport = createInstanceURLReport(numOfLinks.get(), numOfUniqueLinks, numOfBrokenLinks, numOfUndeterminedLinks); } } else { - report.urlReport = createURLReport(numOfLinks.get(), report.getName()); + report.urlReport = createDisabledURLReport(numOfLinks.get()); addMessage(Severity.INFO, "Link validation is disabled"); } } - private void removeOldURLs(Collection links, String recordName, String collectionName) { - //some old runs may have produced links that are not in the records anymore. - //so to clean up the database, we move all of such links to history. - - Bson filter = Filters.and(Filters.eq("collection", collectionName), Filters.eq("record", recordName), Filters.not(Filters.in("url", links))); - MongoCursor cursor = linksChecked.find(filter).iterator(); - - while (cursor.hasNext()) { - - URLElement urlElement = new URLElement(cursor.next()); - - moveToHistory(urlElement); - - } - - //also remove them from linkstobechecked so that they are not checked unnecessarily - linksToBeChecked.deleteMany(filter); - - - } - - private void moveToHistory(URLElement urlElement) { - String url = urlElement.getUrl(); - try { - database.getCollection("linksCheckedHistory").insertOne(urlElement.getMongoDocument()); - } catch (MongoException e) { - //shouldnt happen, but if it does continue the loop - _logger.error("Error with the url: " + url + " while cleaning linkschecked (removing links from older runs). Exception message: " + e.getMessage()); - } - - try { - linksChecked.deleteOne(eq("url", url)); - } catch (MongoException e) { - //shouldnt happen, but if it does continue the loop - _logger.error("Error with the url: " + url + " while cleaning linkschecked (removing links from older runs). Exception message: " + e.getMessage()); - } - } - private void addMessageForStatusCode(int responseCode, String url) { if (responseCode == 200 || responseCode == 302) { @@ -284,8 +181,7 @@ public Score calculateScore(CMDInstanceReport report) { return new Score(score, 1.0, "url-validation", msgs); } - //this method is used for instances - private URLReport createURLReport(long numOfLinks, long numOfUniqueLinks, long numOfBrokenLinks, long numOfUndeterminedLinks, String name) { + private URLReport createInstanceURLReport(long numOfLinks, long numOfUniqueLinks, long numOfBrokenLinks, long numOfUndeterminedLinks) { URLReport report = new URLReport(); report.numOfLinks = numOfLinks; @@ -302,27 +198,30 @@ private URLReport createURLReport(long numOfLinks, long numOfUniqueLinks, long n return report; } - //this method is used for collections - private URLReport createURLReport(long numOfLinks, String name) { + private URLReport createCollectionURLReport(long numOfLinks, String name) { URLReport report = new URLReport(); report.numOfLinks = numOfLinks; + ACDHStatisticsFilter filter = new ACDHStatisticsFilter(null, name, false, false); + long numOfCheckedLinks = Configuration.statisticsResource.countLinksChecked(Optional.of(filter)); - if (_mongoClient != null) { + filter = new ACDHStatisticsFilter(null, name, true, false); + long numOfBrokenLinks = Configuration.statisticsResource.countLinksChecked(Optional.of(filter)); - Bson checkedLinksFilter = Filters.eq("record", name); - long numOfCheckedLinks = linksChecked.countDocuments(checkedLinksFilter); + filter = new ACDHStatisticsFilter(null, name, false, true); + long numOfUndeterminedLinks = Configuration.statisticsResource.countLinksChecked(Optional.of(filter)); - Bson brokenLinksFilter = Filters.and(Filters.eq("record", name), Filters.not(Filters.in("status", 200, 302, 401, 405, 429))); - long numOfBrokenLinks = database.getCollection("linkesChecked").countDocuments(brokenLinksFilter); + long numOfCheckedUndeterminedRemoved = numOfCheckedLinks - numOfUndeterminedLinks; + report.percOfValidLinks = numOfCheckedLinks == 0 ? 0 : (numOfCheckedUndeterminedRemoved - numOfBrokenLinks) / (double) numOfCheckedUndeterminedRemoved; - Bson undeterminedLinksFilter = Filters.and(Filters.eq("record", name), Filters.in("status", 401, 405, 429)); - long numOfUndeterminedLinks = database.getCollection("linkesChecked").countDocuments(undeterminedLinksFilter); + return report; + } - long numOfCheckedUndeterminedRemoved = numOfCheckedLinks - numOfUndeterminedLinks; - report.percOfValidLinks = numOfCheckedLinks == 0 ? 0 : (numOfCheckedUndeterminedRemoved - numOfBrokenLinks) / (double) numOfCheckedUndeterminedRemoved; - } + private URLReport createDisabledURLReport(long numOfLinks) { + URLReport report = new URLReport(); + report.numOfLinks = numOfLinks; + return report; } diff --git a/curation-module-core/src/main/resources/xslt/CollectionsReport2HTML.xsl b/curation-module-core/src/main/resources/xslt/CollectionsReport2HTML.xsl index c36a1a97..66328fa8 100644 --- a/curation-module-core/src/main/resources/xslt/CollectionsReport2HTML.xsl +++ b/curation-module-core/src/main/resources/xslt/CollectionsReport2HTML.xsl @@ -30,7 +30,7 @@ Avg Num Of Res Proxies Ratio Of Valid Records Avg Num Of Empty XML Elements - Avg Facet Coverage + Ratio Facets Non Zero diff --git a/curation-module-core/src/main/resources/xslt/CollectionsReport2TSV.xsl b/curation-module-core/src/main/resources/xslt/CollectionsReport2TSV.xsl index cf0f5aff..72ee9a49 100644 --- a/curation-module-core/src/main/resources/xslt/CollectionsReport2TSV.xsl +++ b/curation-module-core/src/main/resources/xslt/CollectionsReport2TSV.xsl @@ -35,7 +35,7 @@ Avg Num Of Empty XML Elements -Avg Facet Coverage +Ratio Facets Non Zero diff --git a/curation-module-core/src/test/java/eu/clarin/cmdi/curation/cr/profile_parser/CurationModuleTest.java b/curation-module-core/src/test/java/eu/clarin/cmdi/curation/cr/profile_parser/CurationModuleTest.java index 00b565bd..8bac52e4 100644 --- a/curation-module-core/src/test/java/eu/clarin/cmdi/curation/cr/profile_parser/CurationModuleTest.java +++ b/curation-module-core/src/test/java/eu/clarin/cmdi/curation/cr/profile_parser/CurationModuleTest.java @@ -1,5 +1,7 @@ package eu.clarin.cmdi.curation.cr.profile_parser; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import eu.clarin.cmdi.curation.main.Configuration; @@ -14,6 +16,11 @@ public class CurationModuleTest extends TestBase{ + @Before + public void init() { + super.init(); + } + @Test public void testCMD_1_1() { Report report = null; diff --git a/curation-module-web/pom.xml b/curation-module-web/pom.xml index 0061ff89..575da503 100644 --- a/curation-module-web/pom.xml +++ b/curation-module-web/pom.xml @@ -6,7 +6,7 @@ curation-module eu.clarin.cmdi - 3.1 + 3.1.1 4.0.0 @@ -83,7 +83,7 @@ eu.clarin.cmdi curation-module-core - 3.1 + 3.1.1 compile diff --git a/curation-module-web/src/main/java/eu/clarin/helpers/LinkCheckerStatisticsHelper.java b/curation-module-web/src/main/java/eu/clarin/helpers/LinkCheckerStatisticsHelper.java index 4944e759..bfed902b 100644 --- a/curation-module-web/src/main/java/eu/clarin/helpers/LinkCheckerStatisticsHelper.java +++ b/curation-module-web/src/main/java/eu/clarin/helpers/LinkCheckerStatisticsHelper.java @@ -1,120 +1,108 @@ package eu.clarin.helpers; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.Accumulators; -import com.mongodb.client.model.Aggregates; import eu.clarin.cmdi.curation.utils.TimeUtils; -import eu.clarin.cmdi.linkchecker.urlElements.URLElement; -import org.bson.Document; +import eu.clarin.cmdi.rasa.filters.impl.ACDHCheckedLinkFilter; +import eu.clarin.cmdi.rasa.links.CheckedLink; +import eu.clarin.main.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; -import static com.mongodb.client.model.Filters.*; -import static com.mongodb.client.model.Sorts.ascending; -import static com.mongodb.client.model.Sorts.orderBy; - -//this class connects with mongodb and creates a html report out of the queries +//this class creates a html report public class LinkCheckerStatisticsHelper { private static final Logger _logger = LoggerFactory.getLogger(LinkCheckerStatisticsHelper.class); - private MongoCollection linksChecked; - - public LinkCheckerStatisticsHelper(MongoDatabase database) { - this.linksChecked = database.getCollection("linksChecked"); - } - - private AggregateIterable getStatusStatistics() { - - return linksChecked.aggregate(Arrays.asList( - Aggregates.group("$status", - Accumulators.sum("count", 1), - Accumulators.avg("avg_resp", "$duration"), - Accumulators.max("max_resp", "$duration") - ), - Aggregates.sort(orderBy(ascending("_id"))) - )); - - } - - private AggregateIterable getStatusStatistics(String collectionName) { - if (collectionName.equals("Overall")) { - return getStatusStatistics(); - } - - return linksChecked.aggregate(Arrays.asList( - Aggregates.match(eq("collection", collectionName)), - Aggregates.group("$status", - Accumulators.sum("count", 1), - Accumulators.avg("avg_resp", "$duration"), - Accumulators.max("max_resp", "$duration") - ), - Aggregates.sort(orderBy(ascending("_id"))) - )); - } - - private AggregateIterable getStatusStatisticsTotal() { - - return linksChecked.aggregate(Arrays.asList( - Aggregates.group("_id", - Accumulators.sum("count", 1) - ) - )); - - } - - private AggregateIterable getStatusStatisticsTotal(String collectionName) { - if (collectionName.equals("Overall")) { - return getStatusStatisticsTotal(); - } - - return linksChecked.aggregate(Arrays.asList( - Aggregates.match(eq("collection", collectionName)), - Aggregates.group("_id", - Accumulators.sum("count", 1) - ) - )); - - } - - //this method doesn't take 0 status codes into consideration. 0 means there was an error in the URL and there was no request sent. - private AggregateIterable getStatusStatisticsAvg() { - - return linksChecked.aggregate(Arrays.asList( - Aggregates.match(not(eq("status", 0))), - Aggregates.group("_id", - Accumulators.avg("avg_resp", "$duration") - ) - )); - } - - //this method doesn't take 0 status codes into consideration. 0 means there was an error in the URL and there was no request sent. - private AggregateIterable getStatusStatisticsAvg(String collectionName) { - if (collectionName.equals("Overall")) { - return getStatusStatisticsAvg(); - } - - return linksChecked.aggregate(Arrays.asList( - Aggregates.match(and(eq("collection", collectionName), not(eq("status", 0)))), - Aggregates.group("_id", - Accumulators.avg("avg_resp", "$duration") - ) - )); - - } - - public String createURLTable(String collectionName, int status) { +// private AggregateIterable getStatusStatistics() { +// +// return linksChecked.aggregate(Arrays.asList( +// Aggregates.group("$status", +// Accumulators.sum("count", 1), +// Accumulators.avg("avg_resp", "$duration"), +// Accumulators.max("max_resp", "$duration") +// ), +// Aggregates.sort(orderBy(ascending("_id"))) +// )); +// +// } + +// private AggregateIterable getStatusStatistics(String collectionName) { +// if (collectionName.equals("Overall")) { +// return getStatusStatistics(); +// } +// +// return linksChecked.aggregate(Arrays.asList( +// Aggregates.match(eq("collection", collectionName)), +// Aggregates.group("$status", +// Accumulators.sum("count", 1), +// Accumulators.avg("avg_resp", "$duration"), +// Accumulators.max("max_resp", "$duration") +// ), +// Aggregates.sort(orderBy(ascending("_id"))) +// )); +// } + +// private AggregateIterable getStatusStatisticsTotal() { +// +// return linksChecked.aggregate(Arrays.asList( +// Aggregates.group("_id", +// Accumulators.sum("count", 1) +// ) +// )); +// +// } +// +// private AggregateIterable getStatusStatisticsTotal(String collectionName) { +// if (collectionName.equals("Overall")) { +// return getStatusStatisticsTotal(); +// } +// +// return linksChecked.aggregate(Arrays.asList( +// Aggregates.match(eq("collection", collectionName)), +// Aggregates.group("_id", +// Accumulators.sum("count", 1) +// ) +// )); +// +// } + +// //this method doesn't take 0 status codes into consideration. 0 means there was an error in the URL and there was no request sent. +// private AggregateIterable getStatusStatisticsAvg() { +// +// return linksChecked.aggregate(Arrays.asList( +// Aggregates.match(not(eq("status", 0))), +// Aggregates.group("_id", +// Accumulators.avg("avg_resp", "$duration") +// ) +// )); +// } +// +// +// //this method doesn't take 0 status codes into consideration. 0 means there was an error in the URL and there was no request sent. +// private AggregateIterable getStatusStatisticsAvg(String collectionName) { +// if (collectionName.equals("Overall")) { +// return getStatusStatisticsAvg(); +// } +// +// return linksChecked.aggregate(Arrays.asList( +// Aggregates.match(and(eq("collection", collectionName), not(eq("status", 0)))), +// Aggregates.group("_id", +// Accumulators.avg("avg_resp", "$duration") +// ) +// )); +// +// } + + public static String createURLTable(String collectionName, int status) { StringBuilder sb = new StringBuilder(); sb.append("
"); - sb.append("

Link Checking Statistics (Status:"+status+"):

"); - sb.append("

").append(collectionName.replace("_"," ")).append(":

"); + sb.append("

Link Checking Statistics (Status:" + status + "):

"); + sb.append("

").append(collectionName.replace("_", " ")).append(":

"); List columnNames = Arrays.asList("Url", "Category", "Info", "Record"); @@ -139,100 +127,87 @@ public String createURLTable(String collectionName, int status) { return sb.toString(); } - public String getHtmlRowsInBatch(String collectionName, int status, int batchCount) { + public static String getHtmlRowsInBatch(String collectionName, int status, int batchCount) { int start = batchCount * 100; int end = start + 100; StringBuilder sb = new StringBuilder(); - MongoCursor cursor; - if (collectionName.equals("Overall")) { - cursor = linksChecked.find(eq("status", status)).skip(start).limit(end).iterator(); - } else { - cursor = linksChecked.find(and(eq("status", status), eq("collection", collectionName))).skip(batchCount * 100).limit(batchCount * 100 + 100).iterator(); - } - try { - while (cursor.hasNext()) { - - - sb.append(""); - Document doc = cursor.next(); - URLElement urlElement = new URLElement(doc); - - //todo move category into the database instead of checking it everywhere - - - String url = urlElement.getUrl(); - String urlWithBreak = url.replace("_", "_"); - - String category; - if (status == 200) { - category = "Ok"; - sb.append(""); - sb.append("").append(urlWithBreak).append(""); - sb.append(""); - sb.append(""); - sb.append(category); - sb.append(""); - } else if (status == 401 || status == 405 || status == 429) { - category = "Undetermined"; - sb.append(""); - sb.append("").append(urlWithBreak).append(""); - sb.append(""); - sb.append(""); - sb.append(category); - sb.append(""); - } else { - category = "Broken"; - sb.append(""); - sb.append("").append(urlWithBreak).append(""); - sb.append(""); - sb.append(""); - sb.append(category); - sb.append(""); - } - - //button - sb.append(""); - sb.append(""); - sb.append(""); + ACDHCheckedLinkFilter filter = new ACDHCheckedLinkFilter(collectionName,status); + Stream links = Configuration.checkedLinkResource.get(Optional.of(filter),start,end); + + links.forEach(checkedLink -> { + sb.append(""); + //todo move category into the database instead of checking it everywhere + String url = checkedLink.getUrl(); + String urlWithBreak = url.replace("_", "_"); - sb.append(""); - //some html css table too wide work around - String record = urlElement.getRecord(); - if (record != null) { - record = record.replace("_", "_"); - sb.append(record); - } - - sb.append(""); - - //info - sb.append(""); - String message = urlElement.getMessage().replace("_", "_"); - sb.append("Message: ").append(message).append("
"); - //because this field is new, older entries dont have it and it results in null, so a null check to make it more user friendly - String expectedContent = urlElement.getExpectedMimeType() == null ? "Not Specified" : urlElement.getExpectedMimeType(); - String content = urlElement.getContentType(); - - sb.append("Expected Content Type: ").append(expectedContent).append("
"); - sb.append("Content Type: ").append(content).append("
"); - sb.append("Byte Size: ").append(urlElement.getByteSize()).append("
"); - sb.append("Request Duration(ms): ").append(urlElement.getDuration()).append("
"); - - String method = urlElement.getMethod() == null ? "N/A" : urlElement.getMethod(); - sb.append("Method: ").append(method).append("
"); - sb.append("Timestamp: ").append(TimeUtils.humanizeToDate(urlElement.getTimestamp())); + String category; + if (status == 200) { + category = "Ok"; + sb.append(""); + sb.append("").append(urlWithBreak).append(""); + sb.append(""); + sb.append(""); + sb.append(category); + sb.append(""); + } else if (status == 401 || status == 405 || status == 429) { + category = "Undetermined"; + sb.append(""); + sb.append("").append(urlWithBreak).append(""); sb.append(""); - //info end + sb.append(""); + sb.append(category); + sb.append(""); + } else { + category = "Broken"; + sb.append(""); + sb.append("").append(urlWithBreak).append(""); + sb.append(""); + sb.append(""); + sb.append(category); + sb.append(""); + } - sb.append(""); + //button + sb.append(""); + sb.append(""); + sb.append(""); + + sb.append(""); + //some html css table too wide work around + String record = checkedLink.getRecord(); + if (record != null) { + record = record.replace("_", "_"); + sb.append(record); } - } finally { - cursor.close(); - } + sb.append(""); + + //info + sb.append(""); + String message = checkedLink.getMessage().replace("_", "_"); + sb.append("Message: ").append(message).append("
"); + //because this field is new, older entries dont have it and it results in null, so a null check to make it more user friendly + String expectedContent = checkedLink.getExpectedMimeType() == null ? "Not Specified" : checkedLink.getExpectedMimeType(); + String content = checkedLink.getContentType(); + + sb.append("Expected Content Type: ").append(expectedContent).append("
"); + sb.append("Content Type: ").append(content).append("
"); + sb.append("Byte Size: ").append(checkedLink.getByteSize()).append("
"); + sb.append("Request Duration(ms): ").append(checkedLink.getDuration()).append("
"); + + String method = checkedLink.getMethod() == null ? "N/A" : checkedLink.getMethod(); + sb.append("Method: ").append(method).append("
"); + sb.append("Timestamp: ").append(TimeUtils.humanizeToDate(checkedLink.getTimestamp())); + sb.append(""); + //info end + + sb.append(""); + + + }); return sb.toString(); } diff --git a/curation-module-web/src/main/java/eu/clarin/main/Configuration.java b/curation-module-web/src/main/java/eu/clarin/main/Configuration.java index f7c4b906..7b6511ed 100644 --- a/curation-module-web/src/main/java/eu/clarin/main/Configuration.java +++ b/curation-module-web/src/main/java/eu/clarin/main/Configuration.java @@ -1,8 +1,8 @@ package eu.clarin.main; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; +import eu.clarin.cmdi.rasa.linkResources.CheckedLinkResource; +import eu.clarin.cmdi.rasa.linkResources.LinkToBeCheckedResource; +import eu.clarin.cmdi.rasa.linkResources.StatisticsResource; import eu.clarin.helpers.FileManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +28,12 @@ public class Configuration { public static String VIEW_RESOURCES_PATH; public static String OUTPUT_DIRECTORY; public static String BASE_URL; - private static String DATABASE_URI; - private static String DATABASE_NAME; +// private static String DATABASE_URI; +// private static String DATABASE_NAME; - public static MongoDatabase DATABASE; + public static CheckedLinkResource checkedLinkResource; + public static LinkToBeCheckedResource linkToBeCheckedResource; + public static StatisticsResource statisticsResource; public static void init(ServletContext servletContext) throws IOException { @@ -43,6 +45,10 @@ public static void init(ServletContext servletContext) throws IOException { //this is necessary for core module methods. eu.clarin.cmdi.curation.main.Configuration.init(path); + checkedLinkResource = eu.clarin.cmdi.curation.main.Configuration.checkedLinkResource; + linkToBeCheckedResource = eu.clarin.cmdi.curation.main.Configuration.linkToBeCheckedResource; + statisticsResource = eu.clarin.cmdi.curation.main.Configuration.statisticsResource; + _logger.info("Initializing configuration from: " + path); Properties properties = new Properties(); properties.load(new FileInputStream(path)); @@ -90,25 +96,8 @@ private static void loadVariables(Properties properties) { BASE_URL = properties.getProperty("BASE_URL"); - DATABASE_URI = properties.getProperty("DATABASE_URI"); - DATABASE_NAME = properties.getProperty("DATABASE_NAME"); - - loadMongo(); - - } - - private static void loadMongo() { - _logger.info("Connecting to database..."); - MongoClient mongoClient; - - if (DATABASE_URI == null || DATABASE_URI.isEmpty()) {//if it is empty, try localhost - mongoClient = MongoClients.create(); - } else { - mongoClient = MongoClients.create(DATABASE_URI); - } - - DATABASE = mongoClient.getDatabase(DATABASE_NAME); - _logger.info("Connected to database."); +// DATABASE_URI = properties.getProperty("DATABASE_URI"); +// DATABASE_NAME = properties.getProperty("DATABASE_NAME"); } } diff --git a/curation-module-web/src/main/java/eu/clarin/routes/Statistics.java b/curation-module-web/src/main/java/eu/clarin/routes/Statistics.java index 13344cd4..53a0ccb3 100644 --- a/curation-module-web/src/main/java/eu/clarin/routes/Statistics.java +++ b/curation-module-web/src/main/java/eu/clarin/routes/Statistics.java @@ -36,8 +36,8 @@ public Response getStatistics() { @Path("/{collectionName}/{status}") public Response getStatusStatsInit(@PathParam("collectionName") String collectionName, @PathParam("status") int status) { - LinkCheckerStatisticsHelper linkCheckerStatisticsHelper = new LinkCheckerStatisticsHelper(Configuration.DATABASE); - String urlStatistics = linkCheckerStatisticsHelper.createURLTable(collectionName, status); +// LinkCheckerStatisticsHelper linkCheckerStatisticsHelper = new LinkCheckerStatisticsHelper(Configuration.DATABASE); + String urlStatistics = LinkCheckerStatisticsHelper.createURLTable(collectionName, status); return ResponseManager.returnHTML(200, urlStatistics, null); @@ -48,9 +48,9 @@ public Response getStatusStatsInit(@PathParam("collectionName") String collectio @Path("/{collectionName}/{status}/{batchCount}") public Response getStatusStats(@PathParam("collectionName") String collectionName, @PathParam("status") int status, @PathParam("batchCount") int batchCount) { - LinkCheckerStatisticsHelper linkCheckerStatisticsHelper = new LinkCheckerStatisticsHelper(Configuration.DATABASE); +// LinkCheckerStatisticsHelper linkCheckerStatisticsHelper = new LinkCheckerStatisticsHelper(Configuration.DATABASE); - String urlBatchStatistics = linkCheckerStatisticsHelper.getHtmlRowsInBatch(collectionName, status, batchCount); + String urlBatchStatistics = LinkCheckerStatisticsHelper.getHtmlRowsInBatch(collectionName, status, batchCount); return ResponseManager.returnResponse(200, urlBatchStatistics, null); } diff --git a/curation-module-web/src/main/resources/view/html/generic.html b/curation-module-web/src/main/resources/view/html/generic.html index 5e46742d..8a49beac 100644 --- a/curation-module-web/src/main/resources/view/html/generic.html +++ b/curation-module-web/src/main/resources/view/html/generic.html @@ -126,8 +126,6 @@

Curation Module ${version}