Merge pull request #101 from clarin-eric/5.1.2

5.1.2

CHANGES.md

@@ -0,0 +1,10 @@
+# version 5.2.0
+- calculate total linkchecker statistics from category statistics and overall from provider group statistics to prevent database access at different times while linkchecker changes data continuously 
+
+# version 5.1.1
+- upgrading [RASA](https://github.com/clarin-eric/resource-availability-status-api) dependency to version 4.0.1
+
+# version 5.1.0
+- upgrading [RASA](https://github.com/clarin-eric/resource-availability-status-api) dependency to version 4.0.0
+- various code changes because of [changes in RASA API](https://github.com/clarin-eric/resource-availability-status-api/blob/master/CHANGES.md)
+- reviewing log levels

README.md

@@ -1,6 +1,6 @@
 # Clarin Curation Module
 
-Here is the current deployed instance of Curation Module: https://curate.acdh.oeaw.ac.at/
+Here is the current deployed instance of Curation Module: https://curation.clarin.eu/
 
 The goal of this project is to implement software component for curation and quality assessment which can be integrated in the CLARINs VLO workflow. Project is initialized by Metadata Curation Task Force. Specification for the Curation Module is based on the Metadata Quality Assessement Service proposal. Curation Module validates and normalizes single MD records, repositories and profiles, to assess their quality and to produce reports with different information for different actors in VLO workflow. For implementation this project will use some of the existing CLARIN components. 
 

curation-module-core/pom.xml

@@ -4,11 +4,11 @@
 	<parent>
 		<artifactId>curation-module</artifactId>
 		<groupId>eu.clarin.cmdi</groupId>
-		<version>5.1.1</version>
+		<version>5.1.2</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>curation-module-core</artifactId>
-	<version>5.1.1</version>
+	<version>5.1.2</version>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<slf4j.version>1.7.25</slf4j.version>

curation-module-core/src/main/java/eu/clarin/cmdi/curation/main/Main.java

@@ -176,7 +176,6 @@ public static void main(String[] args) throws Exception {
 
                     }
                 }
-                linkCheckerReport.createOverall();
 
                 LOG.info("Creating collections table...");
 

curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CMDProfileReport.java

@@ -12,13 +12,12 @@
 import javax.xml.bind.annotation.XmlRootElement;
 
 import eu.clarin.cmdi.curation.cr.ProfileHeader;
+import eu.clarin.cmdi.curation.utils.TimeUtils;
 import eu.clarin.cmdi.curation.xml.XMLMarshaller;
 
 
 /**
  * A selection of values from a single CMDProfileReport which will form a line in a statistical overview
- *
- * @author Wolfgang Walter SAUER (wowasa) <[email protected]>
  */
 @XmlRootElement(name = "profile-report")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -30,8 +29,8 @@ public class CMDProfileReport implements Report<CMDProfileReport> {
     @XmlAttribute(name = "max-score")
     public double maxScore;
 
-    @XmlAttribute
-    public Long timeStamp = System.currentTimeMillis();
+    @XmlAttribute(name = "creation-time")
+    public String creationTime = TimeUtils.humanizeToDate(System.currentTimeMillis());
 
     @XmlElement(name = "header-section")
     public ProfileHeader header;

curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionReport.java

@@ -6,7 +6,6 @@
 import eu.clarin.cmdi.curation.xml.XMLMarshaller;
 import eu.clarin.cmdi.rasa.DAO.Statistics.CategoryStatistics;
 import eu.clarin.cmdi.rasa.filters.CheckedLinkFilter;
-import eu.clarin.cmdi.rasa.helpers.statusCodeMapper.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,8 +52,8 @@ public class CollectionReport implements Report<CollectionReport> {
     @XmlAttribute(name = "ins-max-score")
     public Double maxPossibleScoreInstance = 0.0;
 
-    @XmlAttribute(name = "timestamp")
-    public String timeStamp = TimeUtils.humanizeToDate(System.currentTimeMillis());
+    @XmlAttribute(name = "creation-time")
+    public String creationTime = TimeUtils.humanizeToDate(System.currentTimeMillis());
 
     @XmlElement(name = "file-section")
     public FileReport fileReport;
@@ -417,6 +416,15 @@ public static class Statistics {
 
         @XmlAttribute
         public String colorCode;
+
+        public Statistics() {
+
+        }
+
+        public Statistics(String category, String colorCode) {
+           this.category = category;
+           this.colorCode = colorCode;
+        }
     }
 
     @XmlRootElement

curation-module-core/src/main/java/eu/clarin/cmdi/curation/report/CollectionsReport.java

@@ -1,5 +1,6 @@
 package eu.clarin.cmdi.curation.report;
 
+import eu.clarin.cmdi.curation.utils.TimeUtils;
 import eu.clarin.cmdi.curation.xml.XMLMarshaller;
 
 import java.io.OutputStream;
@@ -18,132 +19,130 @@
 public class CollectionsReport implements Report<CollectionsReport> {
 
 //    private static final Logger logger = LoggerFactory.getLogger(CollectionsReport.class);
-
-    @XmlElement(name = "collection")
-    private List<Collection> collections = new ArrayList<Collection>();
-
-    @Override
-    public void setParentName(String parentName) {
-
-    }
-
-    @Override
-    public String getParentName() {
-
-        return null;
-    }
-
-    @Override
-    public String getName() {
-
-        return "CollectionsReport";
-    }
-
-    @Override
-    public boolean isValid() {
-
-        return false;
-    }
-
-    @Override
-    public void addSegmentScore(Score segmentScore) {
-
-
-    }
-
-    @Override
-    public void toXML(OutputStream os) {
-        XMLMarshaller<CollectionsReport> instanceMarshaller = new
-                XMLMarshaller<>(CollectionsReport.class);
-        instanceMarshaller.marshal(this, os);
-    }
-
-    @Override
-    public void mergeWithParent(CollectionsReport parentReport) {
-
-
-    }
-
-    public void addReport(Report<?> report) {
-        if (report instanceof CollectionReport) {
-            this.collections.add(new Collection((CollectionReport) report));
-        }
-    }
-
-
-    @XmlRootElement
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class Collection {
-        @XmlAttribute
-        private String name;
-        @XmlElement
-        private String reportName;
-        @XmlElement
-        private double scorePercentage;
-        @XmlElement
-        private long numOfFiles;
-        @XmlElement
-        private int numOfProfiles;
-        @XmlElement
-        private int numOfUniqueLinks;
-        @XmlElement
-        private int numOfCheckedLinks;
-        @XmlElement
-        private double ratioOfValidLinks;
-        @XmlElement
-        private double avgNumOfResProxies;
-        @XmlElement
-        private int numOfResProxies;
-        @XmlElement
-        private double ratioOfValidRecords;
-        @XmlElement
-        private double avgNumOfEmptyXMLElements;
-        @XmlElement
-        private double avgFacetCoverage;
-
-        @XmlElementWrapper(name = "facets")
-        @XmlElement(name = "facet")
-        private List<Facet> facets = new ArrayList<Facet>();
-
-        public Collection() {
-
-        }
-
-        public Collection(CollectionReport report) {
-            this.name = report.getName();
-            this.reportName = report.getName();
-            this.scorePercentage = report.scorePercentage;
-            this.numOfFiles = report.fileReport.numOfFiles;
-            this.numOfProfiles = report.headerReport.profiles.totNumOfProfiles;
-            this.numOfUniqueLinks = report.urlReport.totNumOfUniqueLinks;
-            this.numOfCheckedLinks = report.urlReport.totNumOfCheckedLinks;
-            this.ratioOfValidLinks = report.urlReport.ratioOfValidLinks;
-            this.avgNumOfResProxies = report.resProxyReport.avgNumOfResProxies;
-            this.numOfResProxies = report.resProxyReport.totNumOfResProxies;
-            this.ratioOfValidRecords = report.xmlValidationReport.ratioOfValidRecords;
-            this.avgNumOfEmptyXMLElements = report.xmlPopulatedReport.avgXMLEmptyElement;
-            this.avgFacetCoverage = report.facetReport.coverage;
-
-            report.facetReport.facet.forEach(f -> this.facets.add(new Facet(f.name, f.coverage)));
-        }
-
-    }
-
-    @XmlRootElement
-    @XmlAccessorType(XmlAccessType.FIELD)
-    public static class Facet {
-        @XmlAttribute
-        private String name;
-        @XmlElement
-        private double avgCoverage;
-
-        public Facet() {
-
-        }
-
-        public Facet(String name, double avgCoverage) {
-            this.name = name;
-            this.avgCoverage = avgCoverage;
-        }
-    }
+   @XmlAttribute(name = "creation-time")
+   public String creationTime = TimeUtils.humanizeToDate(System.currentTimeMillis());
+
+   @XmlElement(name = "collection")
+   private List<Collection> collections = new ArrayList<Collection>();
+
+   @Override
+   public void setParentName(String parentName) {
+
+   }
+
+   @Override
+   public String getParentName() {
+
+      return null;
+   }
+
+   @Override
+   public String getName() {
+
+      return "CollectionsReport";
+   }
+
+   @Override
+   public boolean isValid() {
+
+      return false;
+   }
+
+   @Override
+   public void addSegmentScore(Score segmentScore) {
+
+   }
+
+   @Override
+   public void toXML(OutputStream os) {
+      XMLMarshaller<CollectionsReport> instanceMarshaller = new XMLMarshaller<>(CollectionsReport.class);
+      instanceMarshaller.marshal(this, os);
+   }
+
+   @Override
+   public void mergeWithParent(CollectionsReport parentReport) {
+
+   }
+
+   public void addReport(Report<?> report) {
+      if (report instanceof CollectionReport) {
+         this.collections.add(new Collection((CollectionReport) report));
+      }
+   }
+
+   @XmlRootElement
+   @XmlAccessorType(XmlAccessType.FIELD)
+   public static class Collection {
+      @XmlAttribute
+      private String name;
+      @XmlElement
+      private String reportName;
+      @XmlElement
+      private double scorePercentage;
+      @XmlElement
+      private long numOfFiles;
+      @XmlElement
+      private int numOfProfiles;
+      @XmlElement
+      private int numOfUniqueLinks;
+      @XmlElement
+      private int numOfCheckedLinks;
+      @XmlElement
+      private double ratioOfValidLinks;
+      @XmlElement
+      private double avgNumOfResProxies;
+      @XmlElement
+      private int numOfResProxies;
+      @XmlElement
+      private double ratioOfValidRecords;
+      @XmlElement
+      private double avgNumOfEmptyXMLElements;
+      @XmlElement
+      private double avgFacetCoverage;
+
+      @XmlElementWrapper(name = "facets")
+      @XmlElement(name = "facet")
+      private List<Facet> facets = new ArrayList<Facet>();
+
+      public Collection() {
+
+      }
+
+      public Collection(CollectionReport report) {
+         this.name = report.getName();
+         this.reportName = report.getName();
+         this.scorePercentage = report.scorePercentage;
+         this.numOfFiles = report.fileReport.numOfFiles;
+         this.numOfProfiles = report.headerReport.profiles.totNumOfProfiles;
+         this.numOfUniqueLinks = report.urlReport.totNumOfUniqueLinks;
+         this.numOfCheckedLinks = report.urlReport.totNumOfCheckedLinks;
+         this.ratioOfValidLinks = report.urlReport.ratioOfValidLinks;
+         this.avgNumOfResProxies = report.resProxyReport.avgNumOfResProxies;
+         this.numOfResProxies = report.resProxyReport.totNumOfResProxies;
+         this.ratioOfValidRecords = report.xmlValidationReport.ratioOfValidRecords;
+         this.avgNumOfEmptyXMLElements = report.xmlPopulatedReport.avgXMLEmptyElement;
+         this.avgFacetCoverage = report.facetReport.coverage;
+
+         report.facetReport.facet.forEach(f -> this.facets.add(new Facet(f.name, f.coverage)));
+      }
+
+   }
+
+   @XmlRootElement
+   @XmlAccessorType(XmlAccessType.FIELD)
+   public static class Facet {
+      @XmlAttribute
+      private String name;
+      @XmlElement
+      private double avgCoverage;
+
+      public Facet() {
+
+      }
+
+      public Facet(String name, double avgCoverage) {
+         this.name = name;
+         this.avgCoverage = avgCoverage;
+      }
+   }
 }