This method is used for the generic properties normalization
- *- * A date is considered complete if: - *
See more in the specification of EDTF
*The date can contain a label, but can also be null
*/ -public abstract class AbstractEdtfDate implements Serializable { +public abstract class AbstractEdtfDate { - private static final long serialVersionUID = -4111050222535744456L; private final String label; - public AbstractEdtfDate() { + protected AbstractEdtfDate() { this.label = null; } - public AbstractEdtfDate(String label) { + protected AbstractEdtfDate(String label) { this.label = label; } @@ -24,19 +21,9 @@ public String getLabel() { return label; } - public abstract void setApproximate(boolean approx); - - public abstract boolean isApproximate(); - - public abstract void setUncertain(boolean uncertain); - - public abstract boolean isUncertain(); - - public abstract boolean isUnspecified(); - - public abstract boolean isTimeOnly(); + public abstract DateQualification getDateQualification(); - public abstract void switchDayAndMonth(); + public abstract boolean isOpen(); public abstract InstantEdtfDate getFirstDay(); diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/DateBoundaryType.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/DateBoundaryType.java new file mode 100644 index 0000000000..81a574eb2c --- /dev/null +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/DateBoundaryType.java @@ -0,0 +1,38 @@ +package eu.europeana.normalization.dates.edtf; + + +/** + * Enum that indicates what type of date is in an interval. + *+ * A date in an interval can be: + *
1900/..').1900/').It will return a matched enum value or {@link #NO_QUALIFICATION}.
+ * + * @param character the provided character + * @return the enum value + */ + public static DateQualification fromCharacter(String character) { + return Arrays.stream(DateQualification.values()).filter(value -> value.character.equals(character)).findFirst().orElse( + NO_QUALIFICATION); + } + + public String getCharacter() { + return character; + } +} diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/EdtfDatePart.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/EdtfDatePart.java deleted file mode 100644 index ca87aedfcc..0000000000 --- a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/EdtfDatePart.java +++ /dev/null @@ -1,183 +0,0 @@ -package eu.europeana.normalization.dates.edtf; - -import eu.europeana.normalization.dates.YearPrecision; -import java.io.Serializable; -import java.text.DecimalFormat; - -/** - * Class representing the date part an EDTF date. - *- * Support partial dates, including only centuries or decades (e.g., 19XX) - *
- */ -public class EdtfDatePart implements Serializable { - - private static final long serialVersionUID = -7497880706682687923L; - public static final int THRESHOLD_4_DIGITS_YEAR = 9999; - - private boolean uncertain; - private boolean approximate; - - /** - * Indicates whether the date is unknown (e.g. if the input EDTF-compliant date interval string was equal to - * '1900/?').
- */
- private boolean unknown;
-
- /**
- * Indicates whether the date is unspecified (e.g. if the input EDTF-compliant date interval string was equal to
- * '1900/').
- */
- private boolean unspecified;
-
- private Integer year;
- private Integer month;
- private Integer day;
-
- private YearPrecision yearPrecision;
-
- public boolean isUncertain() {
- return uncertain;
- }
-
- public void setUncertain(boolean uncertain) {
- this.uncertain = uncertain;
- }
-
- public boolean isApproximate() {
- return approximate;
- }
-
- public void setApproximate(boolean approximate) {
- this.approximate = approximate;
- }
-
- public boolean isUnknown() {
- return unknown;
- }
-
- public void setUnknown(boolean unknown) {
- this.unknown = unknown;
- }
-
- public Integer getYear() {
- return year;
- }
-
- public void setYear(Integer year) {
- this.year = year;
- }
-
- public Integer getMonth() {
- return month;
- }
-
- public void setMonth(Integer month) {
- this.month = month == null || month == 0 ? null : month;
- }
-
- public Integer getDay() {
- return day;
- }
-
- public void setDay(Integer day) {
- this.day = day == null || day == 0 ? null : day;
- }
-
- public boolean isUnspecified() {
- return unspecified;
- }
-
- public void setUnspecified(boolean unspecified) {
- this.unspecified = unspecified;
- }
-
- public YearPrecision getYearPrecision() {
- return yearPrecision;
- }
-
- public void setYearPrecision(YearPrecision yearPrecision) {
- this.yearPrecision = yearPrecision;
- }
-
- /**
- * Switches the values of the day and month.
- */
- public void switchDayAndMonth() {
- if (day != null) {
- int tempDay = day;
- setDay(month);
- setMonth(tempDay);
- }
- }
-
- public static EdtfDatePart getUnknownInstance() {
- final EdtfDatePart edtfDatePart = new EdtfDatePart();
- edtfDatePart.setUnknown(true);
- return edtfDatePart;
- }
-
- public static EdtfDatePart getUnspecifiedInstance() {
- final EdtfDatePart edtfDatePart = new EdtfDatePart();
- edtfDatePart.setUnspecified(true);
- return edtfDatePart;
- }
-
- @Override
- public String toString() {
- final StringBuilder stringBuilder = new StringBuilder();
- if (unknown || unspecified) {
- stringBuilder.append("..");
- } else if (year < -THRESHOLD_4_DIGITS_YEAR || year > THRESHOLD_4_DIGITS_YEAR) {
- stringBuilder.append("Y").append(year);
- } else {
- stringBuilder.append(serializeYear());
-
- //Append Month and day
- final DecimalFormat decimalFormat = new DecimalFormat("00");
- if (month != null && month > 0) {
- stringBuilder.append("-").append(decimalFormat.format(month));
- if (day != null && day > 0) {
- stringBuilder.append("-").append(decimalFormat.format(day));
- }
- }
- //Append approximate/uncertain
- if (approximate && uncertain) {
- stringBuilder.append("%");
- } else if (approximate) {
- stringBuilder.append("~");
- } else if (uncertain) {
- stringBuilder.append("?");
- }
- }
- return stringBuilder.toString();
- }
-
- /**
- * Adjusts a year with padding and optional precision that replace right most digits with 'X's.
- * - * There are two possibilities: - *
It validates the following: - *
It has been decided that only the date part should be checked, ignoring the time part. This - * could mean that the interval is technically not valid (e.g. start and end are on the same date but the start is later than - * the end). But since we are only interested in dates, we accept this.
- * - * @param intervalEdtfDate the interval date to check - * @return true if it's valid - */ - private static boolean validateInterval(IntervalEdtfDate intervalEdtfDate) { - final InstantEdtfDate startDate = intervalEdtfDate.getStart(); - final InstantEdtfDate endDate = intervalEdtfDate.getEnd(); - final boolean isIntervalValid; - if (startDate != null && validateInstantOfInterval(startDate) && endDate != null && validateInstantOfInterval(endDate)) { - EdtfDatePart startDatePart = startDate.getEdtfDatePart(); - EdtfDatePart endDatePart = endDate.getEdtfDatePart(); - final boolean isStartDatePartSpecific = !startDatePart.isUnknown() && !startDatePart.isUnspecified(); - final boolean isEndDatePartSpecific = !endDatePart.isUnknown() && !endDatePart.isUnspecified(); - if (isStartDatePartSpecific && isEndDatePartSpecific) { - if (startDatePart.getYearPrecision() == null && endDatePart.getYearPrecision() == null) { - isIntervalValid = validateSpecificIntervalDates(startDatePart, endDatePart); - } else { - //Validate year using precision instead - final Integer adjustedStartYear = adjustYearWithPrecision(startDatePart.getYear(), startDatePart.getYearPrecision()); - final Integer adjustedEndYear = adjustYearWithPrecision(endDatePart.getYear(), endDatePart.getYearPrecision()); - isIntervalValid = adjustedStartYear <= adjustedEndYear; - } - } else { - isIntervalValid = isStartDatePartSpecific || isEndDatePartSpecific; - } - } else { - isIntervalValid = false; - } - - return isIntervalValid; - } - - private static boolean validateSpecificIntervalDates(EdtfDatePart startDatePart, EdtfDatePart endDatePart) { - // TODO: 20/07/2022 Should we be using the java.time classes Year, YearMonth, LocalDate etc? (to be handled with MET-4726) - //Sanity check: years should not be null at this stage, but we check to be sure - if (startDatePart.getYear() == null || endDatePart.getYear() == null) { - throw new IllegalArgumentException("Year cannot be null for start or end dates"); - } - boolean isDatesValid = false; - if (startDatePart.getYear().equals(endDatePart.getYear())) { - if (startDatePart.getMonth() == null || endDatePart.getMonth() == null - || startDatePart.getMonth() < endDatePart.getMonth()) { - isDatesValid = true; - } else if (startDatePart.getMonth().equals(endDatePart.getMonth())) { - isDatesValid = - startDatePart.getDay() == null || endDatePart.getDay() == null || startDatePart.getDay() <= endDatePart.getDay(); - } - } else { - isDatesValid = startDatePart.getYear() < endDatePart.getYear(); - } - return isDatesValid; - } - - /** - * Adjusts the year value based on the {@link YearPrecision} supplied. - *The adjustment is not a rounding operation but a discarding operation of the right most digits
- *- * Examples of discarding operations for {@link YearPrecision#CENTURY}: - *
It contains general validity of date part and in addition it cannot have date part null or - * unknown.
- * - * @param instantEdtfDate the instant date to validate - * @return true if the instant is valid - */ - private static boolean validateInstant(InstantEdtfDate instantEdtfDate) { - return validateInstantOfInterval(instantEdtfDate) - && instantEdtfDate.getEdtfDatePart() != null - && !instantEdtfDate.getEdtfDatePart().isUnknown(); - } - - private static boolean validateInstantOfInterval(InstantEdtfDate instantEdtfDate) { - return validateDatePart(instantEdtfDate.getEdtfDatePart()); - } - - private static boolean validateDatePart(EdtfDatePart edtfDatePart) { - boolean isDatePartValid = true; - if (edtfDatePart != null && !(edtfDatePart.isUnknown() || edtfDatePart.isUnspecified())) { - if (edtfDatePart.getYear() == null) { - isDatePartValid = false; - } - if (edtfDatePart.getYearPrecision() == null && edtfDatePart.getMonth() != null) { - if (edtfDatePart.getMonth() < 1 || edtfDatePart.getMonth() > 12) { - isDatePartValid = false; - } else { - isDatePartValid = isDatePartDayValid(edtfDatePart); - } - } - } - return isDatePartValid; - } - - private static boolean isDatePartDayValid(EdtfDatePart edtfDatePart) { - final boolean isDayValid; - if (edtfDatePart.getDay() == null) { - isDayValid = true; - } else { - final boolean isValidDayRange = edtfDatePart.getDay() > 0 && edtfDatePart.getDay() <= 31; - final boolean isNot31Or31AndValid = - edtfDatePart.getDay() != 31 || MONTHS_WITH_31_DAYS.contains(Month.of(edtfDatePart.getMonth())); - final boolean isNotFebruaryOrFebruaryAndValidDay = edtfDatePart.getMonth() != 2 || isValidFebruaryDay(edtfDatePart); - isDayValid = isValidDayRange && isNot31Or31AndValid && isNotFebruaryOrFebruaryAndValidDay; - } - return isDayValid; - } - - private static boolean isValidFebruaryDay(EdtfDatePart edtfDatePart) { - return (edtfDatePart.getDay() > 0 && edtfDatePart.getDay() < 30 && edtfDatePart.getDay() != 29) || Year.isLeap( - edtfDatePart.getYear()); - } - - private static boolean validateIntervalNotInFuture(IntervalEdtfDate intervalEdtfDate) { - return validateInstantNotInFuture(intervalEdtfDate.getStart()) && validateInstantNotInFuture(intervalEdtfDate.getEnd()); - } - - - // TODO: 20/07/2022 This only calculates years and not other parts of the date. - // (this probably won't capture a dates that is days/months in the future but on the current year?) - // Fix this to also check the other parts of the date as well. - // Perhaps the already existent validation of interval dates should be reused instead, with the end date the current date. - private static boolean validateInstantNotInFuture(InstantEdtfDate instantEdtfDate) { - final boolean isYearInPast; - //If null or not specific it's valid - if (instantEdtfDate.getEdtfDatePart() == null || instantEdtfDate.getEdtfDatePart().isUnknown() - || instantEdtfDate.getEdtfDatePart().isUnspecified()) { - isYearInPast = true; - } else { - int currentYear = Year.now().getValue(); - final Integer edtfYear = instantEdtfDate.getEdtfDatePart().getYear(); - final YearPrecision yearPrecision = instantEdtfDate.getEdtfDatePart().getYearPrecision(); - - final Integer adjustedCurrentYear = adjustYearWithPrecision(currentYear, yearPrecision); - final Integer adjustedEdtfYear = adjustYearWithPrecision(edtfYear, yearPrecision); - isYearInPast = adjustedEdtfYear <= adjustedCurrentYear; - } - return isYearInPast; - } - -} diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/InstantEdtfDate.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/InstantEdtfDate.java index 52e0c1807e..bb8fb4bc95 100644 --- a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/InstantEdtfDate.java +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/InstantEdtfDate.java @@ -1,168 +1,291 @@ package eu.europeana.normalization.dates.edtf; +import static eu.europeana.normalization.dates.edtf.DateBoundaryType.DECLARED; +import static eu.europeana.normalization.dates.edtf.DateBoundaryType.OPEN; +import static eu.europeana.normalization.dates.edtf.DateBoundaryType.UNKNOWN; +import static eu.europeana.normalization.dates.edtf.DateQualification.NO_QUALIFICATION; +import static eu.europeana.normalization.dates.edtf.InstantEdtfDateBuilder.THRESHOLD_4_DIGITS_YEAR; +import static eu.europeana.normalization.dates.edtf.Iso8601Parser.ISO_8601_MINIMUM_YEAR_DIGITS; +import static java.lang.Math.abs; +import static java.util.Optional.ofNullable; + import eu.europeana.normalization.dates.YearPrecision; +import eu.europeana.normalization.dates.extraction.DateExtractionException; +import java.lang.invoke.MethodHandles; +import java.text.DecimalFormat; +import java.time.LocalDate; import java.time.Month; +import java.time.MonthDay; import java.time.Year; -import org.apache.commons.lang3.SerializationUtils; +import java.time.YearMonth; +import java.time.temporal.TemporalAccessor; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Part of an EDTF date that represents a point in time with various degrees of precision + * Class representing the date part of an EDTF date. + *+ * Support partial dates, including only centuries or decades (e.g., 19XX). The uncertain and approximate qualifiers, '?' and '~', + * when applied together, are combined into a single qualifier character '%'; + *
*/ -public class InstantEdtfDate extends AbstractEdtfDate { +public final class InstantEdtfDate extends AbstractEdtfDate implements Comparable+ *
During {@link #build()} it will verify all the parameters that have been requested. + * The {@link #build()}, if {@link #withFlexibleDateBuild(boolean)} was called with {@code true}, will also attempt a second time + * by switching month and day values if the original values were invalid. Furthermore, there are a set of constructors that can + * start the builder and will perform a build with specific characteristics: + *
This object during build will overwrite the date parts, if any <@code>.with methods were called, from the + * {@link TemporalAccessor} + *
+ * + * @param temporalAccessor the temporal accessor + */ + public InstantEdtfDateBuilder(TemporalAccessor temporalAccessor) { + this.temporalAccessor = temporalAccessor; + } + + /** + * Returns an instance of {@link InstantEdtfDate} created and validated from the fields set on this builder. + * + * @return the new instant edtf date + * @throws DateExtractionException if something went wrong during date validation + */ + public InstantEdtfDate build() throws DateExtractionException { + InstantEdtfDate instantEdtfDate; + instantEdtfDate = buildInternal(); + //Try once more if flexible date + if (instantEdtfDate == null && flexibleDateBuild) { + swapMonthDay(); + instantEdtfDate = buildInternal(); + } + + //Still nothing, we are done. + if (instantEdtfDate == null) { + throw new DateExtractionException("Could not instantiate date"); + } + return instantEdtfDate; + } + + private InstantEdtfDate buildInternal() { + InstantEdtfDate instantEdtfDate = null; + //Setup defaults + yearPrecision = ofNullable(yearPrecision).orElse(YearPrecision.YEAR); + dateQualification = ofNullable(dateQualification).orElse(NO_QUALIFICATION); + + try { + if (temporalAccessor != null) { + parseTemporalAccessor(); + } + parseYear(); + parseMonthDay(); + validateDateNotInFuture(); + validateStrict(); + instantEdtfDate = new InstantEdtfDate(this); + } catch (DateTimeException | DateExtractionException e) { + LOGGER.debug("Date build failed.", e); + } + return instantEdtfDate; + } + + private void parseTemporalAccessor() { + LOGGER.debug("TemporalAccessor present. Overwriting values."); + day = temporalAccessor.isSupported(ChronoField.DAY_OF_MONTH) ? + temporalAccessor.get(ChronoField.DAY_OF_MONTH) : null; + month = temporalAccessor.isSupported(ChronoField.MONTH_OF_YEAR) ? + temporalAccessor.get(ChronoField.MONTH_OF_YEAR) : null; + year = temporalAccessor.isSupported(ChronoField.YEAR) ? + temporalAccessor.get(ChronoField.YEAR) : null; + } + + private void parseYear() throws DateExtractionException { + Objects.requireNonNull(year, "Year value can never be null"); + if (longDatePrefixedWithY && Math.abs(year) <= THRESHOLD_4_DIGITS_YEAR) { + throw new DateExtractionException( + format("Prefixed year with 'Y' is enabled indicating that year should have absolute value greater than %s", + THRESHOLD_4_DIGITS_YEAR)); + } else if (!longDatePrefixedWithY && Math.abs(year) > THRESHOLD_4_DIGITS_YEAR) { + throw new DateExtractionException( + format("Year absolute value greater than %s, should be prefixed with 'Y'", THRESHOLD_4_DIGITS_YEAR)); + } + yearObj = Year.of(year * yearPrecision.getDuration()); + } + + private void parseMonthDay() throws DateExtractionException { + try { + if (month != null && month >= 1) { + monthObj = Month.of(month); + if (day != null && day >= 1) { + yearMonthDayObj = LocalDate.of(yearObj.getValue(), monthObj.getValue(), day); + } + } + } catch (DateTimeException e) { + throw new DateExtractionException("Failed to instantiate month and day", e); + } + } + + private void validateDateNotInFuture() throws DateExtractionException { + try { + final boolean isYearMonthDayInTheFuture = yearMonthDayObj != null && yearMonthDayObj.isAfter(LocalDate.now()); + final boolean isYearMonthInTheFuture = monthObj != null && YearMonth.of(yearObj.getValue(), month).isAfter(YearMonth.now()); + final boolean isYearInTheFuture = yearObj != null && yearObj.isAfter(Year.now()); + + if (isYearMonthDayInTheFuture || isYearMonthInTheFuture || isYearInTheFuture) { + throw new DateExtractionException("Date cannot be in the future"); + } + + } catch (DateTimeException e) { + throw new DateExtractionException("Failed to instantiate month and day", e); + } + } + + private void validateStrict() throws DateExtractionException { + //If it is not a long year, and we want to be strict we further validate + boolean notLongYearAndStrictBuild = !longDatePrefixedWithY && !flexibleDateBuild; + // TODO: 15/02/2023 Check this instruction. It used to be like that + // return edtfDatePart.isUnknown() || edtfDatePart.isUncertain() || edtfDatePart.getYearPrecision() != null; + // but do we actually need the check on unknown? + boolean isDateNonPrecise = dateQualification == DateQualification.UNCERTAIN || yearPrecision != null; + boolean notCompleteDate = monthObj == null || yearMonthDayObj == null; + if (notLongYearAndStrictBuild && (isDateNonPrecise || notCompleteDate)) { + throw new DateExtractionException("Date is invalid according to our strict profile!"); + } + } + + private void swapMonthDay() { + Integer tempMonth = month; + month = day; + day = tempMonth; + } + + /** + * Add month value. + * + * @param month the month value + * @return the extended builder + */ + public InstantEdtfDateBuilder withMonth(int month) { + this.month = month; + return this; + } + + /** + * Add day value. + * + * @param day the day value + * @return the extended builder + */ + public InstantEdtfDateBuilder withDay(int day) { + this.day = day; + return this; + } + + /** + * Add year precision. + * + * @param yearPrecision the year precision + * @return the extended builder + */ + public InstantEdtfDateBuilder withYearPrecision(YearPrecision yearPrecision) { + this.yearPrecision = yearPrecision; + return this; + } + + /** + * Add date qualification. + * + * @param dateQualification the date qualification + * @return the extended builder + */ + public InstantEdtfDateBuilder withDateQualification(DateQualification dateQualification) { + this.dateQualification = dateQualification; + return this; + } + + /** + * Opt in/out for flexible date building. + * + * @param flexibleDateBuild the boolean (dis|en)abling the flexibility + * @return the extended builder + */ + public InstantEdtfDateBuilder withFlexibleDateBuild(boolean flexibleDateBuild) { + this.flexibleDateBuild = flexibleDateBuild; + return this; + } + + /** + * Declare the date is of long year format, prefixed with 'Y'. + * + * @return the extended builder + */ + public InstantEdtfDateBuilder withLongYearPrefixedWithY() { + this.longDatePrefixedWithY = true; + return this; + } + + public Year getYearObj() { + return yearObj; + } + + public Month getMonthObj() { + return monthObj; + } + + public LocalDate getYearMonthDayObj() { + return yearMonthDayObj; + } + + public YearPrecision getYearPrecision() { + return yearPrecision; + } + + public DateQualification getDateQualification() { + return dateQualification; + } +} diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDate.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDate.java index 0fa47eb42a..28100b047d 100644 --- a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDate.java +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDate.java @@ -1,90 +1,45 @@ package eu.europeana.normalization.dates.edtf; +import static java.lang.String.format; + /** * An EDTF date that represents a period of time specified by a start and end date with various degrees of precision */ public class IntervalEdtfDate extends AbstractEdtfDate { - private static final long serialVersionUID = -8754610674192759880L; - + public static final String DATE_INTERVAL_SEPARATOR = "/"; private InstantEdtfDate start; private InstantEdtfDate end; - public IntervalEdtfDate(InstantEdtfDate start, InstantEdtfDate end) { - this.start = start; - this.end = end; - } - - public IntervalEdtfDate(String label, InstantEdtfDate start, InstantEdtfDate end) { - super(label); - this.start = start; - this.end = end; - } - - public void switchStartWithEnd() { - InstantEdtfDate tempStart = this.start; - this.start = this.end; - this.end = tempStart; - } - - @Override - public boolean isTimeOnly() { - return (start == null || start.isTimeOnly()) && (end == null || end.isTimeOnly()); - } - - @Override - public void setApproximate(boolean approx) { - if (start != null && start.getEdtfDatePart() != null) { - start.getEdtfDatePart().setApproximate(approx); - } - if (end != null && end.getEdtfDatePart() != null) { - end.getEdtfDatePart().setApproximate(approx); - } + IntervalEdtfDate(IntervalEdtfDateBuilder intervalEdtfDateBuilder) { + super(intervalEdtfDateBuilder.getLabel()); + this.start = intervalEdtfDateBuilder.getStart(); + this.end = intervalEdtfDateBuilder.getEnd(); } @Override - public void setUncertain(boolean uncertain) { - if (start != null && start.getEdtfDatePart() != null) { - start.getEdtfDatePart().setUncertain(uncertain); - } - if (end != null && end.getEdtfDatePart() != null) { - end.getEdtfDatePart().setUncertain(uncertain); + public DateQualification getDateQualification() { + // TODO: 24/02/2023 To verify what this should return. + if (start.getDateQualification() == DateQualification.NO_QUALIFICATION) { + return end.getDateQualification(); + } else { + return start.getDateQualification(); } } @Override - public boolean isApproximate() { - return (start != null && start.isApproximate()) || (end != null && end.isApproximate()); - } - - @Override - public boolean isUncertain() { - return (start != null && start.isUncertain()) || (end != null && end.isUncertain()); - } - - @Override - public boolean isUnspecified() { - return (start != null && start.isUnspecified()) || (end != null && end.isUnspecified()); - } - - @Override - public void switchDayAndMonth() { - if (start != null) { - start.switchDayAndMonth(); - } - if (end != null) { - end.switchDayAndMonth(); - } + public boolean isOpen() { + return start.getDateBoundaryType() == DateBoundaryType.OPEN || end.getDateBoundaryType() == DateBoundaryType.OPEN; } @Override public InstantEdtfDate getFirstDay() { - return start == null ? null : start.getFirstDay(); + return start.getFirstDay(); } @Override public InstantEdtfDate getLastDay() { - return end == null ? null : end.getLastDay(); + return end.getLastDay(); } public InstantEdtfDate getStart() { @@ -105,6 +60,6 @@ public void setEnd(InstantEdtfDate end) { @Override public String toString() { - return String.format("%s/%s", start.toString(), end.toString()); + return format("%s%s%s", start.toString(), DATE_INTERVAL_SEPARATOR, end.toString()); } } diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDateBuilder.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDateBuilder.java new file mode 100644 index 0000000000..428a673eda --- /dev/null +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/edtf/IntervalEdtfDateBuilder.java @@ -0,0 +1,137 @@ +package eu.europeana.normalization.dates.edtf; + +import eu.europeana.normalization.dates.extraction.DateExtractionException; +import java.lang.invoke.MethodHandles; +import java.time.DateTimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Builder class for {@link IntervalEdtfDate}. + *+ * During {@link #build()} it will verify all the parameters that have been requested. The {@link #build()}, if + * {@link #withFlexibleDateBuild(boolean)} was called with {@code true}, will also attempt a second time by switching start and + * end values if the original values were invalid. + *
+ */ +public class IntervalEdtfDateBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private InstantEdtfDate start; + private InstantEdtfDate end; + private String label; + + private boolean flexibleDateBuild = false; + + /** + * Constructor which initializes the builder with the start and end date boundaries. + *Boundaries should never be null
+ * + * @param start the start date + * @param end the end date + */ + public IntervalEdtfDateBuilder(InstantEdtfDate start, InstantEdtfDate end) { + this.start = start; + this.end = end; + } + + /** + * Add label if any for the interval + * + * @param label the interval specific label + * @return the extended builder + */ + public IntervalEdtfDateBuilder withLabel(String label) { + this.label = label; + return this; + } + + /** + * Opt in/out for flexible date building. + * + * @param flexibleDateBuild the boolean (dis|en)abling the flexibility + * @return the extended builder + */ + public IntervalEdtfDateBuilder withFlexibleDateBuild(boolean flexibleDateBuild) { + this.flexibleDateBuild = flexibleDateBuild; + return this; + } + + /** + * Returns an instance of {@link IntervalEdtfDate} created and validated from the fields set on this builder. + * + * @return the new interval edtf date + * @throws DateExtractionException if something went wrong during date validation + */ + public IntervalEdtfDate build() throws DateExtractionException { + IntervalEdtfDate intervalEdtfDate; + intervalEdtfDate = buildInternal(); + //Try once more if switching allowed + if (intervalEdtfDate == null && flexibleDateBuild) { + //Retry with swapping month and day + switchStartWithEnd(); + intervalEdtfDate = buildInternal(); + } + + //Still nothing, we are done. + if (intervalEdtfDate == null) { + throw new DateExtractionException("Could not instantiate date"); + } + return intervalEdtfDate; + } + + private void switchStartWithEnd() { + InstantEdtfDate tempStart = this.start; + this.start = this.end; + this.end = tempStart; + } + + /** + * Returns an instance of {@link IntervalEdtfDate} created and validated from the fields set on this builder. + *It validates: + *
+ * Checks for the following formats in order: + *
* Each separator has an entry and each accepts its own date delimiters as well as the unspecified characters supported for - * edges. + * boundaries. *
*/ public enum NumericRangeDateDelimiters implements DateDelimiters { @@ -201,7 +201,7 @@ public enum NumericRangeDateDelimiters implements DateDelimiters { SPACED_DASH_RANGE(" - ", DASH_DOT_SLASH_DELIMITERS, NumericRangeDateDelimiters.DEFAULT_UNSPECIFIED_CHARACTERS), //"[XU]" with "-" delimiter, "[\\-XU]" with "./" delimiters PIPE_RANGE("\\|", DASH_DOT_SLASH_DELIMITERS, NumericRangeDateDelimiters.DEFAULT_UNSPECIFIED_CHARACTERS), - //For space separator we don't accept unspecified edges + //For space separator we don't accept unspecified boundaries //Does not exist in XX SPACE_RANGE(" ", DASH_DOT_SLASH_DELIMITERS, null), //"[XU]" diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/AbstractDateExtractor.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/AbstractDateExtractor.java new file mode 100644 index 0000000000..0538114121 --- /dev/null +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/AbstractDateExtractor.java @@ -0,0 +1,82 @@ +package eu.europeana.normalization.dates.extraction.dateextractors; + +import static eu.europeana.normalization.dates.DateNormalizationResult.getNoMatchResult; +import static java.lang.String.format; + +import eu.europeana.normalization.dates.DateNormalizationResult; +import eu.europeana.normalization.dates.edtf.DateQualification; +import eu.europeana.normalization.dates.extraction.DateExtractionException; +import java.lang.invoke.MethodHandles; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract class implementing interface {@link DateExtractor} with default functionality for all extractors + */ +public abstract class AbstractDateExtractor implements DateExtractor { + + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Utility method for calling {@link #extract(String, DateQualification, boolean)} with allowSwitchesDuringValidation as true. + *It also captures relevant exceptions so that return is performed
+ * + * @param inputValue the input value + * @param dateQualification the date qualification requested + * @return the date normalization result + */ + @Override + public DateNormalizationResult extractDateProperty(String inputValue, DateQualification dateQualification) { + return getDateNormalizationResult(inputValue, dateQualification, true); + } + + /** + * Utility method for calling {@link #extract(String, DateQualification, boolean)} with allowSwitchesDuringValidation as false. + *It also captures relevant exceptions so that return is performed
+ * + * @param inputValue the input value + * @param dateQualification the date qualification requested + * @return the date normalization result + */ + @Override + public DateNormalizationResult extractGenericProperty(String inputValue, DateQualification dateQualification) { + return getDateNormalizationResult(inputValue, dateQualification, false); + } + + private DateNormalizationResult getDateNormalizationResult(String inputValue, DateQualification dateQualification, + boolean flexibleDateBuild) { + DateNormalizationResult dateNormalizationResult = getNoMatchResult(inputValue); + try { + dateNormalizationResult = extract(inputValue, dateQualification, flexibleDateBuild); + } catch (DateExtractionException e) { + LOGGER.debug(format("Date extraction failed %s: ", inputValue), e); + } + //Sanity check to avoid null return. + if (dateNormalizationResult == null) { + dateNormalizationResult = getNoMatchResult(inputValue); + } + return dateNormalizationResult; + } + + /** + * Default method to get the correct date qualification. + *If a requested date qualification is requested we then set that, overwriting any other that would otherwise be computed. + * The date qualification will be overwritten if the requested date qualification in non-null and + * non-{@link DateQualification#NO_QUALIFICATION}. Otherwise we compute it with the supplier provided.
+ * + * @param requestedDateQualification the requested date qualification + * @param dateQualificationSupplier the date qualification supplier + * @return the computed date qualification + */ + DateQualification computeDateQualification(DateQualification requestedDateQualification, + Supplier+ * The end year in this extractor has to: + *
+ * This pattern needs to be executed before the Edtf extractor because EDTF could potentially match yyyy/MM and yyyy-MM. + * Therefore in this extractor we check only the values that are higher than 12 to avoid a mismatch. + *
+ */ +public class BriefRangeDateExtractor extends AbstractDateExtractor { + + private final Pattern briefRangePattern = Pattern.compile("\\??(\\d{3,4})[\\-/](\\d{2})\\??"); + + public DateNormalizationResult extract(String inputValue, DateQualification requestedDateQualification, + boolean flexibleDateBuild) throws DateExtractionException { + final String sanitizedValue = DateFieldSanitizer.cleanSpacesAndTrim(inputValue); + final DateQualification dateQualification = computeDateQualification(requestedDateQualification, () -> + (sanitizedValue.startsWith("?") || sanitizedValue.endsWith("?")) ? UNCERTAIN : NO_QUALIFICATION); + + DateNormalizationResult dateNormalizationResult = DateNormalizationResult.getNoMatchResult(inputValue); + final Matcher matcher = briefRangePattern.matcher(sanitizedValue); + if (matcher.matches()) { + final int startYearFourDigits = Integer.parseInt(matcher.group(1)); + final int startYearLastTwoDigits = startYearFourDigits % CENTURY.getDuration(); + final int endYearTwoDigits = Integer.parseInt(matcher.group(2)); + final int endYearFourDigits = (startYearFourDigits / CENTURY.getDuration()) * CENTURY.getDuration() + endYearTwoDigits; + + if (endYearTwoDigits > Month.DECEMBER.getValue() && startYearLastTwoDigits < endYearTwoDigits) { + final InstantEdtfDate startDate = new InstantEdtfDateBuilder(startYearFourDigits) + .withDateQualification(dateQualification) + .withFlexibleDateBuild(flexibleDateBuild) + .build(); + + final InstantEdtfDate endDate = new InstantEdtfDateBuilder(endYearFourDigits) + .withDateQualification(dateQualification) + .withFlexibleDateBuild(flexibleDateBuild) + .build(); + + dateNormalizationResult = new DateNormalizationResult(DateNormalizationExtractorMatchId.BRIEF_DATE_RANGE, inputValue, + new IntervalEdtfDateBuilder(startDate, endDate).withFlexibleDateBuild(flexibleDateBuild).build()); + } + } + + return dateNormalizationResult; + } +} + diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/CenturyDateExtractor.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/CenturyDateExtractor.java index 19f9f0a6bd..f8c8c46bda 100644 --- a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/CenturyDateExtractor.java +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/CenturyDateExtractor.java @@ -1,22 +1,29 @@ package eu.europeana.normalization.dates.extraction.dateextractors; +import static eu.europeana.normalization.dates.YearPrecision.CENTURY; +import static eu.europeana.normalization.dates.edtf.DateQualification.NO_QUALIFICATION; +import static eu.europeana.normalization.dates.edtf.DateQualification.UNCERTAIN; import static java.util.regex.Pattern.CASE_INSENSITIVE; import static java.util.regex.Pattern.compile; import eu.europeana.normalization.dates.DateNormalizationExtractorMatchId; import eu.europeana.normalization.dates.DateNormalizationResult; -import eu.europeana.normalization.dates.YearPrecision; +import eu.europeana.normalization.dates.DateNormalizationResultStatus; import eu.europeana.normalization.dates.edtf.AbstractEdtfDate; -import eu.europeana.normalization.dates.edtf.EdtfDatePart; +import eu.europeana.normalization.dates.edtf.DateQualification; import eu.europeana.normalization.dates.edtf.InstantEdtfDate; -import eu.europeana.normalization.dates.edtf.IntervalEdtfDate; +import eu.europeana.normalization.dates.edtf.InstantEdtfDateBuilder; +import eu.europeana.normalization.dates.edtf.IntervalEdtfDateBuilder; +import eu.europeana.normalization.dates.extraction.DateExtractionException; import eu.europeana.normalization.dates.extraction.RomanToNumber; import eu.europeana.normalization.dates.sanitize.DateFieldSanitizer; +import java.lang.invoke.MethodHandles; import java.util.Arrays; -import java.util.Objects; import java.util.function.ToIntFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Extractor that matches a century with a decimal or Roman numerals @@ -34,8 +41,9 @@ *The Roman numerals may also be preceded by an abbreviation of century, for example ‘s. XIX’.
*Also supports ranges.
*/ -public class CenturyDateExtractor implements DateExtractor { +public class CenturyDateExtractor extends AbstractDateExtractor { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String NUMERIC_10_TO_21_ENDING_DOTS_REGEX = "(1\\d|2[0-1])\\.{2}"; private static final String NUMERIC_1_TO_21_SUFFIXED_REGEX = "(2?1st|2nd|3rd|(?:1\\d|[4-9]|20)th)\\scentury"; private static final String ROMAN_1_TO_21_REGEX = "(X?(?:IX|IV|VI{0,3}|I{1,3})|X|XXI?)"; @@ -45,28 +53,28 @@ public class CenturyDateExtractor implements DateExtractor { enum PatternCenturyDateOperation { PATTERN_YYYY( compile(QUESTION_MARK + NUMERIC_10_TO_21_ENDING_DOTS_REGEX + QUESTION_MARK, CASE_INSENSITIVE), - century -> Integer.parseInt(century) * 100, DateNormalizationExtractorMatchId.CENTURY_NUMERIC), + Integer::parseInt, DateNormalizationExtractorMatchId.CENTURY_NUMERIC), PATTERN_ENGLISH( compile(QUESTION_MARK + NUMERIC_1_TO_21_SUFFIXED_REGEX + QUESTION_MARK, CASE_INSENSITIVE), - century -> (Integer.parseInt(century.substring(0, century.length() - 2)) - 1) * 100, + century -> (Integer.parseInt(century.substring(0, century.length() - 2)) - 1), DateNormalizationExtractorMatchId.CENTURY_NUMERIC), PATTERN_ROMAN( compile(QUESTION_MARK + CENTURY_PREFIX + ROMAN_1_TO_21_REGEX + QUESTION_MARK, CASE_INSENSITIVE), - century -> (RomanToNumber.romanToDecimal(century) - 1) * 100, + century -> (RomanToNumber.romanToDecimal(century) - 1), DateNormalizationExtractorMatchId.CENTURY_ROMAN), PATTERN_ROMAN_RANGE( compile(QUESTION_MARK + CENTURY_PREFIX + ROMAN_1_TO_21_REGEX + "\\s?-\\s?" + ROMAN_1_TO_21_REGEX + QUESTION_MARK, - CASE_INSENSITIVE), century -> (RomanToNumber.romanToDecimal(century) - 1) * 100, + CASE_INSENSITIVE), century -> (RomanToNumber.romanToDecimal(century) - 1), DateNormalizationExtractorMatchId.CENTURY_RANGE_ROMAN); private final Pattern pattern; - private final ToIntFunctionThis parser supports partial Level0 and Level1 from the Extended + * Date/Time Format (EDTF) Specification. It only validates the date part of a date and the time if existent it is discarded. + * Specifically from Level1, seasons and Unspecified digit(s) from the right are not supported + *
+ */ +public class EdtfDateExtractor extends AbstractDateExtractor { + + private static final Iso8601Parser ISO_8601_PARSER = new Iso8601Parser(); + + @Override + public DateNormalizationResult extract(String inputValue, DateQualification requestedDateQualification, + boolean flexibleDateBuild) throws DateExtractionException { + if (StringUtils.isEmpty(inputValue)) { + throw new DateExtractionException("Empty argument"); + } + final AbstractEdtfDate edtfDate; + if (inputValue.contains(DATE_INTERVAL_SEPARATOR)) { + edtfDate = extractInterval(inputValue, requestedDateQualification, flexibleDateBuild); + } else { + edtfDate = extractInstant(inputValue, requestedDateQualification, flexibleDateBuild); + } + return new DateNormalizationResult(DateNormalizationExtractorMatchId.EDTF, inputValue, edtfDate); + } + + protected IntervalEdtfDate extractInterval(String dateInput, DateQualification requestedDateQualification, + boolean allowSwitchMonthDay) throws DateExtractionException { + String startPart = dateInput.substring(0, dateInput.indexOf(DATE_INTERVAL_SEPARATOR)); + String endPart = dateInput.substring(dateInput.indexOf(DATE_INTERVAL_SEPARATOR) + 1); + final InstantEdtfDate start = extractInstant(startPart, requestedDateQualification, allowSwitchMonthDay); + final InstantEdtfDate end = extractInstant(endPart, requestedDateQualification, allowSwitchMonthDay); + + //Are both ends unknown or open, then it is not a date + if ((end.getDateBoundaryType() == UNKNOWN || end.getDateBoundaryType() == OPEN) && + (start.getDateBoundaryType() == UNKNOWN || start.getDateBoundaryType() == OPEN)) { + throw new DateExtractionException(dateInput); + } + return new IntervalEdtfDateBuilder(start, end).withFlexibleDateBuild(allowSwitchMonthDay).build(); + } + + protected InstantEdtfDate extractInstant(String dateInput, DateQualification requestedDateQualification, + boolean allowSwitchMonthDay) throws DateExtractionException { + final InstantEdtfDate instantEdtfDate; + if (UNKNOWN.getDeserializedRepresentation().equals(dateInput)) { + instantEdtfDate = InstantEdtfDate.getUnknownInstance(); + } else if (OPEN.getDeserializedRepresentation().equals(dateInput)) { + instantEdtfDate = InstantEdtfDate.getOpenInstance(); + } else if (dateInput.startsWith(String.valueOf(OVER_4_DIGITS_YEAR_PREFIX))) { + int year = NumberUtils.toInt(dateInput.substring(1)); + instantEdtfDate = new InstantEdtfDateBuilder(year).withLongYearPrefixedWithY() + .withDateQualification(requestedDateQualification).build(); + } else { + instantEdtfDate = extractInstantEdtfDate(dateInput, requestedDateQualification, allowSwitchMonthDay); + } + return instantEdtfDate; + } + + private static InstantEdtfDate extractInstantEdtfDate(String dateInput, DateQualification requestedDateQualification, + boolean allowSwitchMonthDay) throws DateExtractionException { + Matcher matcher = CHECK_QUALIFICATION_PATTERN.matcher(dateInput); + String dateInputStrippedModifier = dateInput; + DateQualification dateQualification = requestedDateQualification; + + boolean containsQualification = matcher.matches(); + if (containsQualification && (requestedDateQualification == null || requestedDateQualification == NO_QUALIFICATION)) { + final String modifier = matcher.group(1); + if (StringUtils.isNotEmpty(modifier)) { + dateQualification = DateQualification.fromCharacter(String.valueOf(modifier.charAt(0))); + dateInputStrippedModifier = dateInput.substring(0, dateInput.length() - 1); + } + } + + final TemporalAccessor temporalAccessor = ISO_8601_PARSER.parseDatePart(dateInputStrippedModifier); + return new InstantEdtfDateBuilder(temporalAccessor) + .withDateQualification(dateQualification) + .withFlexibleDateBuild(allowSwitchMonthDay) + .build(); + } + +} diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/NumericPartsDateExtractor.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/NumericPartsDateExtractor.java index 1a30d08f93..92b972f8ed 100644 --- a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/NumericPartsDateExtractor.java +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/NumericPartsDateExtractor.java @@ -1,16 +1,18 @@ package eu.europeana.normalization.dates.extraction.dateextractors; +import static eu.europeana.normalization.dates.edtf.DateQualification.NO_QUALIFICATION; +import static eu.europeana.normalization.dates.edtf.DateQualification.UNCERTAIN; import static java.util.Optional.ofNullable; import static java.util.regex.Pattern.compile; import eu.europeana.normalization.dates.DateNormalizationResult; import eu.europeana.normalization.dates.YearPrecision; -import eu.europeana.normalization.dates.edtf.EdtfDatePart; -import eu.europeana.normalization.dates.edtf.InstantEdtfDate; +import eu.europeana.normalization.dates.edtf.DateQualification; +import eu.europeana.normalization.dates.edtf.InstantEdtfDateBuilder; +import eu.europeana.normalization.dates.extraction.DateExtractionException; import eu.europeana.normalization.dates.extraction.NumericPartsPattern; import eu.europeana.normalization.dates.sanitize.DateFieldSanitizer; import java.util.Locale; -import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -20,7 +22,7 @@ * Patterns for numeric dates with variations in the separators of date components. *For Patterns pay attentions on the use of {@link Matcher#matches()} or {@link Matcher#find()} in this method.
*/ -public class NumericPartsDateExtractor implements DateExtractor { +public class NumericPartsDateExtractor extends AbstractDateExtractor { /** * The start of the string can be one or three question marks but not two. @@ -37,8 +39,9 @@ public class NumericPartsDateExtractor implements DateExtractor { private static final String UNKNOWN_CHARACTERS_REGEX = "[XU?-]"; @Override - public DateNormalizationResult extract(String inputValue) { - return extract(inputValue, NumericPartsPattern.NUMERIC_SET); + public DateNormalizationResult extract(String inputValue, DateQualification requestedDateQualification, + boolean flexibleDateBuild) throws DateExtractionException { + return extract(inputValue, requestedDateQualification, NumericPartsPattern.NUMERIC_SET, flexibleDateBuild); } /** @@ -46,30 +49,34 @@ public DateNormalizationResult extract(String inputValue) { * * @param inputValue the input value * @param numericPatternValues the patterns to check against + * @param allowSwitchMonthDay allow switching month and day values if month and day original values are not valid * @return the date normalization result */ - public DateNormalizationResult extract(String inputValue, SetWe reuse the already existent {@link NumericPartsDateExtractor} code for the edges.
+ *We reuse the already existent {@link NumericPartsDateExtractor} code for the boundaries.
*/ -public class NumericPartsRangeDateExtractor implements DateExtractor { +public class NumericPartsRangeDateExtractor extends AbstractDateExtractor { private static final NumericPartsDateExtractor NUMERIC_WITH_MISSING_PARTS_DATE_EXTRACTOR = new NumericPartsDateExtractor(); /** * Extract the date normalization result for a range. *- * The date is split in two edges using the {@link NumericRangeDateDelimiters#values()} as a separator. The result will contain - * the first split that is exactly splitting the original value in two parts(edges) and those two edge are valid parsable edges - * or null if none found. + * The date is split in two boundaries using the {@link NumericRangeDateDelimiters#values()} as a separator. The result will + * contain the first split that is exactly splitting the original value in two parts(boundaries) and those two boundaries are + * valid parsable boundaries or null if none found. *
* * @param inputValue the range value to attempt parsing * @return the date normalization result */ - public DateNormalizationResult extract(String inputValue) { + @Override + public DateNormalizationResult extract(String inputValue, DateQualification requestedDateQualification, + boolean flexibleDateBuild) throws DateExtractionException { final String sanitizedValue = DateFieldSanitizer.cleanSpacesAndTrim(inputValue); - DateNormalizationResult startDate; - DateNormalizationResult endDate; - DateNormalizationResult rangeDate = null; + DateNormalizationResult startDateResult; + DateNormalizationResult endDateResult; + DateNormalizationResult rangeDate = DateNormalizationResult.getNoMatchResult(inputValue); for (NumericRangeDateDelimiters numericRangeSpecialCharacters : NumericRangeDateDelimiters.values()) { // Split with -1 limit does not discard empty splits final String[] sanitizedDateSplitArray = sanitizedValue.split(numericRangeSpecialCharacters.getDatesSeparator(), -1); @@ -45,16 +51,20 @@ public DateNormalizationResult extract(String inputValue) { // This also guarantees that the separator used is not used for unknown characters. if (sanitizedDateSplitArray.length == 2) { // Try extraction and verify - startDate = extractDateNormalizationResult(sanitizedDateSplitArray[0], numericRangeSpecialCharacters); - endDate = extractDateNormalizationResult(sanitizedDateSplitArray[1], numericRangeSpecialCharacters); - if (startDate != null && endDate != null && !areYearsAmbiguous((InstantEdtfDate) startDate.getEdtfDate(), - (InstantEdtfDate) endDate.getEdtfDate(), + startDateResult = extractDateNormalizationResult(sanitizedDateSplitArray[0], numericRangeSpecialCharacters, + requestedDateQualification, + flexibleDateBuild); + endDateResult = extractDateNormalizationResult(sanitizedDateSplitArray[1], numericRangeSpecialCharacters, + requestedDateQualification, flexibleDateBuild); + if (startDateResult.getDateNormalizationResultStatus() == DateNormalizationResultStatus.MATCHED + && endDateResult.getDateNormalizationResultStatus() == DateNormalizationResultStatus.MATCHED + && !areYearsAmbiguous((InstantEdtfDate) startDateResult.getEdtfDate(), (InstantEdtfDate) endDateResult.getEdtfDate(), numericRangeSpecialCharacters)) { final DateNormalizationExtractorMatchId dateNormalizationExtractorMatchId = - getDateNormalizationExtractorId(startDate, endDate); - final IntervalEdtfDate intervalEdtfDate = new IntervalEdtfDate((InstantEdtfDate) startDate.getEdtfDate(), - (InstantEdtfDate) endDate.getEdtfDate()); + getDateNormalizationExtractorId(startDateResult, endDateResult); + final IntervalEdtfDate intervalEdtfDate = new IntervalEdtfDateBuilder((InstantEdtfDate) startDateResult.getEdtfDate(), + (InstantEdtfDate) endDateResult.getEdtfDate()).withFlexibleDateBuild(flexibleDateBuild).build(); rangeDate = new DateNormalizationResult(dateNormalizationExtractorMatchId, inputValue, intervalEdtfDate); break; } @@ -75,9 +85,10 @@ private boolean areYearsAmbiguous(InstantEdtfDate startDate, InstantEdtfDate end NumericRangeDateDelimiters numericRangeSpecialCharacters) { boolean isAmbiguous = false; if (numericRangeSpecialCharacters == NumericRangeDateDelimiters.DASH_RANGE) { - final boolean isStartSpecified = !startDate.getEdtfDatePart().isUnspecified(); - final boolean isStartThreeDigit = isStartSpecified && startDate.getEdtfDatePart().getYear().toString().matches("\\d{3}"); - if (isStartThreeDigit && endDate.isUnspecified()) { + final boolean isStartDeclared = startDate.getDateBoundaryType() == DateBoundaryType.DECLARED; + final boolean isStartThreeDigit = + isStartDeclared && Integer.toString(startDate.getYear().getValue()).matches("\\d{3}"); + if (isStartThreeDigit && endDate.getDateBoundaryType() == DateBoundaryType.OPEN) { isAmbiguous = true; } } @@ -85,15 +96,15 @@ private boolean areYearsAmbiguous(InstantEdtfDate startDate, InstantEdtfDate end } private DateNormalizationResult extractDateNormalizationResult(String dateString, - NumericRangeDateDelimiters numericRangeSpecialCharacters) { + NumericRangeDateDelimiters numericRangeSpecialCharacters, DateQualification requestedDateQualification, + boolean allowSwitchMonthDay) throws DateExtractionException { final DateNormalizationResult dateNormalizationResult; if (numericRangeSpecialCharacters.getUnspecifiedCharacters() != null && dateString.matches( numericRangeSpecialCharacters.getUnspecifiedCharacters())) { - dateNormalizationResult = new DateNormalizationResult(NUMERIC_ALL_VARIANTS, dateString, - new InstantEdtfDate(EdtfDatePart.getUnspecifiedInstance())); + dateNormalizationResult = new DateNormalizationResult(NUMERIC_ALL_VARIANTS, dateString, InstantEdtfDate.getOpenInstance()); } else { - dateNormalizationResult = NUMERIC_WITH_MISSING_PARTS_DATE_EXTRACTOR.extract(dateString, - NumericPartsPattern.NUMERIC_RANGE_SET); + dateNormalizationResult = NUMERIC_WITH_MISSING_PARTS_DATE_EXTRACTOR.extract(dateString, requestedDateQualification, + NumericPartsPattern.NUMERIC_RANGE_SET, allowSwitchMonthDay); } return dateNormalizationResult; } diff --git a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/PatternBcAdDateExtractor.java b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/PatternBcAdDateExtractor.java index e08f823c3d..803d25e00a 100644 --- a/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/PatternBcAdDateExtractor.java +++ b/metis-normalization/src/main/java/eu/europeana/normalization/dates/extraction/dateextractors/PatternBcAdDateExtractor.java @@ -2,9 +2,11 @@ import eu.europeana.normalization.dates.DateNormalizationExtractorMatchId; import eu.europeana.normalization.dates.DateNormalizationResult; -import eu.europeana.normalization.dates.edtf.EdtfDatePart; +import eu.europeana.normalization.dates.edtf.DateQualification; import eu.europeana.normalization.dates.edtf.InstantEdtfDate; -import eu.europeana.normalization.dates.edtf.IntervalEdtfDate; +import eu.europeana.normalization.dates.edtf.InstantEdtfDateBuilder; +import eu.europeana.normalization.dates.edtf.IntervalEdtfDateBuilder; +import eu.europeana.normalization.dates.extraction.DateExtractionException; import java.util.HashSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,7 +17,7 @@ * ‘AC/DC’, but the abbreviations used in other languages will be supported in the future. Or a date range where the start/end * years contain an indication of the era. */ -public class PatternBcAdDateExtractor implements DateExtractor { +public class PatternBcAdDateExtractor extends AbstractDateExtractor { static final HashSet- * The end year in this extractor has to: - *
This pattern needs to be executed before the Edtf one.
- *Most values that match this pattern also match the EDTF pattern, but would result in an invalid date.
- *This pattern only matches values that would not be valid EDTF dates.
- */ -public class PatternBriefDateRangeDateExtractor implements DateExtractor { - - private final Pattern briefDateRangePattern = Pattern.compile( - "(?
* TODO The swagger console does not pick up the @ApiParam settings.
*
* @param datasetId - The id of the dataset which the record belongs to
@@ -116,14 +119,15 @@ public InsertionResult saveRecord(
+ "dataset ID. If a record ID already exists, the record is overwritten. Note that record "
+ "IDs are normalized to contain only the characters a-z, A-Z, 0-9 and `_`.")
@ApiResponses(value = {@ApiResponse(code = 404, message = "Illegal dataset or record ID"),
- @ApiResponse(code = 500, message = "Error processing the file archive")})
+ @ApiResponse(code = 500, message = "Error processing the file archive")})
public InsertionResult saveRecords(
- @ApiParam(value = "Dataset ID (new or existing)", required = true) @RequestParam("datasetId") String datasetId,
- @ApiParam(value = "Date stamp (in ISO format)") @RequestParam(name = "dateStamp", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant dateStamp,
- @ApiParam(value = "The (EDM/RDF) records", required = true) @RequestPart MultipartFile recordsZipFile) {
+ @ApiParam(value = "Dataset ID (new or existing)", required = true) @RequestParam("datasetId") String datasetId,
+ @ApiParam(value = "Date stamp (in ISO format)") @RequestParam(name = "dateStamp", required = false)
+ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant dateStamp,
+ @ApiParam(value = "The (EDM/RDF) records", required = true) @RequestPart MultipartFile recordsZipFile) {
verifyDatasetId(datasetId);
final InsertionResult result = new InsertionResult(datasetId,
- Objects.requireNonNullElseGet(dateStamp, Instant::now));
+ Objects.requireNonNullElseGet(dateStamp, Instant::now));
try (final InputStream inputStream = recordsZipFile.getInputStream()) {
new HttpHarvesterImpl().harvestRecords(inputStream, CompressedFileExtension.ZIP, entry -> {
final byte[] content = entry.getEntryContent().readAllBytes();
@@ -142,7 +146,7 @@ public InsertionResult saveRecords(
/**
* Update record header (metadata information) of the record given by the record ID.
*
- * @param recordId - A unique record id
+ * @param recordId - A unique record id
* @param datasetId - The id of the dataset which the record belongs to
* @param dateStamp - Last time the record was updated. It can also be the date of creation
* @return a summary of the performed actions.
@@ -157,14 +161,14 @@ public InsertionResult saveRecords(
public InsertionResult updateRecordHeader(
@ApiParam(value = "Record ID (existing)", required = true) @PathVariable("recordId") String recordId,
@ApiParam(value = "Dataset ID (new or existing)", required = true) @RequestParam("datasetId") String datasetId,
- @ApiParam(value = "Date stamp (in ISO format)") @RequestParam(name = "dateStamp", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant dateStamp,
+ @ApiParam(value = "Date stamp (in ISO format)") @RequestParam(name = "dateStamp", required = false)
+ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant dateStamp,
@ApiParam(value = "Whether the record is to be marked as deleted", required = true) @RequestParam("markAsDeleted") boolean markAsDeleted) {
- final Record record = recordDao.getRecord(recordId);
- if (record == null) {
- throw new ResponseStatusException(HttpStatus.NOT_FOUND,
- "No record found for this identifier.");
+ final Record oaiRecord = recordDao.getRecord(recordId);
+ if (oaiRecord == null) {
+ throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No record found for this identifier.");
}
- return saveRecord(recordId, datasetId, dateStamp, markAsDeleted, record.getEdmRecord());
+ return saveRecord(recordId, datasetId, dateStamp, markAsDeleted, oaiRecord.getEdmRecord());
}
private void saveRecord(String providedRecordId, String edmRecord, InsertionResult result,
@@ -195,13 +199,12 @@ private void saveRecord(String providedRecordId, String edmRecord, InsertionResu
@ApiResponse(code = 500, message = "Error processing the request")})
public RecordView getRecord(
@ApiParam(value = "Record ID", required = true) @PathVariable("recordId") String recordId) {
- final Record record = recordDao.getRecord(recordId);
- if (record == null) {
- throw new ResponseStatusException(HttpStatus.NOT_FOUND,
- "No record found for this identifier.");
+ final Record oaiRecord = recordDao.getRecord(recordId);
+ if (oaiRecord == null) {
+ throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No record found for this identifier.");
}
- return new RecordView(record.getRecordId(), record.getDatasetId(), record.getDateStamp(),
- record.isDeleted(), record.getEdmRecord());
+ return new RecordView(oaiRecord.getRecordId(), oaiRecord.getDatasetId(), oaiRecord.getDateStamp(),
+ oaiRecord.isDeleted(), oaiRecord.getEdmRecord());
}
@DeleteMapping(value = RestEndpoints.REPOSITORY_RECORDS_RECORD_ID)
@@ -214,8 +217,7 @@ public RecordView getRecord(
public void deleteRecord(
@ApiParam(value = "Record ID", required = true) @PathVariable("recordId") String recordId) {
if (!recordDao.deleteRecord(recordId)) {
- throw new ResponseStatusException(HttpStatus.NOT_FOUND,
- "No record found for this identifier.");
+ throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No record found for this identifier.");
}
}
@@ -230,6 +232,6 @@ private static String normalizeRecordId(String suggestedRecordId) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid record ID.");
}
return UNSUPPORTED_CHARACTERS_PATTERN.matcher(suggestedRecordId)
- .replaceAll(REPLACEMENT_CHARACTER);
+ .replaceAll(REPLACEMENT_CHARACTER);
}
}
diff --git a/metis-repository/src/main/java/eu/europeana/metis/repository/dao/Record.java b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/dao/Record.java
similarity index 97%
rename from metis-repository/src/main/java/eu/europeana/metis/repository/dao/Record.java
rename to metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/dao/Record.java
index 6aab0af349..0b1f1cf14f 100644
--- a/metis-repository/src/main/java/eu/europeana/metis/repository/dao/Record.java
+++ b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/dao/Record.java
@@ -1,4 +1,4 @@
-package eu.europeana.metis.repository.dao;
+package eu.europeana.metis.repository.rest.dao;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import dev.morphia.annotations.Entity;
diff --git a/metis-repository/src/main/java/eu/europeana/metis/repository/dao/RecordDao.java b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/dao/RecordDao.java
similarity index 98%
rename from metis-repository/src/main/java/eu/europeana/metis/repository/dao/RecordDao.java
rename to metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/dao/RecordDao.java
index 560ede8b81..572970600a 100644
--- a/metis-repository/src/main/java/eu/europeana/metis/repository/dao/RecordDao.java
+++ b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/dao/RecordDao.java
@@ -1,4 +1,4 @@
-package eu.europeana.metis.repository.dao;
+package eu.europeana.metis.repository.rest.dao;
import static eu.europeana.metis.utils.CommonStringValues.CRLF_PATTERN;
diff --git a/metis-repository/src/main/java/eu/europeana/metis/repository/rest/InsertionResult.java b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/InsertionResult.java
similarity index 94%
rename from metis-repository/src/main/java/eu/europeana/metis/repository/rest/InsertionResult.java
rename to metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/InsertionResult.java
index ac669505b2..bd840a3023 100644
--- a/metis-repository/src/main/java/eu/europeana/metis/repository/rest/InsertionResult.java
+++ b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/InsertionResult.java
@@ -1,4 +1,4 @@
-package eu.europeana.metis.repository.rest;
+package eu.europeana.metis.repository.rest.view;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.time.Instant;
@@ -6,7 +6,7 @@
import java.util.HashSet;
import java.util.Set;
-class InsertionResult {
+public class InsertionResult {
private final String datasetId;
@JsonSerialize(using = InstantSerializer.class)
diff --git a/metis-repository/src/main/java/eu/europeana/metis/repository/rest/InstantSerializer.java b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/InstantSerializer.java
similarity index 92%
rename from metis-repository/src/main/java/eu/europeana/metis/repository/rest/InstantSerializer.java
rename to metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/InstantSerializer.java
index 0aa0f178ac..4803920e03 100644
--- a/metis-repository/src/main/java/eu/europeana/metis/repository/rest/InstantSerializer.java
+++ b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/InstantSerializer.java
@@ -1,4 +1,4 @@
-package eu.europeana.metis.repository.rest;
+package eu.europeana.metis.repository.rest.view;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonSerializer;
diff --git a/metis-repository/src/main/java/eu/europeana/metis/repository/rest/RecordView.java b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/RecordView.java
similarity index 95%
rename from metis-repository/src/main/java/eu/europeana/metis/repository/rest/RecordView.java
rename to metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/RecordView.java
index be4665ac8d..e4fdaf3f4b 100644
--- a/metis-repository/src/main/java/eu/europeana/metis/repository/rest/RecordView.java
+++ b/metis-repository/metis-repository-rest/src/main/java/eu/europeana/metis/repository/rest/view/RecordView.java
@@ -1,4 +1,4 @@
-package eu.europeana.metis.repository.rest;
+package eu.europeana.metis.repository.rest.view;
import com.fasterxml.jackson.annotation.JsonRawValue;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
diff --git a/metis-repository/metis-repository-rest/src/main/resources/application.properties.example b/metis-repository/metis-repository-rest/src/main/resources/application.properties.example
new file mode 100644
index 0000000000..208481f3ca
--- /dev/null
+++ b/metis-repository/metis-repository-rest/src/main/resources/application.properties.example
@@ -0,0 +1,33 @@
+#Spring
+spring.servlet.multipart.max-file-size=5MB
+spring.servlet.multipart.max-request-size=5MB
+spring.autoconfigure.exclude=\
+ org.springframework.boot.autoconfigure.mongo.embedded.EmbeddedMongoAutoConfiguration, \
+ org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, \
+ org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration
+springdoc.packages-to-scan=eu.europeana.metis.repository.rest
+springdoc.paths-to-match=/**
+
+#Server
+server.error.whitelabel.enabled=false
+
+#Socks Proxy
+socks.proxy.enabled=false
+socks.proxy.host=
+socks.proxy.port=
+socks.proxy.username=
+socks.proxy.password=
+
+#Mongo
+mongo.hosts=
+mongo.port=
+mongo.authentication.db=
+mongo.username=
+mongo.password=
+mongo.enable.ssl=
+mongo.application.name=
+mongo.record.db=
+
+#Truststore
+truststore.path=
+truststore.password=
diff --git a/metis-repository/metis-repository-rest/src/main/resources/log4j2.xml b/metis-repository/metis-repository-rest/src/main/resources/log4j2.xml
new file mode 100644
index 0000000000..71275a030c
--- /dev/null
+++ b/metis-repository/metis-repository-rest/src/main/resources/log4j2.xml
@@ -0,0 +1,13 @@
+
+