Skip to content

Commit

Permalink
Merge pull request #527 from digital-preservation/DR2-2049_makeNumber…
Browse files Browse the repository at this point in the history
…OfDisplayedErrorsConfigurable

Dr2 2049 make number of displayed errors configurable
techncl authored Jan 23, 2025
2 parents b0a131f + 5f889e6 commit 4b6a2cd
Showing 1 changed file with 44 additions and 19 deletions.
Original file line number Diff line number Diff line change
@@ -136,7 +136,10 @@ object CsvValidatorUi extends SimpleSwingApplication {
}
}

private def validate(csvFilePath: String, csvEncoding: Charset, csvSchemaFilePath: String, csvSchemaEncoding: Charset, failOnFirstError: Boolean, pathSubstitutions: List[(String, String)], enforceCaseSensitivePathChecks: Boolean, progress: Option[ProgressCallback], validateEncoding: Boolean, skipFileChecks: Boolean, outputTextSuffix: String)(output: String => Unit) : Unit = {
private def validate(csvFilePath: String, csvEncoding: Charset, csvSchemaFilePath: String, csvSchemaEncoding: Charset,
failOnFirstError: Boolean, pathSubstitutions: List[(String, String)], enforceCaseSensitivePathChecks: Boolean,
progress: Option[ProgressCallback], validateEncoding: Boolean, potentialMaxNumOfLines: String,
skipFileChecks: Boolean, outputTextSuffix: String)(output: String => Unit) : Unit = {

def toConsole(msg: String): Unit = Swing.onEDT {
output(msg)
@@ -145,24 +148,32 @@ object CsvValidatorUi extends SimpleSwingApplication {
var badLines = 0
var truncated = false

def rowCallback(row: ValidatedNel[FailMessage, Any]): Unit = row match {
val maxNumOfLines = Try(potentialMaxNumOfLines.toInt) match {
case Success(number) if number > 0 => number
case _ =>
toConsole("Error: Maximum number of errors to display should be more than 0")
0
}
val safeToRunValidation = csvFilePath.nonEmpty && csvSchemaFilePath.nonEmpty && maxNumOfLines > 0

def logRowCallback(maxBadLines: Int)(row: ValidatedNel[FailMessage, Any]): Unit = row match {
case Invalid(failures) =>
if (badLines > 2000) {
if (badLines >= maxBadLines) {
if (!truncated) {
toConsole("Too many errors/warnings, truncating")
toConsole(
s"...\n\nNote: Number of errors to display has reached the set limit of $maxBadLines; " +
"increase this limit and re-run in order to display more errors."
)
truncated = true
}
} else {
toConsole(CsvValidatorCmdApp.prettyPrint(failures))
}
} else toConsole(CsvValidatorCmdApp.prettyPrint(failures))

badLines += failures.size

case _ =>
}

if(csvFilePath.nonEmpty && csvSchemaFilePath.nonEmpty) {
if(safeToRunValidation) {
val (status, cliExitCode) = CsvValidatorCmdApp.validate(
TextFile(Paths.get(csvFilePath), csvEncoding, validateEncoding),
TextFile(Paths.get(csvSchemaFilePath), csvSchemaEncoding),
@@ -172,7 +183,7 @@ object CsvValidatorUi extends SimpleSwingApplication {
trace = false,
progress,
skipFileChecks,
rowCallback
logRowCallback(maxNumOfLines)
)

cliExitCode match {
@@ -344,7 +355,7 @@ object CsvValidatorUi extends SimpleSwingApplication {
Swing.onEDT {
progressBar.max = total
progressBar.value = processed
progressBar.label = s"Line ${processed} of ${total}"
progressBar.label = s"Row $processed of $total"
}
}
}
@@ -381,6 +392,7 @@ object CsvValidatorUi extends SimpleSwingApplication {
settingsPanel.enforceCaseSensitivePathChecks,
Some(progress),
settingsPanel.validateUtf8,
settingsPanel.numOfLinesToDisplay,
skipFileChecks,
suffix
)
@@ -465,15 +477,17 @@ object CsvValidatorUi extends SimpleSwingApplication {
}

val settingsGroup = new SJXTaskPane("Settings", true)
private val lblCsvEncoding = new Label("CSV Encoding")
private val lblCsvEncoding = new Label("CSV Encoding:")
private val cmbCsvEncoding = new ComboBox(CHARACTER_ENCODINGS)
private val lblCsvSchemaEncoding = new Label("CSV Schema Encoding")
private val lblCsvSchemaEncoding = new Label("CSV Schema Encoding:")
private val cmbCsvSchemaEncoding = new ComboBox(CHARACTER_ENCODINGS)
private val cbFailOnFirstError = new CheckBox("Fail on first error")
cbFailOnFirstError.tooltip = "Indicates whether to fail on the first error, or whether to collect all errors!"
private val cbValidateUtf8 = new CheckBox("Validate CSV for valid UTF-8 characters")
cbValidateUtf8.selected = true
private val lblPathSubstitutions = new Label("Path Substitutions")
private val tfDisplayLinesLabel = new Label("Maximum number of errors to display:")
private val tfDisplayLines = new TextField("2000", 5)
private val lblPathSubstitutions = new Label("Path Substitutions:")
private val cbEnforceCaseSensitivePathChecks = new CheckBox("Enforce case-sensitive file path checks")
cbEnforceCaseSensitivePathChecks.tooltip = "Performs additional checks to ensure that the case of file-paths in the CSV file match those of the filesystem"

@@ -561,33 +575,43 @@ object CsvValidatorUi extends SimpleSwingApplication {

c.gridx = 0
c.gridy = 2
layout(cbFailOnFirstError) = c
c.insets = new Insets(0, 0, 0, 0)
layout(tfDisplayLinesLabel) = c

c.gridx = 1
c.gridy = 2
c.insets = new Insets(0, 0, 0, 0)
layout(tfDisplayLines) = c

c.gridx = 0
c.gridy = 3
layout(cbEnforceCaseSensitivePathChecks) = c
layout(cbFailOnFirstError) = c

c.gridx = 0
c.gridy = 4
layout(cbValidateUtf8) = c
layout(cbEnforceCaseSensitivePathChecks) = c

c.gridx = 0
c.gridy = 5
layout(cbValidateUtf8) = c

c.gridx = 0
c.gridy = 6
c.insets = new Insets(0,10,0,0)
layout(lblPathSubstitutions) = c

c.gridx = 0
c.gridy = 6
c.gridy = 7
c.gridwidth = 2
layout(spTblPathSubstitutions) = c

c.gridx = 0
c.gridy = 7
c.gridy = 8
c.anchor = Anchor.LineStart
layout(btnRemovePathSubstitution) = c

c.gridx = 1
c.gridy = 7
c.gridy = 8
c.anchor = Anchor.LastLineEnd
layout(btnAddPathSubstitution) = c
}
@@ -601,5 +625,6 @@ object CsvValidatorUi extends SimpleSwingApplication {
def pathSubstitutions: List[(String, String)] = tblPathSubstitutions.pathSubstitutions
def enforceCaseSensitivePathChecks: Boolean = cbEnforceCaseSensitivePathChecks.selected
def validateUtf8 : Boolean = cbValidateUtf8.selected
def numOfLinesToDisplay: String = tfDisplayLines.text.strip()
}
}

0 comments on commit 4b6a2cd

Please sign in to comment.