remove duplicate occurrences from short version of dump (fix #65)

gnames · Jan 31, 2023 · 4e5894b · 4e5894b
1 parent b51b9fb
commit 4e5894b
Show file tree

Hide file tree

Showing 7 changed files with 44 additions and 0 deletions.
diff --git a/internal/bhlindex.go b/internal/bhlindex.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"sync"
 
@@ -257,9 +258,12 @@ func processOutput[O output.Output](
 	ch <-chan []O,
 ) error {
 	var o O
+	var shortOccur bool
 	var err error
 	var w *os.File
 	var count int
+	var pageID string
+	pageName := make(map[string]bool)
 	for rows := range ch {
 		select {
 		case <-ctx.Done():
@@ -268,6 +272,10 @@ func processOutput[O output.Output](
 			for i := range rows {
 				o = rows[i]
 				if count == 0 {
+					t := reflect.TypeOf(o).String()
+					if t == "output.OutputOccurrenceShort" {
+						shortOccur = true
+					}
 					path := filepath.Join(bi.OutputDir, o.Name()+bi.extension())
 					w, err = os.Create(path)
 					if err != nil {
@@ -285,6 +293,18 @@ func processOutput[O output.Output](
 					fmt.Fprintf(os.Stderr, "\r%s\r", strings.Repeat(" ", 80))
 					log.Info().Msgf("Processed %d %s", count, o.Name())
 				}
+				if shortOccur {
+					pg, nm := o.PageNameIDs()
+					if pageID != pg {
+						pageName = make(map[string]bool)
+						pageID = pg
+					}
+					key := pg + "|" + nm
+					if pageName[key] {
+						continue
+					}
+					pageName[key] = true
+				}
 				_, err = w.WriteString(output.Format(rows[i], bi.OutputFormat) + "\n")
 				if err != nil {
 					return err

diff --git a/internal/ent/output/interface.go b/internal/ent/output/interface.go
@@ -32,4 +32,8 @@ type Output interface {
 
 	// jsonOutput provides a method to generate a JSON output.
 	jsonOutput(bool) string
+
+	// PageNameIDs returns PageID and Name, if available, empty int and string
+	// otherwize
+	PageNameIDs() (string, string)
 }
diff --git a/internal/ent/output/name-format.go b/internal/ent/output/name-format.go
@@ -53,3 +53,7 @@ func (on OutputName) jsonOutput(pretty bool) string {
 	res, _ := enc.Encode(on)
 	return string(res)
 }
+
+func (on OutputName) PageNameIDs() (string, string) {
+	return "", ""
+}
diff --git a/internal/ent/output/name-short-format.go b/internal/ent/output/name-short-format.go
@@ -71,3 +71,7 @@ func (on OutputNameShort) jsonOutput(pretty bool) string {
 	res, _ := enc.Encode(out)
 	return string(res)
 }
+
+func (on OutputNameShort) PageNameIDs() (string, string) {
+	return "", ""
+}
diff --git a/internal/ent/output/occur-format.go b/internal/ent/output/occur-format.go
@@ -41,3 +41,7 @@ func (o OutputOccurrence) jsonOutput(pretty bool) string {
 	res, _ := enc.Encode(o)
 	return string(res)
 }
+
+func (o OutputOccurrence) PageNameIDs() (string, string) {
+	return strconv.Itoa(o.PageID), o.NameID
+}
diff --git a/internal/ent/output/occur-short-format.go b/internal/ent/output/occur-short-format.go
@@ -37,3 +37,7 @@ func (o OutputOccurrenceShort) jsonOutput(pretty bool) string {
 	res, _ := enc.Encode(oShort)
 	return string(res)
 }
+
+func (o OutputOccurrenceShort) PageNameIDs() (string, string) {
+	return strconv.Itoa(o.PageID), o.NameID
+}
diff --git a/internal/ent/output/odds-verif-format.go b/internal/ent/output/odds-verif-format.go
@@ -27,3 +27,7 @@ func (o OutputOddsVerification) jsonOutput(pretty bool) string {
 	res, _ := enc.Encode(o)
 	return string(res)
 }
+
+func (o OutputOddsVerification) PageNameIDs() (string, string) {
+	return "", ""
+}