dump relation odds/verification (close #62)

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+## [v1.0.0-RC4] - 2023-01-10 Tue
+
+- Add [#62]: dump odds vs verification stats.
 - Add [#61]: shortened and filtered dump for BHL-related data.
 - Add [#60]: normalize odds according to verification results.
 

cmd/dump.go

@@ -86,6 +86,14 @@ default format is CSV`,
 			err = fmt.Errorf("DumpOccurrences %w", err)
 			log.Fatal().Err(err).Msg("Dump of occurrences failed")
 		}
+
+		err = bhli.DumpOddsVerification(dmp)
+		if err != nil {
+			err = fmt.Errorf("DumpOccurrences %w", err)
+			log.Fatal().Err(err).Msg("Dump of occurrences failed")
+		}
+
+		log.Info().Msg("All files are created successfully")
 	},
 }
 

internal/bhlindex.go

@@ -197,6 +197,37 @@ func (bi *bhlindex) DumpOccurrences(dmp output.Dumper) error {
 	return nil
 }
 
+func (bi *bhlindex) DumpOddsVerification(dmp output.Dumper) error {
+	var w *os.File
+	outs, err := dmp.DumpOddsVerification()
+	if err != nil {
+		err = fmt.Errorf("-> DumpOddsVerification: %w", err)
+		return err
+	}
+	for i := range outs {
+		o := outs[i]
+		if i == 0 {
+			path := filepath.Join(bi.OutputDir, o.Name()+bi.extension())
+			w, err = os.Create(path)
+			if err != nil {
+				return err
+			}
+			if bi.OutputFormat != gnfmt.CompactJSON {
+				_, err = w.WriteString(output.CSVHeader(o, bi.OutputFormat) + "\n")
+				if err != nil {
+					return err
+				}
+			}
+		}
+		_, err = w.WriteString(output.Format(outs[i], bi.OutputFormat) + "\n")
+		if err != nil {
+			return err
+		}
+	}
+	log.Info().Msgf("Dumped %d odds/verification records", len(outs))
+	return nil
+}
+
 // GetVersion outputs the version of BHLindex.
 func (bi *bhlindex) GetVersion() gnvers.Version {
 	return gnvers.Version{Version: Version, Build: Build}

internal/ent/output/interface.go

@@ -12,6 +12,10 @@ type Dumper interface {
 
 	// DumpOccurrences traverses database and outputs names occurrences in JSON, TSV or CSV format.
 	DumpOccurrences(context.Context, chan<- []Output, []int) error
+
+	// DumpOddsVerification gets result of mapping between Odds values and the percentage of
+	// successful verifications.
+	DumpOddsVerification() ([]Output, error)
 }
 
 // Output interface provides generic functions outputs of verified names, and

internal/ent/output/odds-verif-format.go

@@ -7,7 +7,7 @@ import (
 )
 
 func (o OutputOddsVerification) Name() string {
-	return "odds-verification"
+	return "odds_verification"
 }
 
 func (o OutputOddsVerification) header() []string {

internal/interface.go

@@ -31,6 +31,10 @@ type BHLindex interface {
 	// TSV, or JSON formats.
 	DumpNames(output.Dumper) error
 
+	// DumpOddsVerification exports mapping of Odds values to verification
+	// percentage
+	DumpOddsVerification(output.Dumper) error
+
 	// GetVersion outputs the version of BHLindex.
 	GetVersion() gnvers.Version
 

internal/io/dumpio/db-dump.go

@@ -140,6 +140,38 @@ func (d *dumpio) outputOccurs(id, limit int, ds []int) ([]output.Output, error)
 	return res, nil
 }
 
+func (d dumpio) getOccurVerif() ([]output.Output, error) {
+	var rows *sql.Rows
+	var err error
+
+	q := `
+  SELECT
+    odds_log10, names_num, verif_percent
+  FROM odds_verifications
+  ORDER BY odds_log10`
+	rows, err = d.db.Query(q)
+	if err != nil {
+		return nil, fmt.Errorf("-> Query %w", err)
+	}
+	defer rows.Close()
+
+	var count int
+	var res []output.Output
+	for rows.Next() {
+		o := output.OutputOddsVerification{}
+		err := rows.Scan(
+			&o.OddsLog10, &o.NamesNum, &o.VerifPercent,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("-> Scan %w", err)
+		}
+		res = append(res, o)
+		count++
+	}
+
+	return res, nil
+}
+
 func getDataSources(ds []int) string {
 	var dataSources string
 	if len(ds) > 0 {

internal/io/dumpio/dumpio.go

@@ -122,9 +122,14 @@ func (d *dumpio) DumpOccurrences(ctx context.Context, ch chan<- []output.Output,
 			)
 		}
 	}
-	fmt.Fprintf(os.Stderr, "\r%s", strings.Repeat(" ", 80))
+	fmt.Fprintf(os.Stderr, "\r%s\r", strings.Repeat(" ", 80))
 	log.Info().Msgf("Dumped %s occurrences",
 		humanize.Comma(int64(count)),
 	)
 	return nil
 }
+
+func (d dumpio) DumpOddsVerification() ([]output.Output, error) {
+	log.Info().Msg("Dumping odds vs verification percentage")
+	return d.getOccurVerif()
+}

scripts/filter.rb

@@ -32,7 +32,7 @@
 r.close
 w.close
 
-puts "\n\nFiltering occurrences.csv to occurrences_filtered.csv\n\n"
+puts "\nFiltering occurrences.csv to occurrences_filtered.csv\n\n"
 
 r = File.open('occurrences.csv')
 w = File.open('occurrences_filtered.csv', 'w:utf-8')