Merge pull request #13 from mellowdrifter/main

Adjust concurrency

.gitignore

@@ -0,0 +1,5 @@
+rpsl-parser/proto/rpsl.pb.go
+rpsl-tools/data/rad2.db
+rpsl-tools/data/rad3.db
+rpsl-tools/data/radb.db
+rpsl-parser/proto/rpsl.pb.go

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.6
 	github.com/manrs-tools/contrib v0.0.0-20190703155503-b9ddd2b08592
+	github.com/pkg/profile v1.6.0
 )
 
 require google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/manrs-tools/contrib v0.0.0-20190703155503-b9ddd2b08592 h1:QdJoifdAPuKychiugiNVpKPCR00BGWyvcsNF2sGwjnE=
 github.com/manrs-tools/contrib v0.0.0-20190703155503-b9ddd2b08592/go.mod h1:TWtXtOwANTt3GOnfY4RfYy5Sdju84mVEKu0qBi5X7gQ=
+github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM=
+github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=

rpsl-parser/examples/testdata/test.db

@@ -0,0 +1,55 @@
+aut-num:        AS10279
+as-name:        WCCOM-AS
+descr:          WEST CAROLINA COMM, LLC
+descr:          229 Hwy 28 By-Pass
+descr:          Abbeville, SC 29620
+descr:          US
+admin-c:        WCTEL1-ARIN
+tech-c:         TWN1-ARIN
+mnt-by:         MNT-WCCL-3
+changed:        [email protected] 20191217
+source:         ARIN
+
+aut-num:        AS10302
+as-name:        NEGIA-AS
+descr:          Snappy Internet & Telecom
+descr:          Miami, Florida
+member-of:      AS-NEGIA-BACKBONE
+tech-c:         FI14-ARIN
+admin-c:        FI14-ARIN
+mnt-by:         MNT-COFS
+notify:         [email protected]
+changed:        [email protected] 20110920
+source:         ARIN
+
+aut-num:        AS10352
+as-name:        SOLARUS-AS
+descr:          Solarus
+                440 E Grand Avenue
+                Wisconsin Rapids WI 54494
+                United States
+mp-export:      afi any.unicast to AS-ANY announce AS-10352ALL
+admin-c:        NEO4-ARIN
+tech-c:         JSL85-ARIN
+tech-c:         NEO4-ARIN
+mnt-by:         MNT-WCTC
+created:        2021-07-08T11:59:46Z
+last-modified:  2021-07-08T11:59:46Z
+source:         ARIN
+
+aut-num:        AS10381
+as-name:        SWCP-AS
+descr:          Southwest Cyberport, Inc
+                5021 Indian School NE
+                Suite 600
+                Albuquerque NM 87110
+                United States
+admin-c:        JKC3-ARIN
+tech-c:         COSTL2-ARIN
+tech-c:         JKC3-ARIN
+tech-c:         MC143-ARIN
+tech-c:         NOS19-ARIN
+mnt-by:         MNT-SWCP
+created:        2020-07-07T21:45:24Z
+last-modified:  2021-10-07T17:53:59Z
+source:         ARIN

rpsl-parser/irr.go

@@ -24,12 +24,24 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"sync"
 	"unicode"
 
 	"github.com/golang/glog"
 	rppb "github.com/manrs-tools/contrib/rpsl-parser/proto"
 )
 
+type Records struct {
+	Records []*rppb.Record
+	mu      sync.Mutex
+}
+
+func (r *Records) AddRecord(record *rppb.Record) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.Records = append(r.Records, record)
+}
+
 // Reader is a struct to manage access to the irr database file.
 type Reader struct {
 	reader *bufio.Reader
@@ -328,7 +340,7 @@ func (r *Reader) initRecord() (*rppb.Record, error) {
 }
 
 // Parse parses through the content sending resulting records into a channel to the caller.
-func Parse(rdr *Reader, rc chan<- *rppb.Record) {
+func Parse(rdr *Reader, records *Records) {
 	// Read the file content, return all accumulated records.
 	// Return in case of parsing errors on record/keyword type.
 	// Return in case of reading error.
@@ -348,15 +360,15 @@ func Parse(rdr *Reader, rc chan<- *rppb.Record) {
 
 			err := rdr.consumeColon()
 			if err != nil {
-				rc <- rec
+				records.AddRecord(rec)
 				glog.Infof("failed to consume a key's colon separator: %v", err)
 				break
 			}
 
 			val, re, err := rdr.readValue()
 			if err != nil {
 				addKV(rec, key, val)
-				rc <- rec
+				records.AddRecord(rec)
 				if err == io.EOF {
 					// EOF in a read means moving to the next file.
 					glog.Infof("found an EOF while reading a value: %v", err)
@@ -373,6 +385,6 @@ func Parse(rdr *Reader, rc chan<- *rppb.Record) {
 				break
 			}
 		}
-		rc <- rec
+		records.AddRecord(rec)
 	}
 }