Merge pull request #24 from Ilhasoft/update/v8.2.0

Update/v8.2.0

.github/workflows/ci.yml

@@ -2,33 +2,35 @@ name: CI
 on: [push, pull_request]
 env:
   go-version: "1.19.x"
-  es-version: "7.17.2"
 jobs:
   test:
     name: Test
-    strategy:
-      matrix:
-        pg-version: ["12", "13"]
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_DB: indexer_test
+          POSTGRES_USER: indexer_test
+          POSTGRES_PASSWORD: temba
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      elastic:
+          image: elasticsearch:7.17.9
+          ports:
+            - 9200:9200
+            - 9300:9300
+          env:
+            discovery.type: single-node
+
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
-
-      - name: Install ElasticSearch
-        uses: nyaruka/elasticsearch-action@master
-        with:
-          elastic version: ${{ env.es-version }}
-
-      - name: Install PostgreSQL
-        uses: harmon758/postgresql-action@v1
-        with:
-          postgresql version: ${{ matrix.pg-version }}
-          postgresql db: elastic_test
-          postgresql user: nyaruka
-          postgresql password: nyaruka
+        uses: actions/checkout@v3
 
       - name: Install Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.go-version }}
 
@@ -37,7 +39,7 @@ jobs:
 
       - name: Upload coverage
         if: success()
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
@@ -49,17 +51,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ env.go-version }}
 
       - name: Publish release
-        uses: goreleaser/goreleaser-action@v1
+        uses: goreleaser/goreleaser-action@v4
         with:
-          version: v0.147.2
-          args: release --rm-dist
+          args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -1,3 +1,25 @@
+v8.2.0 (2023-07-31)
+-------------------------
+ * Fix docker file
+
+v8.0.3 (2023-05-22)
+-------------------------
+ * Change some debug logging to trace level
+
+v8.0.2 (2023-05-22)
+-------------------------
+ * Stop logging all ES requests in debug mode (only log errors)
+
+v8.0.1 (2023-05-19)
+-------------------------
+ * Add major version to import path to allow use as library
+ * Use services in github actions
+
+v8.0.0 (2023-01-09)
+-------------------------
+ * Test against postgres 14
+ * Test against latest elastic search 7.17.x
+
 v7.5.0
 ----------
  * Change index setting to contacts_index

Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.20
+
+WORKDIR /usr/src/app
+
+# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+COPY . .
+RUN go build -v -o /usr/local/bin/app github.com/nyaruka/rp-indexer/v8/cmd/rp-indexer
+
+CMD ["app"]

LICENSE

@@ -1,6 +1,6 @@
-Indexer - content indexing service for the TextIt platform.
+Indexer - content indexing service for RapidPro/TextIt.
 
-Copyright (C) 2018-2022 TextIt.
+Copyright (C) 2018-2022 Nyaruka, Inc.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as

cmd/rp-indexer/main.go

@@ -10,8 +10,8 @@ import (
 	"github.com/evalphobia/logrus_sentry"
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/ezconf"
-	indexer "github.com/nyaruka/rp-indexer"
-	"github.com/nyaruka/rp-indexer/indexers"
+	indexer "github.com/nyaruka/rp-indexer/v8"
+	"github.com/nyaruka/rp-indexer/v8/indexers"
 	log "github.com/sirupsen/logrus"
 )
 

daemon.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/nyaruka/gocommon/analytics"
-	"github.com/nyaruka/rp-indexer/indexers"
+	"github.com/nyaruka/rp-indexer/v8/indexers"
 	"github.com/sirupsen/logrus"
 )
 

go.mod

@@ -1,4 +1,4 @@
-module github.com/nyaruka/rp-indexer
+module github.com/nyaruka/rp-indexer/v8
 
 go 1.19
 
@@ -28,7 +28,7 @@ require (
 	github.com/nyaruka/librato v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect
-	golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -57,17 +57,17 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4=
-golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
-golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

goreleaser.yml

@@ -2,15 +2,18 @@ build:
   main: ./cmd/rp-indexer/main.go
   binary: rp-indexer
   goos:
-    - windows
     - darwin
     - linux
   goarch:
     - amd64
     - arm64
 
+changelog:
+  filters:
+    exclude:
+      - "^Update CHANGELOG.md"
+
 archives:
   - files:
-    - LICENSE
-    - README.md
-
+      - LICENSE
+      - README.md

indexers/base.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/nyaruka/gocommon/jsonx"
-	"github.com/nyaruka/rp-indexer/utils"
+	"github.com/nyaruka/rp-indexer/v8/utils"
 	"github.com/sirupsen/logrus"
 )
 
@@ -275,7 +275,7 @@ func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) {
 	createdCount, deletedCount, conflictedCount := 0, 0, 0
 	for _, item := range response.Items {
 		if item.Index.ID != "" {
-			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("index response")
+			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Trace("index response")
 			if item.Index.Status == 200 || item.Index.Status == 201 {
 				createdCount++
 			} else if item.Index.Status == 409 {
@@ -284,7 +284,7 @@ func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) {
 				logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).WithField("result", item.Index.Result).Error("error indexing document")
 			}
 		} else if item.Delete.ID != "" {
-			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response")
+			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Trace("delete response")
 			if item.Delete.Status == 200 {
 				deletedCount++
 			} else if item.Delete.Status == 409 {

indexers/base_test.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/nyaruka/rp-indexer/indexers"
+	"github.com/nyaruka/rp-indexer/v8/indexers"
 	"github.com/olivere/elastic/v7"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -24,7 +24,7 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) {
 	testDB, err := os.ReadFile("../testdb.sql")
 	require.NoError(t, err)
 
-	db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")
+	db, err := sql.Open("postgres", "postgres://indexer_test:temba@localhost:5432/indexer_test?sslmode=disable")
 	require.NoError(t, err)
 
 	_, err = db.Exec(string(testDB))

indexers/contacts.go

@@ -209,14 +209,14 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti
 			lastModified = modifiedOn
 
 			if isActive {
-				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Debug("modified contact")
+				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Trace("modified contact")
 
 				subBatch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID))
 				subBatch.WriteString("\n")
 				subBatch.WriteString(contactJSON)
 				subBatch.WriteString("\n")
 			} else {
-				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Debug("deleted contact")
+				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Trace("deleted contact")
 
 				subBatch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID))
 				subBatch.WriteString("\n")

indexers/contacts_test.go

@@ -7,7 +7,7 @@ import (
 
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/gocommon/jsonx"
-	"github.com/nyaruka/rp-indexer/indexers"
+	"github.com/nyaruka/rp-indexer/v8/indexers"
 	"github.com/olivere/elastic/v7"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

utils/http.go

@@ -50,43 +50,39 @@ func shouldRetry(request *http.Request, response *http.Response, withDelay time.
 }
 
 // MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct
-func MakeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) {
+func MakeJSONRequest(method string, url string, body []byte, dest any) (*http.Response, error) {
+	l := log.WithField("url", url).WithField("method", method)
+
 	req, _ := httpx.NewRequest(method, url, bytes.NewReader(body), map[string]string{"Content-Type": "application/json"})
 	resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
-
-	l := log.WithField("url", url).WithField("method", method).WithField("request", body)
 	if err != nil {
-		l.WithError(err).Error("error making ES request")
+		l.WithError(err).Error("error making request")
 		return resp, err
 	}
 	defer resp.Body.Close()
 
 	// if we have a body, try to decode it
-	jsonBody, err := ioutil.ReadAll(resp.Body)
+	respBody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		l.WithError(err).Error("error reading ES response")
+		l.WithError(err).Error("error reading response")
 		return resp, err
 	}
 
-	l = l.WithField("response", string(jsonBody)).WithField("status", resp.StatusCode)
+	l = l.WithField("response", string(respBody)).WithField("status", resp.StatusCode)
 
 	// error if we got a non-200
 	if resp.StatusCode != http.StatusOK {
 		l.WithError(err).Error("error reaching ES")
-		return resp, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, jsonBody)
-	}
-
-	if jsonStruct == nil {
-		l.Debug("ES request successful")
-		return resp, nil
+		return resp, fmt.Errorf("received non-200 response %d: %s", resp.StatusCode, respBody)
 	}
 
-	err = json.Unmarshal(jsonBody, jsonStruct)
-	if err != nil {
-		l.WithError(err).Error("error unmarshalling ES response")
-		return resp, err
+	if dest != nil {
+		err = json.Unmarshal(respBody, dest)
+		if err != nil {
+			l.WithError(err).Error("error unmarshalling response")
+			return resp, err
+		}
 	}
 
-	l.Debug("ES request successful")
 	return resp, nil
 }

utils/http_test.go

@@ -5,7 +5,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/nyaruka/rp-indexer/utils"
+	"github.com/nyaruka/rp-indexer/v8/utils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )