diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9d7661..84fd438 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,33 +2,35 @@ name: CI on: [push, pull_request] env: go-version: "1.19.x" - es-version: "7.17.2" jobs: test: name: Test - strategy: - matrix: - pg-version: ["12", "13"] runs-on: ubuntu-latest + + services: + postgres: + image: postgres:14-alpine + env: + POSTGRES_DB: indexer_test + POSTGRES_USER: indexer_test + POSTGRES_PASSWORD: temba + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + elastic: + image: elasticsearch:7.17.9 + ports: + - 9200:9200 + - 9300:9300 + env: + discovery.type: single-node + steps: - name: Checkout code - uses: actions/checkout@v1 - - - name: Install ElasticSearch - uses: nyaruka/elasticsearch-action@master - with: - elastic version: ${{ env.es-version }} - - - name: Install PostgreSQL - uses: harmon758/postgresql-action@v1 - with: - postgresql version: ${{ matrix.pg-version }} - postgresql db: elastic_test - postgresql user: nyaruka - postgresql password: nyaruka + uses: actions/checkout@v3 - name: Install Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: ${{ env.go-version }} @@ -37,7 +39,7 @@ jobs: - name: Upload coverage if: success() - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true @@ -49,17 +51,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v1 + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Install Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: ${{ env.go-version }} - name: Publish release - uses: goreleaser/goreleaser-action@v1 + uses: goreleaser/goreleaser-action@v4 with: - version: v0.147.2 - args: release --rm-dist + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec75fc..1a09aa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +v8.2.0 (2023-07-31) +------------------------- + * Fix docker file + +v8.0.3 (2023-05-22) +------------------------- + * Change some debug logging to trace level + +v8.0.2 (2023-05-22) +------------------------- + * Stop logging all ES requests in debug mode (only log errors) + +v8.0.1 (2023-05-19) +------------------------- + * Add major version to import path to allow use as library + * Use services in github actions + +v8.0.0 (2023-01-09) +------------------------- + * Test against postgres 14 + * Test against latest elastic search 7.17.x + v7.5.0 ---------- * Change index setting to contacts_index diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3171e5a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.20 + +WORKDIR /usr/src/app + +# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change +COPY go.mod go.sum ./ +RUN go mod download && go mod verify + +COPY . . +RUN go build -v -o /usr/local/bin/app github.com/nyaruka/rp-indexer/v8/cmd/rp-indexer + +CMD ["app"] \ No newline at end of file diff --git a/LICENSE b/LICENSE index ea42b46..7320d99 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -Indexer - content indexing service for the TextIt platform. +Indexer - content indexing service for RapidPro/TextIt. -Copyright (C) 2018-2022 TextIt. +Copyright (C) 2018-2022 Nyaruka, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go index d9893a4..429766d 100644 --- a/cmd/rp-indexer/main.go +++ b/cmd/rp-indexer/main.go @@ -10,8 +10,8 @@ import ( "github.com/evalphobia/logrus_sentry" _ "github.com/lib/pq" "github.com/nyaruka/ezconf" - indexer "github.com/nyaruka/rp-indexer" - "github.com/nyaruka/rp-indexer/indexers" + indexer "github.com/nyaruka/rp-indexer/v8" + "github.com/nyaruka/rp-indexer/v8/indexers" log "github.com/sirupsen/logrus" ) diff --git a/daemon.go b/daemon.go index 8831075..1ede36f 100644 --- a/daemon.go +++ b/daemon.go @@ -6,7 +6,7 @@ import ( "time" "github.com/nyaruka/gocommon/analytics" - "github.com/nyaruka/rp-indexer/indexers" + "github.com/nyaruka/rp-indexer/v8/indexers" "github.com/sirupsen/logrus" ) diff --git a/go.mod b/go.mod index 5cef951..bfeec25 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/nyaruka/rp-indexer +module github.com/nyaruka/rp-indexer/v8 go 1.19 @@ -28,7 +28,7 @@ require ( github.com/nyaruka/librato v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect - golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 67b9235..dc113f8 100644 --- a/go.sum +++ b/go.sum @@ -57,17 +57,17 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4= -golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/goreleaser.yml b/goreleaser.yml index 58364d8..86e271c 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -2,15 +2,18 @@ build: main: ./cmd/rp-indexer/main.go binary: rp-indexer goos: - - windows - darwin - linux goarch: - amd64 - arm64 +changelog: + filters: + exclude: + - "^Update CHANGELOG.md" + archives: - files: - - LICENSE - - README.md - + - LICENSE + - README.md diff --git a/indexers/base.go b/indexers/base.go index dad6dbc..5c85a68 100644 --- a/indexers/base.go +++ b/indexers/base.go @@ -10,7 +10,7 @@ import ( "time" "github.com/nyaruka/gocommon/jsonx" - "github.com/nyaruka/rp-indexer/utils" + "github.com/nyaruka/rp-indexer/v8/utils" "github.com/sirupsen/logrus" ) @@ -275,7 +275,7 @@ func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) { createdCount, deletedCount, conflictedCount := 0, 0, 0 for _, item := range response.Items { if item.Index.ID != "" { - logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("index response") + logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Trace("index response") if item.Index.Status == 200 || item.Index.Status == 201 { createdCount++ } else if item.Index.Status == 409 { @@ -284,7 +284,7 @@ func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) { logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).WithField("result", item.Index.Result).Error("error indexing document") } } else if item.Delete.ID != "" { - logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response") + logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Trace("delete response") if item.Delete.Status == 200 { deletedCount++ } else if item.Delete.Status == 409 { diff --git a/indexers/base_test.go b/indexers/base_test.go index 60ab229..9def23f 100644 --- a/indexers/base_test.go +++ b/indexers/base_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/nyaruka/rp-indexer/indexers" + "github.com/nyaruka/rp-indexer/v8/indexers" "github.com/olivere/elastic/v7" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -24,7 +24,7 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) { testDB, err := os.ReadFile("../testdb.sql") require.NoError(t, err) - db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable") + db, err := sql.Open("postgres", "postgres://indexer_test:temba@localhost:5432/indexer_test?sslmode=disable") require.NoError(t, err) _, err = db.Exec(string(testDB)) diff --git a/indexers/contacts.go b/indexers/contacts.go index ae466d8..779fe00 100644 --- a/indexers/contacts.go +++ b/indexers/contacts.go @@ -209,14 +209,14 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti lastModified = modifiedOn if isActive { - logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Debug("modified contact") + logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Trace("modified contact") subBatch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID)) subBatch.WriteString("\n") subBatch.WriteString(contactJSON) subBatch.WriteString("\n") } else { - logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Debug("deleted contact") + logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Trace("deleted contact") subBatch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID)) subBatch.WriteString("\n") diff --git a/indexers/contacts_test.go b/indexers/contacts_test.go index d284ce2..69ce80a 100644 --- a/indexers/contacts_test.go +++ b/indexers/contacts_test.go @@ -7,7 +7,7 @@ import ( _ "github.com/lib/pq" "github.com/nyaruka/gocommon/jsonx" - "github.com/nyaruka/rp-indexer/indexers" + "github.com/nyaruka/rp-indexer/v8/indexers" "github.com/olivere/elastic/v7" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/utils/http.go b/utils/http.go index 152f499..4fc6d0d 100644 --- a/utils/http.go +++ b/utils/http.go @@ -50,43 +50,39 @@ func shouldRetry(request *http.Request, response *http.Response, withDelay time. } // MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct -func MakeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) { +func MakeJSONRequest(method string, url string, body []byte, dest any) (*http.Response, error) { + l := log.WithField("url", url).WithField("method", method) + req, _ := httpx.NewRequest(method, url, bytes.NewReader(body), map[string]string{"Content-Type": "application/json"}) resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil) - - l := log.WithField("url", url).WithField("method", method).WithField("request", body) if err != nil { - l.WithError(err).Error("error making ES request") + l.WithError(err).Error("error making request") return resp, err } defer resp.Body.Close() // if we have a body, try to decode it - jsonBody, err := ioutil.ReadAll(resp.Body) + respBody, err := ioutil.ReadAll(resp.Body) if err != nil { - l.WithError(err).Error("error reading ES response") + l.WithError(err).Error("error reading response") return resp, err } - l = l.WithField("response", string(jsonBody)).WithField("status", resp.StatusCode) + l = l.WithField("response", string(respBody)).WithField("status", resp.StatusCode) // error if we got a non-200 if resp.StatusCode != http.StatusOK { l.WithError(err).Error("error reaching ES") - return resp, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, jsonBody) - } - - if jsonStruct == nil { - l.Debug("ES request successful") - return resp, nil + return resp, fmt.Errorf("received non-200 response %d: %s", resp.StatusCode, respBody) } - err = json.Unmarshal(jsonBody, jsonStruct) - if err != nil { - l.WithError(err).Error("error unmarshalling ES response") - return resp, err + if dest != nil { + err = json.Unmarshal(respBody, dest) + if err != nil { + l.WithError(err).Error("error unmarshalling response") + return resp, err + } } - l.Debug("ES request successful") return resp, nil } diff --git a/utils/http_test.go b/utils/http_test.go index fd8dea7..e31c041 100644 --- a/utils/http_test.go +++ b/utils/http_test.go @@ -5,7 +5,7 @@ import ( "net/http/httptest" "testing" - "github.com/nyaruka/rp-indexer/utils" + "github.com/nyaruka/rp-indexer/v8/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" )