Merge pull request #523 from OdyseeTeam/improve-error-logging

Improve error logging

.github/workflows/golangci-lint.yml

@@ -9,24 +9,24 @@ on:
 permissions:
   contents: read
   # Optional: allow read access to pull request. Use with `only-new-issues` option.
-  # pull-requests: read
+  pull-requests: read
 
 jobs:
   golangci:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.22.x'
+          go-version: '1.23'
           cache: false
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
-          version: 'v1.56.2'
-          # only-new-issues: true
+          version: 'v1.60'
+          only-new-issues: true
       - id: govulncheck
         uses: golang/govulncheck-action@v1

.github/workflows/test-release.yml

@@ -15,13 +15,13 @@ jobs:
     name: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: FedericoCarboni/setup-ffmpeg@v2
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.22.x'
+          go-version: '1.23'
         id: go
 
       - name: Retrieve release details
@@ -78,26 +78,26 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.22.x'
+          go-version: '1.23'
         id: go
 
       - run: git branch
 
       - name: Retrieve release details
         id: release
         run: |
-          echo "APP_NAME=$(git describe --tags|sed -e 's/\-v.*//')" >> $GITHUB_OUTPUT
-          echo "APP_VERSION=$(git describe --tags|sed 's/api-v\([0-9.]*\)-.*/\1/')" >> $GITHUB_OUTPUT
+          echo "APP_NAME=$(git describe --tags|sed -e 's/\-v.*//')" >> $GITHUB_ENV
+          echo "APP_VERSION=$(git describe --tags --match 'api-v*'|sed 's/api-v\([0-9.]*\)/\1/')" >> $GITHUB_ENV
 
       - name: Release details
         run: |
-            echo "::group::${{ steps.release.outputs.APP_NAME }}"
-            echo "version ${{ steps.release.outputs.APP_VERSION }}"
+            echo "::group::${{ env.APP_NAME }}"
+            echo "version ${{ env.APP_VERSION }}"
             echo "env ${{ github.ref }}"
             echo "::endgroup::"
 
@@ -110,21 +110,21 @@ jobs:
       # odysee-api deployment
 
       - name: Build odysee-api
-        if: steps.release.outputs.APP_NAME == 'api'
+        if: env.APP_NAME == 'api'
         run: make oapi
 
       - name: Build and push odysee-api docker image
-        if: steps.release.outputs.APP_NAME == 'api'
+        if: env.APP_NAME == 'api'
         uses: docker/build-push-action@v2
         with:
           push: true
           context: .
           tags: |
-            odyseeteam/odysee-${{ steps.release.outputs.APP_NAME }}:latest
-            odyseeteam/odysee-${{ steps.release.outputs.APP_NAME }}:${{ steps.release.outputs.APP_VERSION }}
+            odyseeteam/odysee-${{ env.APP_NAME }}:latest
+            odyseeteam/odysee-${{ env.APP_NAME }}:${{ env.APP_VERSION }}
 
       - name: Deploy odysee-api
-        if: steps.release.outputs.APP_NAME == 'api'
+        if: env.APP_NAME == 'api'
         uses: octokit/[email protected]
         id: deploy-odysee-api
         env:
@@ -134,29 +134,29 @@ jobs:
           repository: odyseeteam/misc-deployment
           event_type: deploy-odysee
           client_payload: |
-            tag: ${{ steps.release.outputs.APP_VERSION }}
+            tag: ${{ env.APP_VERSION }}
             ref: ${{ github.ref }}
             service: odysee-api
 
       # Watchman deployment
 
       - name: Build watchman
-        if: steps.release.outputs.APP_NAME == 'watchman'
+        if: env.APP_NAME == 'watchman'
         run: make watchman
 
       - name: Build watchman docker image
-        if: steps.release.outputs.APP_NAME == 'watchman'
+        if: env.APP_NAME == 'watchman'
         uses: docker/build-push-action@v2
         with:
           push: true
-          context: apps/${{ steps.release.outputs.APP_NAME }}/
+          context: apps/${{ env.APP_NAME }}/
           tags: |
-            odyseeteam/${{ steps.release.outputs.APP_NAME }}:latest
-            odyseeteam/${{ steps.release.outputs.APP_NAME }}:${{ steps.release.outputs.APP_VERSION }}
-          file: apps/${{ steps.release.outputs.APP_NAME }}/Dockerfile
+            odyseeteam/${{ env.APP_NAME }}:latest
+            odyseeteam/${{ env.APP_NAME }}:${{ env.APP_VERSION }}
+          file: apps/${{ env.APP_NAME }}/Dockerfile
 
       - name: Deploy watchman
-        if: steps.release.outputs.APP_NAME == 'watchman'
+        if: env.APP_NAME == 'watchman'
         uses: octokit/[email protected]
         id: deploy-watchman
         env:
@@ -166,6 +166,6 @@ jobs:
           repository: OdyseeTeam/misc-deployment
           event_type: deploy-odysee
           client_payload: |
-            tag: ${{ steps.release.outputs.APP_VERSION }}
+            tag: ${{ env.APP_VERSION }}
             ref: ${{ github.ref }}
-            service: ${{ steps.release.outputs.APP_NAME }}
+            service: ${{ env.APP_NAME }}

Makefile

@@ -19,15 +19,6 @@ prepare_test:
 		$(go) install $($(go) list -tags tools -f '{{range $_, $p := .Imports}}{{$p}} {{end}}')
 	$(go) run . db_migrate_up
 
-.PHONY: test_circleci
-test_circleci:
-	scripts/wait_for_wallet.sh
-	cd tools &&\
-		$(go) install $($(go) list -tags tools -f '{{range $_, $p := .Imports}}{{$p}} {{end}}')
-	$(go) run . db_migrate_up
-	$(go) test -covermode=count -coverprofile=coverage.out ./...
-	goveralls -coverprofile=coverage.out -service=circle-ci -ignore=models/ -repotoken $(COVERALLS_TOKEN)
-
 .PHONY: clean
 clean:
 	rm -rf ./dist

app/arweave/arweave.go

@@ -0,0 +1,81 @@
+package arweave
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+func ReplaceAssetUrls(baseUrl string, structure any, collPath, itemPath string) (any, error) {
+	var origUrls []string
+	urlPaths := map[string][]string{}
+
+	jsonData, err := json.Marshal(structure)
+	if err != nil {
+		return nil, err
+	}
+
+	items := gjson.GetBytes(jsonData, collPath)
+	items.ForEach(func(key, value gjson.Result) bool {
+		urlPath := fmt.Sprintf("%s.%s.%s", collPath, key.String(), itemPath)
+		url := gjson.GetBytes(jsonData, urlPath).String()
+		origUrls = append(origUrls, url)
+		if slice, exists := urlPaths[url]; exists {
+			urlPaths[url] = append(slice, urlPath)
+		} else {
+			urlPaths[url] = []string{urlPath}
+		}
+		return true
+	})
+
+	resolver := NewArfleetResolver(baseUrl)
+	subsUrls, err := resolver.ResolveUrls(origUrls)
+	if err != nil {
+		return nil, err
+	}
+
+	for oldURL, newURL := range subsUrls {
+		for _, path := range urlPaths[oldURL] {
+			jsonData, _ = sjson.SetBytes(jsonData, path, newURL)
+		}
+	}
+
+	var d any
+	return d, json.Unmarshal(jsonData, &d)
+}
+
+func ReplaceAssetUrl(baseUrl string, structure any, path string) (any, error) {
+	jsonData, err := json.Marshal(structure)
+	if err != nil {
+		return nil, err
+	}
+
+	origUrl := gjson.GetBytes(jsonData, path).String()
+
+	resolver := NewArfleetResolver(baseUrl)
+	subsUrls, err := resolver.ResolveUrls([]string{origUrl})
+
+	if err != nil {
+		return nil, err
+	}
+	if newUrl, ok := subsUrls[origUrl]; ok {
+		jsonData, err = sjson.SetBytes(jsonData, path, newUrl)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var d any
+	return d, json.Unmarshal(jsonData, &d)
+}
+
+func GetClaimUrl(baseUrl, claim_id string) (string, error) {
+	resolver := NewArfleetResolver(baseUrl)
+	r, err := resolver.ResolveClaims([]string{claim_id})
+	if err != nil {
+		return "", err
+	}
+	return r[claim_id], nil
+}

app/arweave/arweave_test.go

@@ -0,0 +1,63 @@
+package arweave
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/ybbus/jsonrpc"
+)
+
+func TestReplaceAssetUrls(t *testing.T) {
+	t.Skip("skipping this in automated mode as it requires extra setup on arfleet")
+
+	require := require.New(t)
+	assert := assert.New(t)
+
+	absPath, _ := filepath.Abs("./testdata/claim_search.json")
+	f, err := os.ReadFile(absPath)
+	require.NoError(err)
+	var resp jsonrpc.RPCResponse
+	require.NoError(json.Unmarshal(f, &resp))
+	result, err := ReplaceAssetUrls("http://odycdn.com", resp.Result, "items", "value.thumbnail.url")
+	require.NoError(err)
+
+	out, err := json.MarshalIndent(result, "", "  ")
+	require.NoError(err)
+	re := regexp.MustCompile(`http://odycdn.com/explore/\w{64}\?filename=\w{64}\.webp`)
+	matches := re.FindAllString(string(out), -1)
+	assert.Equal(2, len(matches))
+}
+
+func TestReplaceAssetUrl(t *testing.T) {
+	t.Skip("skipping this in automated mode as it requires extra setup on arfleet")
+
+	require := require.New(t)
+	assert := assert.New(t)
+
+	absPath, _ := filepath.Abs("./testdata/resolve.json")
+	f, err := os.ReadFile(absPath)
+	require.NoError(err)
+	var resp jsonrpc.RPCResponse
+	require.NoError(json.Unmarshal(f, &resp))
+	result, err := ReplaceAssetUrl("http://odycdn.com", resp.Result.(map[string]any)["lbry://@MySillyReactions#d1ae6a9097b44691d318a5bfc6dc1240311c75e2"], "value.thumbnail.url")
+	require.NoError(err)
+
+	out, err := json.MarshalIndent(result, "", "  ")
+	require.NoError(err)
+	assert.Regexp(`http://odycdn.com/explore/\w{64}\?filename=\w{64}\.jpg`, string(out))
+}
+
+func TestGetClaimUrl(t *testing.T) {
+	t.Skip("skipping this in automated mode as it requires extra setup on arfleet")
+
+	require := require.New(t)
+	assert := assert.New(t)
+	url, err := GetClaimUrl("https://cdnhost.com", "91e8caf6d1e740aaa6235d4eb81b21ec21cb2652")
+	require.NoError(err)
+	assert.Regexp(`^https://cdnhost.com/explore/\w+\?data_item_id=\w+&filename=\w+.mp4$`, url)
+}