Remove ladder from manifests

.gitignore

@@ -2,3 +2,4 @@
 coverage.*
 *.sqlite*
 ./transcoder
+./data

client/client_test.go

@@ -75,7 +75,7 @@ func (s *clientSuite) TestPlayFragment() {
 	)
 
 	// Request stream and wait until it's available.
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	wait := time.NewTicker(1000 * time.Millisecond)
 Waiting:
 	for {

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   conductor:
-    image: odyseeteam/transcoder-conductor:24.2.1
+    image: odyseeteam/transcoder-conductor:24.2.2
     platform: linux/amd64
     container_name: tc-conductor
     command:
@@ -26,7 +26,7 @@ services:
         max_attempts: 3
         window: 120s
   cworker:
-    image: odyseeteam/transcoder-cworker:24.2.1
+    image: odyseeteam/transcoder-cworker:24.2.2
     platform: linux/amd64
     container_name: tc-cworker
     command:
@@ -52,7 +52,7 @@ services:
     ports:
       - '6379:6379'
     volumes:
-      - redis-data:/data
+      - ./data/redis:/data
     command: >
       --requirepass odyredis --appendonly yes
     labels:
@@ -66,7 +66,7 @@ services:
     environment:
       POSTGRES_PASSWORD: odyseeteam
     volumes:
-      - db-data:/var/lib/postgresql/data
+      - ./data/postgres:/var/lib/postgresql/data
     command:
       - -clogging_collector=on
       - -clog_statement=all
@@ -84,15 +84,15 @@ services:
       - "38861:38861"
       - "41949:41949"
     volumes:
-      - minio-data:/data
+      - ./data/minio:/data
     environment:
       MINIO_ROOT_USER: ody
       MINIO_ROOT_PASSWORD: odyseetes3
     command: server --address 0.0.0.0:9000 /data
   minio-prepare:
     image: minio/mc
     volumes:
-      - minio-data:/data
+      - ./data/minio:/data
     entrypoint: >
       /bin/sh -c "
       /usr/bin/mc config host add myminio http://minio:9000 ody odyseetes3;
@@ -102,9 +102,3 @@ services:
       exit 0;
       "
     depends_on: ["minio"]
-
-volumes:
-  tower-data: {}
-  db-data: {}
-  minio-data: {}
-  redis-data: {}

encoder/encoder_test.go

@@ -54,34 +54,16 @@ func (s *encoderSuite) TearDownSuite() {
 
 func (s *encoderSuite) TestCheckFastStart() {
 	absPath, _ := filepath.Abs(s.file.Name())
-	e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default))
+	e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath(""))
 	s.Require().NoError(err)
 	m, err := e.GetMetadata(absPath)
 	s.Require().NoError(err)
 	s.True(m.FastStart)
 }
 
-func (s *encoderSuite) TestLadder() {
-	url := "CahlenLee_20220123_GrapheneOSOnAPixel4XL#1a208b628290b2514b632958c623c08fc0c190d2"
-	e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default))
-	s.Require().NoError(err)
-
-	c, err := resolve.ResolveStream(url)
-	s.Require().NoError(err)
-	file, _, err := c.Download(s.T().TempDir())
-	s.Require().NoError(err)
-	file.Close()
-	res, err := e.Encode(file.Name(), s.out)
-	s.Require().NoError(err)
-	s.Equal([]ladder.Tier{
-		{Definition: "360p", Width: 640, Height: 360, VideoBitrate: 500_000, AudioBitrate: "96k", Framerate: FPS0},
-		{Definition: "144p", Width: 256, Height: 144, VideoBitrate: 100_000, AudioBitrate: "96k", Framerate: FPS15},
-	}, res.Ladder.Tiers)
-}
-
 func (s *encoderSuite) TestEncode() {
 	absPath, _ := filepath.Abs(s.file.Name())
-	e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default))
+	e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath(""))
 	s.Require().NoError(err)
 
 	res, err := e.Encode(absPath, s.out)
@@ -136,7 +118,7 @@ v3.m3u8`,
 
 func TestTweakRealStreams(t *testing.T) {
 	t.Skip()
-	encoder, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default))
+	encoder, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath(""))
 	require.NoError(t, err)
 
 	testCases := []struct {

encoder/pool_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/lbryio/transcoder/ladder"
 	"github.com/lbryio/transcoder/pkg/logging/zapadapter"
 	"github.com/lbryio/transcoder/pkg/resolve"
+
 	"github.com/stretchr/testify/suite"
 )
 
@@ -43,12 +44,12 @@ func (s *poolSuite) TearDownSuite() {
 
 func (s *poolSuite) TestEncode() {
 	absPath, _ := filepath.Abs(s.file.Name())
-	enc, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default))
+	enc, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath(""))
 	s.Require().NoError(err)
 	p := NewPool(enc, 10)
 
 	res := (<-p.Encode(absPath, s.out).Value()).(*Result)
-
+	s.Require().NotNil(res, "result shouldn't be nil")
 	vs := res.OrigMeta.VideoStream
 	s.Equal(1920, vs.GetWidth())
 	s.Equal(1080, vs.GetHeight())
@@ -78,7 +79,7 @@ v1.m3u8
 #EXT-X-STREAM-INF:BANDWIDTH=655600,RESOLUTION=640x360,CODECS="avc1.\w+,mp4a.40.2"
 v2.m3u8
 
-#EXT-X-STREAM-INF:BANDWIDTH=180400,RESOLUTION=256x144,CODECS="avc1.\w+,mp4a.40.2"
+#EXT-X-STREAM-INF:BANDWIDTH=215600,RESOLUTION=256x144,CODECS="avc1.\w+,mp4a.40.2"
 v3.m3u8`,
 		"v0.m3u8":       "v0_s000000.ts",
 		"v1.m3u8":       "v1_s000000.ts",

ladder/ladder.go

@@ -3,6 +3,7 @@ package ladder
 import (
 	"math"
 	"strconv"
+	"strings"
 
 	"github.com/shopspring/decimal"
 	yaml "gopkg.in/yaml.v3"
@@ -94,6 +95,10 @@ func (x Ladder) ArgumentSet(out string) *ArgumentSet {
 	}
 }
 
+func (x Ladder) String() string {
+	return strings.Join(x.ArgumentSet("...").GetStrArguments(), " ")
+}
+
 func nsRate(w, h int) int {
 	return int(math.Ceil(float64(800*600) / nsRateFactor))
 }

library/stream.go

@@ -14,8 +14,6 @@ import (
 	"sort"
 	"time"
 
-	"github.com/lbryio/transcoder/ladder"
-
 	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
@@ -57,8 +55,8 @@ type Manifest struct {
 	Size     int64  `yaml:",omitempty"`
 	Checksum string `yaml:",omitempty"`
 
-	Ladder ladder.Ladder `yaml:",omitempty"`
-	Files  []string      `yaml:",omitempty"`
+	FfmpegArgs string   `yaml:"ffmpeg_args,omitempty"`
+	Files      []string `yaml:",omitempty"`
 }
 
 type StreamWalker func(fi fs.FileInfo, fullPath, name string) error

library/testing.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/Pallinder/go-randomdata"
-	"github.com/lbryio/transcoder/ladder"
 	"github.com/lbryio/transcoder/library/db"
 	"github.com/lbryio/transcoder/pkg/migrator"
 
@@ -120,7 +119,6 @@ func GenerateDummyStream() *Stream {
 			SDHash:       randomdata.Alphanumeric(96),
 			TranscodedAt: time.Now(),
 			Size:         int64(randomdata.Number(10000, 5000000)),
-			Ladder:       ladder.Default,
 		},
 	}
 	s.Manifest.TID = s.generateTID()

manager/manager_test.go

@@ -3,6 +3,7 @@ package manager
 import (
 	"math/rand"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/Pallinder/go-randomdata"
@@ -125,7 +126,7 @@ func (s *managerSuite) TestVideo() {
 	expectedUrls := []string{urlsPriority[0], urlsEnabled[0], urlsLevel5[0], urlsNotEnabled[0], urlsEnabled[1]}
 	receivedUrls := []string{}
 	for r := range mgr.Requests() {
-		receivedUrls = append(receivedUrls, r.URI)
+		receivedUrls = append(receivedUrls, strings.TrimPrefix(r.URI, "lbry://"))
 		if len(receivedUrls) == len(expectedUrls) {
 			mgr.pool.Stop()
 			break

pkg/conductor/cmd/main.go

@@ -9,6 +9,7 @@ import (
 	"syscall"
 
 	"github.com/lbryio/transcoder/encoder"
+	"github.com/lbryio/transcoder/internal/version"
 	"github.com/lbryio/transcoder/ladder"
 	"github.com/lbryio/transcoder/library"
 	ldb "github.com/lbryio/transcoder/library/db"
@@ -31,7 +32,6 @@ import (
 	"github.com/spf13/viper"
 	"github.com/valyala/fasthttp"
 	"github.com/valyala/fasthttp/fasthttpadaptor"
-	"go.etcd.io/etcd/api/v3/version"
 	"go.uber.org/zap"
 )
 

pkg/conductor/tasks/tasks.go

@@ -186,7 +186,7 @@ func (r *EncoderRunner) Run(ctx context.Context, t *asynq.Task) error {
 			}
 		}
 
-		time.Sleep(10 * time.Second)
+		time.Sleep(5 * time.Second)
 		// This is removed twice to not wait for upload to finish before freeing up disk space
 		os.RemoveAll(origFile)
 
@@ -204,7 +204,8 @@ func (r *EncoderRunner) Run(ctx context.Context, t *asynq.Task) error {
 			errMtr.WithLabelValues(metrics.StageMetadataFill).Inc()
 			return fmt.Errorf("failed to fill manifest: %w", err)
 		}
-		stream.Manifest.Ladder = res.Ladder
+
+		stream.Manifest.FfmpegArgs = res.Ladder.String()
 		metrics.OutputBytes.Add(float64(stream.Size()))
 		metrics.TranscodedCount.Inc()
 		d, _ := strconv.ParseFloat(res.OrigMeta.FMeta.GetFormat().GetDuration(), 64)

pkg/resolve/resolve.go

@@ -67,6 +67,10 @@ func ResolveStream(uri string) (*ResolvedStream, error) {
 		return nil, err
 	}
 
+	if claim.ShortURL == "" {
+		return nil, ErrClaimNotFound
+	}
+
 	if claim.SigningChannel == nil {
 		return nil, ErrNoSigningChannel
 	}

storage/s3_test.go

@@ -115,8 +115,8 @@ func setupS3(ctx context.Context) (*s3Container, error) {
 		ExposedPorts: []string{"9000/tcp"},
 		WaitingFor:   wait.ForHTTP("/minio/health/ready").WithPort(p),
 		Env: map[string]string{
-			"MINIO_ACCESS_KEY": minioAccessKey,
-			"MINIO_SECRET_KEY": minioSecretKey,
+			"MINIO_ROOT_USER":     minioAccessKey,
+			"MINIO_ROOT_PASSWORD": minioSecretKey,
 		},
 		Entrypoint: []string{"sh"},
 		Cmd:        []string{"-c", fmt.Sprintf("mkdir -p /data/%s && /usr/bin/minio server /data", "")},
@@ -145,11 +145,6 @@ func setupS3(ctx context.Context) (*s3Container, error) {
 		return nil, err
 	}
 
-	err = container.StopLogProducer()
-	if err != nil {
-		return nil, err
-	}
-
 	ip, err := container.Host(ctx)
 	if err != nil {
 		return nil, err