From aa8899147ccff8f9253685a9d4c162b43e3280ad Mon Sep 17 00:00:00 2001 From: Andriy Biletsky Date: Tue, 20 Feb 2024 00:07:04 +0700 Subject: [PATCH] Remove ladder from manifests --- .gitignore | 1 + client/client_test.go | 2 +- docker-compose.yml | 18 ++++++------------ encoder/encoder_test.go | 24 +++--------------------- encoder/pool_test.go | 7 ++++--- ladder/ladder.go | 5 +++++ library/stream.go | 6 ++---- library/testing.go | 2 -- manager/manager_test.go | 3 ++- pkg/conductor/cmd/main.go | 2 +- pkg/conductor/tasks/tasks.go | 5 +++-- pkg/resolve/resolve.go | 4 ++++ storage/s3_test.go | 9 ++------- 13 files changed, 34 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index 5dd07a6..1c30e4a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ coverage.* *.sqlite* ./transcoder +./data diff --git a/client/client_test.go b/client/client_test.go index 2620df4..395175c 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -75,7 +75,7 @@ func (s *clientSuite) TestPlayFragment() { ) // Request stream and wait until it's available. - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) wait := time.NewTicker(1000 * time.Millisecond) Waiting: for { diff --git a/docker-compose.yml b/docker-compose.yml index 9e0b5b1..a0b9161 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.7' services: conductor: - image: odyseeteam/transcoder-conductor:24.2.1 + image: odyseeteam/transcoder-conductor:24.2.2 platform: linux/amd64 container_name: tc-conductor command: @@ -26,7 +26,7 @@ services: max_attempts: 3 window: 120s cworker: - image: odyseeteam/transcoder-cworker:24.2.1 + image: odyseeteam/transcoder-cworker:24.2.2 platform: linux/amd64 container_name: tc-cworker command: @@ -52,7 +52,7 @@ services: ports: - '6379:6379' volumes: - - redis-data:/data + - ./data/redis:/data command: > --requirepass odyredis --appendonly yes labels: @@ -66,7 +66,7 @@ services: environment: POSTGRES_PASSWORD: odyseeteam volumes: - - db-data:/var/lib/postgresql/data + - ./data/postgres:/var/lib/postgresql/data command: - -clogging_collector=on - -clog_statement=all @@ -84,7 +84,7 @@ services: - "38861:38861" - "41949:41949" volumes: - - minio-data:/data + - ./data/minio:/data environment: MINIO_ROOT_USER: ody MINIO_ROOT_PASSWORD: odyseetes3 @@ -92,7 +92,7 @@ services: minio-prepare: image: minio/mc volumes: - - minio-data:/data + - ./data/minio:/data entrypoint: > /bin/sh -c " /usr/bin/mc config host add myminio http://minio:9000 ody odyseetes3; @@ -102,9 +102,3 @@ services: exit 0; " depends_on: ["minio"] - -volumes: - tower-data: {} - db-data: {} - minio-data: {} - redis-data: {} diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go index 41ca337..b5d1866 100644 --- a/encoder/encoder_test.go +++ b/encoder/encoder_test.go @@ -54,34 +54,16 @@ func (s *encoderSuite) TearDownSuite() { func (s *encoderSuite) TestCheckFastStart() { absPath, _ := filepath.Abs(s.file.Name()) - e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default)) + e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath("")) s.Require().NoError(err) m, err := e.GetMetadata(absPath) s.Require().NoError(err) s.True(m.FastStart) } -func (s *encoderSuite) TestLadder() { - url := "CahlenLee_20220123_GrapheneOSOnAPixel4XL#1a208b628290b2514b632958c623c08fc0c190d2" - e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default)) - s.Require().NoError(err) - - c, err := resolve.ResolveStream(url) - s.Require().NoError(err) - file, _, err := c.Download(s.T().TempDir()) - s.Require().NoError(err) - file.Close() - res, err := e.Encode(file.Name(), s.out) - s.Require().NoError(err) - s.Equal([]ladder.Tier{ - {Definition: "360p", Width: 640, Height: 360, VideoBitrate: 500_000, AudioBitrate: "96k", Framerate: FPS0}, - {Definition: "144p", Width: 256, Height: 144, VideoBitrate: 100_000, AudioBitrate: "96k", Framerate: FPS15}, - }, res.Ladder.Tiers) -} - func (s *encoderSuite) TestEncode() { absPath, _ := filepath.Abs(s.file.Name()) - e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default)) + e, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath("")) s.Require().NoError(err) res, err := e.Encode(absPath, s.out) @@ -136,7 +118,7 @@ v3.m3u8`, func TestTweakRealStreams(t *testing.T) { t.Skip() - encoder, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default)) + encoder, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath("")) require.NoError(t, err) testCases := []struct { diff --git a/encoder/pool_test.go b/encoder/pool_test.go index fcaf9e1..1e42591 100644 --- a/encoder/pool_test.go +++ b/encoder/pool_test.go @@ -10,6 +10,7 @@ import ( "github.com/lbryio/transcoder/ladder" "github.com/lbryio/transcoder/pkg/logging/zapadapter" "github.com/lbryio/transcoder/pkg/resolve" + "github.com/stretchr/testify/suite" ) @@ -43,12 +44,12 @@ func (s *poolSuite) TearDownSuite() { func (s *poolSuite) TestEncode() { absPath, _ := filepath.Abs(s.file.Name()) - enc, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default)) + enc, err := NewEncoder(Configure().Log(zapadapter.NewKV(nil)).Ladder(ladder.Default).SpritegenPath("")) s.Require().NoError(err) p := NewPool(enc, 10) res := (<-p.Encode(absPath, s.out).Value()).(*Result) - + s.Require().NotNil(res, "result shouldn't be nil") vs := res.OrigMeta.VideoStream s.Equal(1920, vs.GetWidth()) s.Equal(1080, vs.GetHeight()) @@ -78,7 +79,7 @@ v1.m3u8 #EXT-X-STREAM-INF:BANDWIDTH=655600,RESOLUTION=640x360,CODECS="avc1.\w+,mp4a.40.2" v2.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=180400,RESOLUTION=256x144,CODECS="avc1.\w+,mp4a.40.2" +#EXT-X-STREAM-INF:BANDWIDTH=215600,RESOLUTION=256x144,CODECS="avc1.\w+,mp4a.40.2" v3.m3u8`, "v0.m3u8": "v0_s000000.ts", "v1.m3u8": "v1_s000000.ts", diff --git a/ladder/ladder.go b/ladder/ladder.go index 1ee9e32..488bd62 100644 --- a/ladder/ladder.go +++ b/ladder/ladder.go @@ -3,6 +3,7 @@ package ladder import ( "math" "strconv" + "strings" "github.com/shopspring/decimal" yaml "gopkg.in/yaml.v3" @@ -94,6 +95,10 @@ func (x Ladder) ArgumentSet(out string) *ArgumentSet { } } +func (x Ladder) String() string { + return strings.Join(x.ArgumentSet("...").GetStrArguments(), " ") +} + func nsRate(w, h int) int { return int(math.Ceil(float64(800*600) / nsRateFactor)) } diff --git a/library/stream.go b/library/stream.go index 8914d12..ab803d8 100644 --- a/library/stream.go +++ b/library/stream.go @@ -14,8 +14,6 @@ import ( "sort" "time" - "github.com/lbryio/transcoder/ladder" - "github.com/karrick/godirwalk" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -57,8 +55,8 @@ type Manifest struct { Size int64 `yaml:",omitempty"` Checksum string `yaml:",omitempty"` - Ladder ladder.Ladder `yaml:",omitempty"` - Files []string `yaml:",omitempty"` + FfmpegArgs string `yaml:"ffmpeg_args,omitempty"` + Files []string `yaml:",omitempty"` } type StreamWalker func(fi fs.FileInfo, fullPath, name string) error diff --git a/library/testing.go b/library/testing.go index 43be1f2..8bdbad4 100644 --- a/library/testing.go +++ b/library/testing.go @@ -11,7 +11,6 @@ import ( "time" "github.com/Pallinder/go-randomdata" - "github.com/lbryio/transcoder/ladder" "github.com/lbryio/transcoder/library/db" "github.com/lbryio/transcoder/pkg/migrator" @@ -120,7 +119,6 @@ func GenerateDummyStream() *Stream { SDHash: randomdata.Alphanumeric(96), TranscodedAt: time.Now(), Size: int64(randomdata.Number(10000, 5000000)), - Ladder: ladder.Default, }, } s.Manifest.TID = s.generateTID() diff --git a/manager/manager_test.go b/manager/manager_test.go index e1c37f1..653bb40 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -3,6 +3,7 @@ package manager import ( "math/rand" "sort" + "strings" "testing" "github.com/Pallinder/go-randomdata" @@ -125,7 +126,7 @@ func (s *managerSuite) TestVideo() { expectedUrls := []string{urlsPriority[0], urlsEnabled[0], urlsLevel5[0], urlsNotEnabled[0], urlsEnabled[1]} receivedUrls := []string{} for r := range mgr.Requests() { - receivedUrls = append(receivedUrls, r.URI) + receivedUrls = append(receivedUrls, strings.TrimPrefix(r.URI, "lbry://")) if len(receivedUrls) == len(expectedUrls) { mgr.pool.Stop() break diff --git a/pkg/conductor/cmd/main.go b/pkg/conductor/cmd/main.go index 2552b7c..d0929cb 100644 --- a/pkg/conductor/cmd/main.go +++ b/pkg/conductor/cmd/main.go @@ -9,6 +9,7 @@ import ( "syscall" "github.com/lbryio/transcoder/encoder" + "github.com/lbryio/transcoder/internal/version" "github.com/lbryio/transcoder/ladder" "github.com/lbryio/transcoder/library" ldb "github.com/lbryio/transcoder/library/db" @@ -31,7 +32,6 @@ import ( "github.com/spf13/viper" "github.com/valyala/fasthttp" "github.com/valyala/fasthttp/fasthttpadaptor" - "go.etcd.io/etcd/api/v3/version" "go.uber.org/zap" ) diff --git a/pkg/conductor/tasks/tasks.go b/pkg/conductor/tasks/tasks.go index 80700e5..0583293 100644 --- a/pkg/conductor/tasks/tasks.go +++ b/pkg/conductor/tasks/tasks.go @@ -186,7 +186,7 @@ func (r *EncoderRunner) Run(ctx context.Context, t *asynq.Task) error { } } - time.Sleep(10 * time.Second) + time.Sleep(5 * time.Second) // This is removed twice to not wait for upload to finish before freeing up disk space os.RemoveAll(origFile) @@ -204,7 +204,8 @@ func (r *EncoderRunner) Run(ctx context.Context, t *asynq.Task) error { errMtr.WithLabelValues(metrics.StageMetadataFill).Inc() return fmt.Errorf("failed to fill manifest: %w", err) } - stream.Manifest.Ladder = res.Ladder + + stream.Manifest.FfmpegArgs = res.Ladder.String() metrics.OutputBytes.Add(float64(stream.Size())) metrics.TranscodedCount.Inc() d, _ := strconv.ParseFloat(res.OrigMeta.FMeta.GetFormat().GetDuration(), 64) diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go index 8805f66..fa99c0b 100644 --- a/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -67,6 +67,10 @@ func ResolveStream(uri string) (*ResolvedStream, error) { return nil, err } + if claim.ShortURL == "" { + return nil, ErrClaimNotFound + } + if claim.SigningChannel == nil { return nil, ErrNoSigningChannel } diff --git a/storage/s3_test.go b/storage/s3_test.go index eac0cec..89885cb 100644 --- a/storage/s3_test.go +++ b/storage/s3_test.go @@ -115,8 +115,8 @@ func setupS3(ctx context.Context) (*s3Container, error) { ExposedPorts: []string{"9000/tcp"}, WaitingFor: wait.ForHTTP("/minio/health/ready").WithPort(p), Env: map[string]string{ - "MINIO_ACCESS_KEY": minioAccessKey, - "MINIO_SECRET_KEY": minioSecretKey, + "MINIO_ROOT_USER": minioAccessKey, + "MINIO_ROOT_PASSWORD": minioSecretKey, }, Entrypoint: []string{"sh"}, Cmd: []string{"-c", fmt.Sprintf("mkdir -p /data/%s && /usr/bin/minio server /data", "")}, @@ -145,11 +145,6 @@ func setupS3(ctx context.Context) (*s3Container, error) { return nil, err } - err = container.StopLogProducer() - if err != nil { - return nil, err - } - ip, err := container.Host(ctx) if err != nil { return nil, err