chore: replace minio with aliyun oss

Dockerfile

@@ -17,12 +17,11 @@ COPY scripts/build_all.sh build_all.sh
 
 RUN bash ./build_all.sh
 
-FROM ubuntu AS script
+FROM golang:1.19 AS script
 WORKDIR /root
 COPY scripts/wait.sh wait.sh
-COPY configs configs
 
-FROM golang:1.19 AS runtime
+FROM ubuntu AS runtime
 
 ENV CGO_ENABLED 0
 ENV GOOS linux
@@ -31,11 +30,8 @@ ENV GOPROXY https://goproxy.cn,direct
 WORKDIR /root
 COPY --from=build /root .
 COPY --from=script /root/wait.sh ./wait.sh
-COPY --from=script /root/configs ./configs
 COPY scripts/wait.sh wait.sh
-RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list \
-    && apt update \
+RUN apt update \
     && apt install -y netcat \
-    && apt install -y ffmpeg \
     && apt install -y wget \
     && wget https://ghproxy.com/https://raw.githubusercontent.com/eficode/wait-for/v2.2.3/wait-for

cmd/publish/handler/handler.go

@@ -12,9 +12,8 @@ import (
 	"github.com/bytecamp-galaxy/mini-tiktok/kitex_gen/rpcmodel"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/conf"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/errno"
-	"github.com/bytecamp-galaxy/mini-tiktok/pkg/minio"
+	"github.com/bytecamp-galaxy/mini-tiktok/pkg/oss"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/snowflake"
-	"github.com/bytecamp-galaxy/mini-tiktok/pkg/utils"
 	"github.com/cloudwego/kitex/pkg/kerrors"
 	"github.com/google/uuid"
 	"net/http"
@@ -48,48 +47,57 @@ func (s *PublishServiceImpl) PublishVideo(ctx context.Context, req *publish.Publ
 
 	v := conf.Init()
 	videoBucketName := v.GetString("minio.video-bucket-name")
-	coverBucketName := v.GetString("minio.cover-bucket-name")
+	//coverBucketName := v.GetString("minio.cover-bucket-name")
 
 	// 上传视频
-	err = minio.UploadFile(videoBucketName, fileName, reader, int64(len(videoData)))
+	//err = minio.UploadFile(videoBucketName, fileName, reader, int64(len(videoData)))
+	//if err != nil {
+	//	return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
+	//}
+
+	err = oss.UploadFile(videoBucketName, fileName, reader)
 	if err != nil {
-		return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
+		return nil, kerrors.NewBizStatusError(int32(errno.ErrOSS), err.Error())
 	}
 
 	// 获取视频链接
-	playUrl, err := minio.GetFileUrl(videoBucketName, fileName, 0)
-	if err != nil {
-		return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
-	}
+	//playUrl, err := minio.GetFileUrl(videoBucketName, fileName, 0)
+	//if err != nil {
+	//	return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
+	//}
+
+	playUrl := oss.GetFileUrl(videoBucketName, fileName)
 
 	// 获取封面
-	coverUid := uuid.New()
-	coverPath := coverUid.String() + "." + "jpg"
-	coverData, err := utils.ReadFrameAsJpeg(playUrl.String())
-	if err != nil {
-		return nil, kerrors.NewBizStatusError(int32(errno.ErrEncodingFailed), err.Error())
-	}
+	//coverUid := uuid.New()
+	//coverPath := coverUid.String() + "." + "jpg"
+	//coverData, err := utils.ReadFrameAsJpeg(playUrl.String())
+	//if err != nil {
+	//	return nil, kerrors.NewBizStatusError(int32(errno.ErrEncodingFailed), err.Error())
+	//}
 
 	// 上传封面
-	coverReader := bytes.NewReader(coverData)
-	err = minio.UploadFile(coverBucketName, coverPath, coverReader, int64(len(coverData)))
-	if err != nil {
-		return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
-	}
+	//coverReader := bytes.NewReader(coverData)
+	//err = minio.UploadFile(coverBucketName, coverPath, coverReader, int64(len(coverData)))
+	//if err != nil {
+	//	return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
+	//}
 
 	// 获取封面链接
-	coverUrl, err := minio.GetFileUrl(coverBucketName, coverPath, 0)
-	if err != nil {
-		return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
-	}
+	//coverUrl, err := minio.GetFileUrl(coverBucketName, coverPath, 0)
+	//if err != nil {
+	//	return nil, kerrors.NewBizStatusError(int32(errno.ErrMinio), err.Error())
+	//}
+
+	coverUrl := oss.VideoSnapshot(playUrl)
 
 	// 封装 video
 	vid := snowflake.Generate()
 	video := &model.Video{
 		ID:       vid,
 		AuthorID: authorId,
-		PlayUrl:  playUrl.String(),
-		CoverUrl: coverUrl.String(),
+		PlayUrl:  playUrl,
+		CoverUrl: coverUrl,
 		Title:    videoTitle,
 	}
 

cmd/publish/main.go

@@ -9,8 +9,8 @@ import (
 	"github.com/bytecamp-galaxy/mini-tiktok/kitex_gen/publish/publishservice"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/conf"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/log"
-	"github.com/bytecamp-galaxy/mini-tiktok/pkg/minio"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/mw"
+	"github.com/bytecamp-galaxy/mini-tiktok/pkg/oss"
 	"github.com/bytecamp-galaxy/mini-tiktok/pkg/snowflake"
 	"github.com/cloudwego/kitex/pkg/limit"
 	"github.com/cloudwego/kitex/pkg/rpcinfo"
@@ -28,8 +28,8 @@ func main() {
 	log.SetOutput(v.GetString("publish-server.log-path"))
 	log.InitKLogger()
 
-	// init minio
-	minio.Init()
+	// init oss
+	oss.Init()
 
 	// init db
 	dal.Init(false)

configs/global.yaml

@@ -14,6 +14,13 @@ minio:
   video-bucket-name: "video"
   cover-bucket-name: "cover"
 
+aliyun-oss:
+  endpoint: oss-cn-nanjing.aliyuncs.com
+  ak: xxx
+  sk: xxx
+  video-bucket-name: "mini-tiktok-video"
+  cover-bucket-name: "mini-tiktok-cover"
+
 mysql:
   host: "127.0.0.1"
   port: 3306

docker-compose.yml

@@ -6,6 +6,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
         condition: service_healthy
@@ -16,6 +17,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
         condition: service_healthy
@@ -26,6 +28,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
         condition: service_healthy
@@ -36,6 +39,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
         condition: service_healthy
@@ -46,6 +50,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
         condition: service_healthy
@@ -56,6 +61,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
         condition: service_healthy
@@ -66,6 +72,7 @@ services:
     network_mode: host
     volumes:
       - ./logs:/root/logs
+      - ./configs:/root/configs
     depends_on:
       mysql:
        condition: service_healthy

go.mod

@@ -5,6 +5,7 @@ go 1.19
 replace github.com/apache/thrift => github.com/apache/thrift v0.13.0
 
 require (
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
 	github.com/apache/thrift v0.13.0
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/bytecamp-galaxy/jwt v0.0.1
@@ -42,6 +43,7 @@ require (
 	golang.org/x/crypto v0.5.0
 	golang.org/x/net v0.5.0
 	golang.org/x/tools v0.5.0
+	gopkg.in/vansante/go-ffprobe.v2 v2.1.1
 	gorm.io/driver/mysql v1.4.5
 	gorm.io/gen v0.3.19
 	gorm.io/gorm v1.24.3
@@ -163,6 +165,7 @@ require (
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/time v0.1.0 // indirect
 	google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
 	google.golang.org/grpc v1.52.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect

go.sum

@@ -61,6 +61,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible h1:KXeJoM1wo9I/6xPTyt6qCxoSZnmASiAjlrr0dyTUKt8=
+github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
@@ -1149,6 +1151,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA=
+golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1358,6 +1361,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/vansante/go-ffprobe.v2 v2.1.1 h1:DIh5fMn+tlBvG7pXyUZdemVmLdERnf2xX6XOFF+0BBU=
+gopkg.in/vansante/go-ffprobe.v2 v2.1.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/errno/base.go

@@ -42,6 +42,9 @@ const (
 	// ErrMinio - 500: Minio error.
 	ErrMinio
 
+	// ErrOSS - 500: OSS error.
+	ErrOSS
+
 	// ErrOpenFormFile - 500: Open request's form file error.
 	ErrOpenFormFile
 
@@ -51,6 +54,6 @@ const (
 	// ErrInvalidVideo - 400: Video does not exist.
 	ErrInvalidVideo
 
-	// ErrInvalidVideoType - 400: Uploaded video type unsupported. (.mp4 only)
+	// ErrInvalidVideoType - 400: Uploaded video type unsupported.
 	ErrInvalidVideoType
 )

pkg/errno/errno_generated.md

@@ -34,8 +34,9 @@ mini-tiktok 系统支持的错误码列表如下：
 | ErrRPCLink | 100011 | 500 | RPC service link error |
 | ErrEncodingFailed | 100012 | 500 | Encoding failed |
 | ErrMinio | 100013 | 500 | Minio error |
-| ErrOpenFormFile | 100014 | 500 | Open request's form file error |
-| ErrInvalidUser | 100015 | 400 | User does not exist |
-| ErrInvalidVideo | 100016 | 400 | Video does not exist |
-| ErrInvalidVideoType | 100017 | 400 | Uploaded video type unsupported. ( |
+| ErrOSS | 100014 | 500 | OSS error |
+| ErrOpenFormFile | 100015 | 500 | Open request's form file error |
+| ErrInvalidUser | 100016 | 400 | User does not exist |
+| ErrInvalidVideo | 100017 | 400 | Video does not exist |
+| ErrInvalidVideoType | 100018 | 400 | Uploaded video type unsupported |
 

pkg/minio/command.go

@@ -2,6 +2,7 @@ package minio
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/url"
 	"time"
@@ -32,6 +33,14 @@ func CreateBucket(bucketName string) error {
 	} else {
 		klog.Infof("bucket %s create successfully", bucketName)
 	}
+
+	policy := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, bucketName)
+	err = minioClient.SetBucketPolicy(context.Background(), bucketName, policy)
+	if err != nil {
+		klog.Debugf("%s", err.Error())
+		return err
+	}
+
 	return nil
 }
 

pkg/oss/command.go

@@ -0,0 +1,34 @@
+package oss
+
+import (
+	"errors"
+	"fmt"
+	"github.com/bytecamp-galaxy/mini-tiktok/pkg/conf"
+	"io"
+)
+
+func UploadFile(bucketName string, objectName string, reader io.Reader) (err error) {
+	v := conf.Init()
+	switch bucketName {
+	case v.GetString("aliyun-oss.video-bucket-name"):
+		err = videoBucket.PutObject(objectName, reader)
+	case v.GetString("aliyun-oss.cover-bucket-name"):
+		err = coverBucket.PutObject(objectName, reader)
+	default:
+		return errors.New("bucket name does not exist")
+	}
+	return
+}
+
+func GetFileUrl(bucketName string, objectName string) string {
+	v := conf.Init()
+	return fmt.Sprintf("https://%s.%s/%s",
+		bucketName,
+		v.GetString("endpoint"),
+		objectName)
+}
+
+func VideoSnapshot(playUrl string) string {
+	return fmt.Sprintf("%d?x-oss-process=video/snapshot,t_0,f_jpg,w_0,h_0,m_fast",
+		playUrl)
+}