Skip to content

Commit

Permalink
WIP JXL
Browse files Browse the repository at this point in the history
  • Loading branch information
@n0vad3v
n0vad3v committed Mar 11, 2024
1 parent ddcb532 commit cb5192e
Show file tree
Hide file tree
Showing 12 changed files with 311 additions and 60 deletions.
95 changes: 63 additions & 32 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"regexp"
"runtime"
"slices"
"strconv"
"strings"
"time"
Expand All @@ -29,7 +30,7 @@ const (
"EXHAUST_PATH": "./exhaust",
"IMG_MAP": {},
"ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg","heic","nef"],
"ENABLE_AVIF": false,
"CONVERT_TYPES": ["webp","avif","jxl"],
"ENABLE_EXTRA_PARAMS": false
"READ_BUFFER_SIZE": 4096,
"CONCURRENCY": 262144,
Expand All @@ -47,7 +48,7 @@ var (
ProxyMode bool
Prefetch bool
Config = NewWebPConfig()
Version = "0.10.8"
Version = "0.11.0"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
ConvertLock = cache.New(5*time.Minute, 10*time.Minute)
RemoteRaw = "./remote-raw"
Expand All @@ -63,31 +64,41 @@ type MetaFile struct {
}

type WebpConfig struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ImageMap map[string]string `json:"IMG_MAP"`
ExhaustPath string `json:"EXHAUST_PATH"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
ReadBufferSize int `json:"READ_BUFFER_SIZE"`
Concurrency int `json:"CONCURRENCY"`
DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
CacheTTL int `json:"CACHE_TTL"`
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ConvertTypes []string `json:"CONVERT_TYPES"`
ImageMap map[string]string `json:"IMG_MAP"`
ExhaustPath string `json:"EXHAUST_PATH"`

EnableWebP bool `json:"ENABLE_WEBP"`
EnableAVIF bool `json:"ENABLE_AVIF"`
EnableJXL bool `json:"ENABLE_JXL"`

EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
ReadBufferSize int `json:"READ_BUFFER_SIZE"`
Concurrency int `json:"CONCURRENCY"`
DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
CacheTTL int `json:"CACHE_TTL"`
}

func NewWebPConfig() *WebpConfig {
return &WebpConfig{
Host: "0.0.0.0",
Port: "3333",
ImgPath: "./pics",
Quality: 80,
AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "svg", "nef", "heic", "webp"},
ImageMap: map[string]string{},
ExhaustPath: "./exhaust",
EnableAVIF: false,
Host: "0.0.0.0",
Port: "3333",
ImgPath: "./pics",
Quality: 80,
AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "svg", "nef", "heic", "webp"},
ConvertTypes: []string{"webp"},
ImageMap: map[string]string{},
ExhaustPath: "./exhaust",

EnableWebP: false,
EnableAVIF: false,
EnableJXL: false,

EnableExtraParams: false,
ReadBufferSize: 4096,
Concurrency: 262144,
Expand Down Expand Up @@ -115,6 +126,16 @@ func LoadConfig() {
switchProxyMode()
Config.ImageMap = parseImgMap(Config.ImageMap)

if slices.Contains(Config.ConvertTypes, "webp") {
Config.EnableWebP = true
}
if slices.Contains(Config.ConvertTypes, "avif") {
Config.EnableAVIF = true
}
if slices.Contains(Config.ConvertTypes, "jxl") {
Config.EnableJXL = true
}

// Read from ENV for override
if os.Getenv("WEBP_HOST") != "" {
Config.Host = os.Getenv("WEBP_HOST")
Expand All @@ -139,16 +160,24 @@ func LoadConfig() {
if os.Getenv("WEBP_ALLOWED_TYPES") != "" {
Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",")
}
if os.Getenv("WEBP_ENABLE_AVIF") != "" {
enableAVIF := os.Getenv("WEBP_ENABLE_AVIF")
if enableAVIF == "true" {

// Override enabled convert types
if os.Getenv("WEBP_CONVERT_TYPES") != "" {
Config.ConvertTypes = strings.Split(os.Getenv("WEBP_CONVERT_TYPES"), ",")
Config.EnableWebP = false
Config.EnableAVIF = false
Config.EnableJXL = false
if slices.Contains(Config.ConvertTypes, "webp") {
Config.EnableWebP = true
}
if slices.Contains(Config.ConvertTypes, "avif") {
Config.EnableAVIF = true
} else if enableAVIF == "false" {
Config.EnableAVIF = false
} else {
log.Warnf("WEBP_ENABLE_AVIF is not a valid boolean, using value in config.json %t", Config.EnableAVIF)
}
if slices.Contains(Config.ConvertTypes, "jxl") {
Config.EnableJXL = true
}
}

if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" {
enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS")
if enableExtraParams == "true" {
Expand Down Expand Up @@ -223,8 +252,10 @@ func parseImgMap(imgMap map[string]string) map[string]string {
}

type ExtraParams struct {
Width int // in px
Height int // in px
Width int // in px
Height int // in px
MaxWidth int // in px
MaxHeight int // in px
}

func switchProxyMode() {
Expand Down
56 changes: 53 additions & 3 deletions encoder/encoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func init() {
intMinusOne.Set(-1)
}

func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) {
func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) {
// Wait for the conversion to complete and return the converted image
retryDelay := 100 * time.Millisecond // Initial retry delay

Expand All @@ -53,7 +53,7 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP
defer config.ConvertLock.Delete(rawPath)

var wg sync.WaitGroup
wg.Add(2)
wg.Add(3)
if !helper.ImageExists(avifPath) && config.Config.EnableAVIF {
go func() {
err := convertImage(rawPath, avifPath, "avif", extraParams)
Expand All @@ -66,7 +66,7 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP
wg.Done()
}

if !helper.ImageExists(webpPath) {
if !helper.ImageExists(webpPath) && config.Config.EnableWebP {
go func() {
err := convertImage(rawPath, webpPath, "webp", extraParams)
if err != nil {
Expand All @@ -77,6 +77,19 @@ func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraP
} else {
wg.Done()
}

if !helper.ImageExists(jxlPath) && config.Config.EnableJXL {
go func() {
err := convertImage(rawPath, jxlPath, "jxl", extraParams)
if err != nil {
log.Errorln(err)
}
defer wg.Done()
}()
} else {
wg.Done()
}

wg.Wait()

if c != nil {
Expand Down Expand Up @@ -123,11 +136,48 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
err = webpEncoder(img, rawPath, optimizedPath)
case "avif":
err = avifEncoder(img, rawPath, optimizedPath)
case "jxl":
err = jxlEncoder(img, rawPath, optimizedPath)
}

return err
}

func jxlEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error {
var (
buf []byte
quality = config.Config.Quality
err error
)

// If quality >= 100, we use lossless mode
if quality >= 100 {
buf, _, err = img.ExportJxl(&vips.JxlExportParams{
Effort: 2,
Lossless: true,
})
} else {
buf, _, err = img.ExportJxl(&vips.JxlExportParams{
Effort: 2,
Quality: quality,
Lossless: false,
})
}

if err != nil {
log.Warnf("Can't encode source image: %v to JXL", err)
return err
}

if err := os.WriteFile(optimizedPath, buf, 0600); err != nil {
log.Error(err)
return err
}

convertLog("JXL", rawPath, optimizedPath, quality)
return nil
}

func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string) error {
var (
buf []byte
Expand Down
6 changes: 3 additions & 3 deletions encoder/prefetch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ func PrefetchImages() {
}
// RawImagePath string, ImgFilename string, reqURI string
metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias)
avif, webp := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)
_ = os.MkdirAll(path.Dir(avif), 0755)
avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)
_ = os.MkdirAll(path.Dir(avifAbsPath), 0755)
log.Infof("Prefetching %s", picAbsPath)
go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)
go ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, finishChan)
_ = bar.Add(<-finishChan)
return nil
})
Expand Down
57 changes: 54 additions & 3 deletions encoder/process.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,69 @@ import (
)

func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
imageHeight := img.Height()
imageWidth := img.Width()

imgHeightWidthRatio := float32(imageHeight) / float32(imageWidth)

// Here we have width, height and max_width, max_height
// Both pairs cannot be used at the same time

// max_height and max_width are used to make sure bigger images are resized to max_height and max_width
// e.g, 500x500px image with max_width=200,max_height=100 will be resized to 100x100
// while smaller images are untouched

// If both are used, we will use width and height

if extraParams.MaxHeight > 0 && extraParams.MaxWidth > 0 {
// If any of it exceeds
if imageHeight > extraParams.MaxHeight || imageWidth > extraParams.MaxWidth {
// Check which dimension exceeds most
heightExceedRatio := float32(imageHeight) / float32(extraParams.MaxHeight)
widthExceedRatio := float32(imageWidth) / float32(extraParams.MaxWidth)
// If height exceeds more, like 500x500 -> 200x100 (2.5 < 5)
// Take max_height as new height ,resize and retain ratio
if heightExceedRatio > widthExceedRatio {
err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0)
if err != nil {
return err
}
} else {
err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
}
}
}

if extraParams.MaxHeight > 0 && imageHeight > extraParams.MaxHeight && extraParams.MaxWidth == 0 {
err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0)
if err != nil {
return err
}
}

if extraParams.MaxWidth > 0 && imageWidth > extraParams.MaxWidth && extraParams.MaxHeight == 0 {
err := img.Thumbnail(extraParams.MaxWidth, int(float32(extraParams.MaxWidth)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
}

if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention)
if err != nil {
return err
}
} else if extraParams.Width > 0 && extraParams.Height == 0 {
}
if extraParams.Width > 0 && extraParams.Height == 0 {
err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
} else if extraParams.Height > 0 && extraParams.Width == 0 {
}
if extraParams.Height > 0 && extraParams.Width == 0 {
err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0)
if err != nil {
return err
Expand Down
Loading

0 comments on commit cb5192e

Please sign in to comment.