Correctly handle non-image file when using remote backend (#382)

* Correctly handle non-image file when using remote backend

* Bump version

config/config.go

@@ -50,11 +50,12 @@ var (
 	DumpConfig          bool
 	ShowVersion         bool
 	ProxyMode           bool
+	AllowAllExtensions  bool
 	Prefetch            bool // Prefech in go-routine, with WebP Server Go launch normally
 	PrefetchForeground  bool // Standalone prefetch, prefetch and exit
 	AllowNonImage       bool
 	Config              = NewWebPConfig()
-	Version             = "0.13.2"
+	Version             = "0.13.3"
 	WriteLock           = cache.New(5*time.Minute, 10*time.Minute)
 	ConvertLock         = cache.New(5*time.Minute, 10*time.Minute)
 	LocalHostAlias      = "local"
@@ -300,6 +301,10 @@ func LoadConfig() {
 		}
 	}
 
+	if Config.AllowedTypes[0] == "*" {
+		AllowAllExtensions = true
+	}
+
 	log.Debugln("Config init complete")
 	log.Debugln("Config", Config)
 }

handler/remote.go

@@ -54,8 +54,8 @@ func downloadFile(filepath string, url string) {
 	// Check if remote content-type is image using check by filetype instead of content-type returned by origin
 	kind, _ := filetype.Match(bodyBytes.Bytes())
 	mime := kind.MIME.Value
-	if !strings.Contains(mime, "image") {
-		log.Errorf("remote file %s is not image, remote content has MIME type of %s", url, mime)
+	if !strings.Contains(mime, "image") && !config.AllowAllExtensions {
+		log.Errorf("remote file %s is not image and AllowedTypes is not '*', remote content has MIME type of %s", url, mime)
 		return
 	}
 
@@ -126,7 +126,7 @@ func pingURL(url string) string {
 	var etag, length string
 	resp, err := http.Head(url)
 	if err != nil {
-		log.Errorln("Connection to remote error when pingUrl!")
+		log.Errorln("Connection to remote error when pingUrl:"+url, err)
 		return ""
 	}
 	defer resp.Body.Close()

handler/router.go

@@ -70,13 +70,6 @@ func Convert(c *fiber.Ctx) error {
 		return nil
 	}
 
-	// Check if the file extension is allowed and not with image extension
-	// In this case we will serve the file directly
-	// Since here we've already sent non-image file, "raw" is not supported by default in the following code
-	if helper.CheckAllowedExtension(filename) && !helper.CheckImageExtension(filename) {
-		return c.SendFile(path.Join(config.Config.ImgPath, reqURI))
-	}
-
 	// Rewrite the target backend if a mapping rule matches the hostname
 	if hostMap, hostMapFound := config.Config.ImageMap[reqHost]; hostMapFound {
 		log.Debugf("Found host mapping %s -> %s", reqHostname, hostMap)
@@ -107,11 +100,9 @@ func Convert(c *fiber.Ctx) error {
 				break
 			}
 		}
-
 	}
 
 	if proxyMode {
-
 		if !mapMode {
 			// Don't deal with the encoding to avoid upstream compatibilities
 			reqURI = c.Path()
@@ -126,6 +117,19 @@ func Convert(c *fiber.Ctx) error {
 		log.Debugf("realRemoteAddr is %s", realRemoteAddr)
 	}
 
+	// Check if the file extension is allowed and not with image extension
+	// In this case we will serve the file directly
+	// Since here we've already sent non-image file, "raw" is not supported by default in the following code
+	if config.AllowAllExtensions && !helper.CheckImageExtension(filename) {
+		// If the file is not in the ImgPath, we'll have to use the proxy mode to download it
+		if !proxyMode {
+			return c.SendFile(path.Join(config.Config.ImgPath, reqURI))
+		} else {
+			fetchRemoteImg(realRemoteAddr, targetHostName)
+			return c.SendFile(path.Join(config.Config.RemoteRawPath, targetHostName, helper.HashString(realRemoteAddr)))
+		}
+	}
+
 	var rawImageAbs string
 	var metadata = config.MetaFile{}
 	if proxyMode {

handler/router_test.go

@@ -215,11 +215,12 @@ func TestConvertNotAllowed(t *testing.T) {
 func TestConvertPassThrough(t *testing.T) {
 	setupParam()
 	config.Config.AllowedTypes = []string{"*"}
+	config.AllowAllExtensions = true
 
 	var app = fiber.New()
 	app.Get("/*", Convert)
 
-	// not allowed, but we have the file, this should return File extension not allowed
+	// Since AllowedTypes is *, we should be able to access config.json
 	url := "http://127.0.0.1:3333/config.json"
 	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
 	defer resp.Body.Close()
@@ -228,6 +229,28 @@ func TestConvertPassThrough(t *testing.T) {
 	assert.Contains(t, string(data), "HOST")
 }
 
+func TestConvertPassThroughWithRemoteBackend(t *testing.T) {
+	setupParam()
+	config.Config.AllowedTypes = []string{"*"}
+	config.Config.ImgPath = "https://docs.webp.sh"
+	config.ProxyMode = true
+	config.AllowAllExtensions = true
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	// This should send request to https://docs.webp.sh/sw.js and render this file
+	url := "http://127.0.0.1:3333/sw.js"
+	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// TODO: No idea why this is not working, it shows text/html instead of application/javascript
+	// assert.Equal(t, "application/javascript", resp.Header.Get("Content-Type"))
+
+	assert.Contains(t, string(data), "addEventListener")
+}
+
 func TestConvertProxyModeBad(t *testing.T) {
 	setupParam()
 	config.ProxyMode = true