Partial fix for bugs

config/config.go

@@ -55,7 +55,7 @@ var (
 	PrefetchForeground  bool // Standalone prefetch, prefetch and exit
 	AllowNonImage       bool
 	Config              = NewWebPConfig()
-	Version             = "0.13.3"
+	Version             = "0.13.4"
 	WriteLock           = cache.New(5*time.Minute, 10*time.Minute)
 	ConvertLock         = cache.New(5*time.Minute, 10*time.Minute)
 	LocalHostAlias      = "local"

handler/remote.go

@@ -31,32 +31,33 @@ func cleanProxyCache(cacheImagePath string) {
 	}
 }
 
-func downloadFile(filepath string, url string) {
+// Download file and return response header
+func downloadFile(filepath string, url string) http.Header {
 	resp, err := http.Get(url)
 	if err != nil {
 		log.Errorln("Connection to remote error when downloadFile!")
-		return
+		return nil
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != fiber.StatusOK {
 		log.Errorf("remote returned %s when fetching remote image", resp.Status)
-		return
+		return resp.Header
 	}
 
 	// Copy bytes here
 	bodyBytes := new(bytes.Buffer)
 	_, err = bodyBytes.ReadFrom(resp.Body)
 	if err != nil {
-		return
+		return nil
 	}
 
 	// Check if remote content-type is image using check by filetype instead of content-type returned by origin
 	kind, _ := filetype.Match(bodyBytes.Bytes())
 	mime := kind.MIME.Value
 	if !strings.Contains(mime, "image") && !config.AllowAllExtensions {
 		log.Errorf("remote file %s is not image and AllowedTypes is not '*', remote content has MIME type of %s", url, mime)
-		return
+		return nil
 	}
 
 	_ = os.MkdirAll(path.Dir(filepath), 0755)
@@ -68,15 +69,16 @@ func downloadFile(filepath string, url string) {
 	err = os.WriteFile(filepath, bodyBytes.Bytes(), 0600)
 	if err != nil {
 		// not likely to happen
-		return
+		return nil
 	}
 
 	// Delete lock here
 	config.WriteLock.Delete(filepath)
 
+	return resp.Header
 }
 
-func fetchRemoteImg(url string, subdir string) config.MetaFile {
+func fetchRemoteImg(url string, subdir string) (metaContent config.MetaFile) {
 	// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 	// How do we know if the remote img is changed? we're using hash(etag+length)
 	var etag string
@@ -101,7 +103,8 @@ func fetchRemoteImg(url string, subdir string) config.MetaFile {
 	}
 
 	metadata := helper.ReadMetadata(url, etag, subdir)
-	localRawImagePath := path.Join(config.Config.RemoteRawPath, subdir, metadata.Id)
+	remoteFileExtension := path.Ext(url)
+	localRawImagePath := path.Join(config.Config.RemoteRawPath, subdir, metadata.Id) + remoteFileExtension
 	localExhaustImagePath := path.Join(config.Config.ExhaustPath, subdir, metadata.Id)
 
 	if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
@@ -115,7 +118,9 @@ func fetchRemoteImg(url string, subdir string) config.MetaFile {
 			// local file not exists
 			log.Info("Remote file not found in remote-raw, re-fetching...")
 		}
-		downloadFile(localRawImagePath, url)
+		_ = downloadFile(localRawImagePath, url)
+		// Update metadata with newly downloaded file
+		helper.WriteMetadata(url, etag, subdir)
 	}
 	return metadata
 }

handler/router.go

@@ -113,20 +113,21 @@ func Convert(c *fiber.Ctx) error {
 
 		// Replace host in the URL
 		// realRemoteAddr = strings.Replace(reqURIwithQuery, reqHost, targetHost, 1)
-		realRemoteAddr = targetHost + reqURIwithQuery
+		realRemoteAddr, _ = url.JoinPath(targetHost, reqURIwithQuery)
 		log.Debugf("realRemoteAddr is %s", realRemoteAddr)
 	}
 
 	// Check if the file extension is allowed and not with image extension
 	// In this case we will serve the file directly
 	// Since here we've already sent non-image file, "raw" is not supported by default in the following code
 	if config.AllowAllExtensions && !helper.CheckImageExtension(filename) {
-		// If the file is not in the ImgPath, we'll have to use the proxy mode to download it
 		if !proxyMode {
 			return c.SendFile(path.Join(config.Config.ImgPath, reqURI))
 		} else {
-			fetchRemoteImg(realRemoteAddr, targetHostName)
-			return c.SendFile(path.Join(config.Config.RemoteRawPath, targetHostName, helper.HashString(realRemoteAddr)))
+			// If the file is not in the ImgPath, we'll have to use the proxy mode to download it
+			_ = fetchRemoteImg(realRemoteAddr, targetHostName)
+			localFilename := path.Join(config.Config.RemoteRawPath, targetHostName, helper.HashString(realRemoteAddr)) + path.Ext(realRemoteAddr)
+			return c.SendFile(localFilename)
 		}
 	}
 
@@ -137,7 +138,7 @@ func Convert(c *fiber.Ctx) error {
 		// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 
 		metadata = fetchRemoteImg(realRemoteAddr, targetHostName)
-		rawImageAbs = path.Join(config.Config.RemoteRawPath, targetHostName, metadata.Id)
+		rawImageAbs = path.Join(config.Config.RemoteRawPath, targetHostName, metadata.Id) + path.Ext(realRemoteAddr)
 	} else {
 		// not proxyMode, we'll use local path
 		metadata = helper.ReadMetadata(reqURIwithQuery, "", targetHostName)

handler/router_test.go

@@ -293,6 +293,48 @@ func TestConvertProxyModeWork(t *testing.T) {
 	assert.Equal(t, "image/jpeg", helper.GetContentType(data))
 }
 
+func TestConvertProxyModeNonImageWork(t *testing.T) {
+	setupParam()
+	config.ProxyMode = true
+	config.Config.AllowedTypes = []string{"*"}
+	config.AllowAllExtensions = true
+	config.Config.ImgPath = "https://docs.webp.sh"
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	url := "http://127.0.0.1:3333/sw.js"
+
+	resp, _ := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "text/javascript; charset=utf-8", resp.Header.Get("Content-Type"))
+}
+
+func TestConvertMapProxyModeWork(t *testing.T) {
+	setupParam()
+	config.ProxyMode = true
+	config.Config.ImageMap = map[string]string{
+		"/": "https://docs.webp.sh",
+	}
+
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	url := "http://127.0.0.1:3333/images/webp_server.jpg"
+
+	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "image/webp", helper.GetContentType(data))
+
+	// test proxyMode with Safari
+	resp, data = requestToServer(url, app, safariUA, acceptLegacy)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", helper.GetContentType(data))
+}
+
 func TestConvertProxyImgMap(t *testing.T) {
 	setupParam()
 	config.ProxyMode = false
@@ -379,7 +421,7 @@ func TestConvertProxyImgMapCWD(t *testing.T) {
 	}
 }
 
-func TestConvertBigger(t *testing.T) {
+func TestConvertedFileIsBigger(t *testing.T) {
 	setupParam()
 	config.Config.Quality = 100
 

helper/metadata.go

@@ -12,10 +12,12 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func getId(p string) (string, string, string) {
-	var id string
+// Get ID and filepath
+// For ProxyMode, pass in p the remote-raw path
+func getId(p string, subdir string) (id string, filePath string, santizedPath string) {
 	if config.ProxyMode {
-		return HashString(p), "", ""
+		fileID := HashString(p)
+		return fileID, path.Join(config.Config.RemoteRawPath, subdir, fileID) + path.Ext(p), ""
 	}
 	parsed, _ := url.Parse(p)
 	width := parsed.Query().Get("width")
@@ -24,16 +26,17 @@ func getId(p string) (string, string, string) {
 	max_height := parsed.Query().Get("max_height")
 	// santizedPath will be /webp_server.jpg?width=200\u0026height=\u0026max_width=\u0026max_height= in local mode when requesting /webp_server.jpg?width=200
 	// santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh
-	santizedPath := parsed.Path + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height
+	santizedPath = parsed.Path + "?width=" + width + "&height=" + height + "&max_width=" + max_width + "&max_height=" + max_height
 	id = HashString(santizedPath)
+	filePath = path.Join(config.Config.ImgPath, parsed.Path)
 
-	return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath
+	return id, filePath, santizedPath
 }
 
 func ReadMetadata(p, etag string, subdir string) config.MetaFile {
 	// try to read metadata, if we can't read, create one
 	var metadata config.MetaFile
-	var id, _, _ = getId(p)
+	var id, _, _ = getId(p, subdir)
 
 	if buf, err := os.ReadFile(path.Join(config.Config.MetadataPath, subdir, id+".json")); err != nil {
 		// First time reading metadata, create one
@@ -53,7 +56,7 @@ func ReadMetadata(p, etag string, subdir string) config.MetaFile {
 func WriteMetadata(p, etag string, subdir string) config.MetaFile {
 	_ = os.MkdirAll(path.Join(config.Config.MetadataPath, subdir), 0755)
 
-	var id, filepath, sant = getId(p)
+	var id, filepath, sant = getId(p, subdir)
 
 	var data = config.MetaFile{
 		Id: id,
@@ -68,6 +71,7 @@ func WriteMetadata(p, etag string, subdir string) config.MetaFile {
 	}
 
 	imageMeta := getImageMeta(filepath)
+
 	data.ImageMeta = imageMeta
 
 	buf, _ := json.Marshal(data)
@@ -166,7 +170,7 @@ func getImageMeta(filePath string) (metadata config.ImageMeta) {
 }
 
 func DeleteMetadata(p string, subdir string) {
-	var id, _, _ = getId(p)
+	var id, _, _ = getId(p, subdir)
 	metadataPath := path.Join(config.Config.MetadataPath, subdir, id+".json")
 	err := os.Remove(metadataPath)
 	if err != nil {

helper/metadata_test.go

@@ -13,11 +13,11 @@ func TestGetId(t *testing.T) {
 	t.Run("proxy mode", func(t *testing.T) {
 		// Test case 1: Proxy mode
 		config.ProxyMode = true
-		id, jointPath, santizedPath := getId(p)
+		id, jointPath, santizedPath := getId(p, "")
 
 		// Verify the return values
 		expectedId := HashString(p)
-		expectedPath := ""
+		expectedPath := "remote-raw/8d8576343c4cb816.jpg?width=200&height=300"
 		expectedSantizedPath := ""
 		if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath {
 			t.Errorf("Test case 1 failed: Expected (%s, %s, %s), but got (%s, %s, %s)",
@@ -28,7 +28,7 @@ func TestGetId(t *testing.T) {
 		// Test case 2: Non-proxy mode
 		config.ProxyMode = false
 		p = "/image.jpg?width=400&height=500"
-		id, jointPath, santizedPath := getId(p)
+		id, jointPath, santizedPath := getId(p, "")
 
 		// Verify the return values
 		parsed, _ := url.Parse(p)