Non iconv conversion of non utf-8 encoded pages now works. Fixed other encoding issues and various Content-Type header problems. ParseContentType now turns everything into lowercase.

Author: pietroglyph

init.go

@@ -83,22 +83,31 @@ func (fn reqHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Allo
 	if e := fn(w, r); e != nil { // e is *appError, not os.Error
 		if e.Code == 404 { // Serve a pretty (potentially cached) file for 404 errors, if it exists
 			w.WriteHeader(404)
-			if Config.Verbose {
+			if Config.Verbose && e.Error != nil {
 				fmt.Println(e.Error.Error(), "\n", e.Message) // Print the error message
 			}
 			if FileCache["404"] != nil { // Serve the cached file if one exists
 				io.WriteString(w, string(FileCache["404"]))
 			} else { // Read a non-cached file from disk and serve it because there isn't a cached one
 				file, err := ioutil.ReadFile(Config.PublicDir + "/404.html")
 				if err != nil {
-					http.Error(w, e.Message+"\n"+e.Error.Error(), e.Code) // Serve a generic error message if the file isn't cahced and doesn't exist
+					if e.Error == nil {
+						http.Error(w, e.Message, e.Code) // Serve a generic error message if the file isn't cached and doesn't exist
+					} else {
+						http.Error(w, e.Message+"\n"+e.Error.Error(), e.Code) // Serve a generic error message if the file isn't cached and doesn't exist
+					}
 					return
 				}
 				io.WriteString(w, string(file))
 			}
 		} else { // If it's not a 404 error just serve a generic message
-			fmt.Println(e.Error.Error(), "\n", e.Message) // Print the error message
-			http.Error(w, e.Message+"\n"+e.Error.Error(), e.Code)
+			if e.Error == nil {
+				fmt.Println(e.Message)
+				http.Error(w, e.Message, e.Code) // Serve a generic error message if the file isn't cached and doesn't exist
+			} else {
+				fmt.Println(e.Error.Error(), "\n", e.Message)         // Print the error message
+				http.Error(w, e.Message+"\n"+e.Error.Error(), e.Code) // Serve a generic error message if the file isn't cached and doesn't exist
+			}
 		}
 
 	}

proxy.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	goenc "github.com/mattn/go-encoding"
+	goenc "github.com/pietroglyph/go-encoding"
 	// "golang.org/x/text/encoding"
 	// "golang.org/x/text/transform"
 )
@@ -86,8 +86,6 @@ func proxy(resWriter http.ResponseWriter, reqHttp *http.Request) *reqError { //
 		return &reqError{err, "Couldn't read returned body.", 500}
 	}
 
-	copyHeaders(resWriter.Header(), httpCliResp.Header) // Copy headers from response to our request from the server we are proxying
-
 	prox.ConType, err = parseContentType(httpCliResp.Header.Get("Content-Type")) // Get the MIME type of what we recieved from the Content-Type header
 	if err != nil {
 		prox.ConType, err = parseContentType(http.DetectContentType(prox.Body)) // Looks like we couldn't parse the Content-Type header, so we'll have to detect content type from the actual response body
@@ -96,7 +94,23 @@ func proxy(resWriter http.ResponseWriter, reqHttp *http.Request) *reqError { //
 		}
 	}
 
-	if prox.ConType.Parameters["charset"] == "" {
+	for header := range httpCliResp.Header { // Copy over headers from the http response to our http response writer
+		if !Config.DisableCORS {
+			if header == "Content-Type" && prox.ConType.Type == "text" && prox.ConType.Subtype == "html" {
+				resWriter.Header().Add(header, "text/html; charset=utf-8")
+			} else {
+				resWriter.Header().Add(header, httpCliResp.Header.Get(header))
+			}
+		} else if header != "Content-Security-Policy" && header != "Content-Type" {
+			resWriter.Header().Add(header, httpCliResp.Header.Get(header))
+		} else if header == "Content-Type" && prox.ConType.Type == "text" && prox.ConType.Subtype == "html" {
+			resWriter.Header().Add(header, "text/html; charset=utf-8")
+		} else {
+			resWriter.Header().Add(header, httpCliResp.Header.Get(header))
+		}
+	} // TODO: This conditonal chain is a nightmare and should be fixed sometime
+
+	if prox.ConType.Parameters["charset"] == "" { // Make sure that we have a charset if the website doesn't provide one (which is fairly common)
 		tempConType, err := parseContentType(http.DetectContentType(prox.Body))
 		if err != nil {
 			fmt.Println(err.Error()) // Instead of failing we will just give the user a non-formatted page and print the error
@@ -108,18 +122,23 @@ func proxy(resWriter http.ResponseWriter, reqHttp *http.Request) *reqError { //
 	if prox.ConType.Type == "text" && prox.ConType.Subtype == "html" && prox.ConType.Parameters["charset"] != "" { // Does it say it's html with a valid charset
 		resReader := strings.NewReader(string(prox.Body))
 		if prox.ConType.Parameters["charset"] != "utf-8" {
-			decoder := goenc.GetEncoding(prox.ConType.Parameters["charset"]).NewDecoder()
-			prox.Document, err = goquery.NewDocumentFromReader(decoder.Reader(resReader)) // Parse the response from our target website
+			encoding := goenc.GetEncoding(prox.ConType.Parameters["charset"])
+			if encoding == nil {
+				return &reqError{nil, prox.ConType.Parameters["charset"] + " is an invalid encoding.", 400}
+			}
+			fmt.Println(encoding, prox.ConType.Parameters["charset"])
+			decoder := encoding.NewDecoder()
+			prox.Document, err = goquery.NewDocumentFromReader(decoder.Reader(resReader)) // Parse the response from our target website whose body has been freshly utf-8 encoded
 			if err != nil {                                                               // Looks like we can't parse this, let's just spit out the raw response
 				fmt.Fprint(resWriter, string(prox.Body))
-				fmt.Println(err.Error())
+				fmt.Println(err.Error(), prox.Url)
 				return nil
 			}
 		} else {
 			prox.Document, err = goquery.NewDocumentFromReader(resReader)
 			if err != nil { // Looks like we can't parse this, let's just spit out the raw response
 				fmt.Fprint(resWriter, string(prox.Body))
-				fmt.Println(err.Error())
+				fmt.Println(err.Error(), prox.Url)
 				return nil
 			}
 		}

util.go

@@ -7,7 +7,6 @@ package main
 import (
 	"encoding/base64"
 	"errors"
-	"net/http"
 	"net/url"
 	"strings"
 )
@@ -19,12 +18,14 @@ type ContentType struct { // The ContentType type holds easily usable informatio
 }
 
 func parseContentType(rawcontype string) (*ContentType, error) { // Parse a MIME string into a ContentType struct
+	rawcontype = strings.ToLower(rawcontype)
 	var contentType ContentType
 	contentType.Parameters = make(map[string]string)
 	contype := strings.Split(rawcontype, " ")
 	contype[0] = strings.Replace(contype[0], ";", "", -1)
 	mimetype := strings.Split(contype[0], "/")
 	if len(mimetype) <= 1 {
+
 		return new(ContentType), errors.New("contype: malformed content-type MIME type provided")
 	}
 	if len(contype) > 1 {
@@ -65,15 +66,3 @@ func formatUri(rawurl string, host string, proxyhost string) (string, error) { /
 	formattedurl := "http://" + proxyhost + "/p/?u=" + encodedurl
 	return formattedurl, nil
 }
-
-func copyHeaders(dest http.Header, source http.Header) { // Copy one http.Header to another http.Header
-	for header := range source {
-		if !Config.DisableCORS {
-			dest.Add(header, source.Get(header))
-		} else if header != "Content-Security-Policy" && header != "Content-Type" {
-			dest.Add(header, source.Get(header))
-		} else if header == "Content-Type" {
-			dest.Add(header, "text/html; charset=utf-8")
-		}
-	}
-}