Initial implementation of go-translate

Makefile

@@ -0,0 +1,15 @@
+.PHONY: all build test lint clean
+
+all: lint test build
+
+build:
+	go run main.go
+
+test:
+	go test -v ./...
+
+lint:
+	golangci-lint run -E gofmt -E golint --exclude-use-default=false
+
+clean:
+	rm -f go-translate

README.md

@@ -0,0 +1,40 @@
+# Translation relay server for brave
+
+`go-translate` implements a translation relay server for use in brave-core written in Go.
+
+The intended audience for this server is all users of brave-core.
+
+The translation relay server supports 2 endpoints
+
+1) The `POST /translate` endpoint processes translate requests in Google format, sends corresponding requests in Microsoft format to Microsoft translate server, then returns responses in Google format back to the brave-core client.
+
+2) The `GET /language` endpoint processes requests of getting the support language list in Google format, sends corresponding requests in Microsoft format to Microsoft translate server, then returns responses in Google format back to the brave-core client.
+
+There are also a few static resources requested during in-page translation will be handled by go-translate and will be proxied through a brave to avoid introducing direct connection to any Google server.
+
+
+## Dependencies
+
+- Install Go 1.12 or later.
+- Dependencies are managed by go modules.
+- `go get -u github.com/golangci/golangci-lint/cmd/golangci-lint`
+
+## Setup
+
+```
+git clone [email protected]:brave/go-translate.git
+cd ~/path-to/go-translate
+make build
+```
+
+## Run lint:
+
+`make lint`
+
+## Run tests:
+
+`make test`
+
+## Build and run go-translate:
+
+`make build`

controller/controller.go

@@ -0,0 +1,200 @@
+package controller
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"time"
+
+	"github.com/brave/go-translate/language"
+	"github.com/brave/go-translate/translate"
+	"github.com/go-chi/chi"
+	log "github.com/sirupsen/logrus"
+)
+
+// MSTranslateServer specifies the remote MS translate server used by
+// brave-core, and it can be set to a mock server during testing.
+var MSTranslateServer = "https://api.cognitive.microsofttranslator.com"
+
+// GoogleTranslateServerProxy specifies the proxy server for requesting
+// resource from google translate server, and it can be set to a mock server
+// during testing.
+var GoogleTranslateServerProxy = "https://translate.brave.com"
+
+const (
+	// GoogleTranslateServer specifies the remote google translate server.
+	GoogleTranslateServer = "https://translate.googleapis.com"
+
+	// GStaticServerProxy specifies the proxy server for requesting resource
+	// from google gstatic server.
+	GStaticServerProxy = "https://translate-static.brave.com"
+
+	languageEndpoint = "/languages?api-version=3.0&scope=translation"
+)
+
+// TranslateRouter add routers for translate requests and translate script
+// requests.
+func TranslateRouter() chi.Router {
+	r := chi.NewRouter()
+
+	r.Post("/translate", Translate)
+	r.Get("/language", GetLanguageList)
+
+	r.Get("/translate_a/element.js", GetTranslateScript)
+	r.Get("/element/*/js/element/element_main.js", GetTranslateScript)
+	r.Get("/translate_static/js/element/main.js", GetTranslateScript)
+
+	r.Get("/translate_static/css/translateelement.css", GetGoogleTranslateResource)
+	r.Get("/images/branding/product/1x/translate_24dp.png", GetGStaticResource)
+	r.Get("/images/branding/product/2x/translate_24dp.png", GetGStaticResource)
+
+	return r
+}
+
+func getHTTPClient() *http.Client {
+	return &http.Client{
+		Timeout: time.Second * 10,
+	}
+}
+
+// GetLanguageList send a request to Microsoft server and convert the response
+// into google format and reply back to the client.
+func GetLanguageList(w http.ResponseWriter, r *http.Request) {
+	// Send a get language list request to MS
+	req, err := http.NewRequest("GET", MSTranslateServer+languageEndpoint, nil)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error creating MS request: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	client := getHTTPClient()
+	msResp, err := client.Do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error sending request to MS server: %v", err), http.StatusInternalServerError)
+		return
+	}
+	defer func() {
+		err := msResp.Body.Close()
+		if err != nil {
+			log.Errorf("Error closing response body stream: %v", err)
+		}
+	}()
+
+	// Set response header
+	w.Header().Set("Content-Type", msResp.Header["Content-Type"][0])
+	w.WriteHeader(msResp.StatusCode)
+
+	// Copy resonse body if status is not OK
+	if msResp.StatusCode != http.StatusOK {
+		_, err = io.Copy(w, msResp.Body)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Error copying MS response body: %v", err), http.StatusInternalServerError)
+		}
+		return
+	}
+
+	// Convert to google format language list and write it back
+	msBody, err := ioutil.ReadAll(msResp.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error reading MS response body: %v", err), http.StatusInternalServerError)
+		return
+	}
+	body, err := language.ToGoogleLanguageList(msBody)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error converting to google language list: %v", err), http.StatusInternalServerError)
+		return
+	}
+	_, err = w.Write(body)
+	if err != nil {
+		log.Errorf("Error writing response body for translate requests: %v", err)
+	}
+}
+
+// Translate converts a Google format translate request into a Microsoft format
+// one which will be send to the Microsoft server, and write a Google format
+// response back to the client.
+func Translate(w http.ResponseWriter, r *http.Request) {
+	// Convert google format request to MS format
+	req, isAuto, err := translate.ToMicrosoftRequest(r, MSTranslateServer)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error converting to MS request: %v", err), http.StatusBadRequest)
+		return
+	}
+
+	// Send translate request to MS server
+	client := getHTTPClient()
+	msResp, err := client.Do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error sending request to MS server: %v", err), http.StatusInternalServerError)
+		return
+	}
+	defer func() {
+		err := msResp.Body.Close()
+		if err != nil {
+			log.Errorf("Error closing response body stream: %v", err)
+		}
+	}()
+
+	// Set Header
+	w.Header().Set("Content-Type", msResp.Header["Content-Type"][0])
+	w.Header().Set("Access-Control-Allow-Origin", "*") // same as Google response
+
+	// Copy resonse body if status is not OK
+	if msResp.StatusCode != http.StatusOK {
+		w.WriteHeader(msResp.StatusCode)
+		_, err = io.Copy(w, msResp.Body)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Error copying MS response body: %v", err), http.StatusInternalServerError)
+		}
+		return
+	}
+
+	// Set google format response body
+	msBody, err := ioutil.ReadAll(msResp.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error reading MS response body: %v", err), http.StatusInternalServerError)
+		return
+	}
+	body, err := translate.ToGoogleResponseBody(msBody, isAuto)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Error converting to google response body: %v", err), http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(msResp.StatusCode)
+	_, err = w.Write(body)
+	if err != nil {
+		log.Errorf("Error writing response body for translate requests: %v", err)
+	}
+}
+
+// GetTranslateScript use a reverse proxy to forward handle translate script
+// requests to brave's proxy server. We're not replying a HTTP redirect
+// directly because the Originin the response header will be cleared to null
+// instead of having the value "https://translate.googleapis.com" when the
+// client follows the redirect to a cross origin, so we would violate the CORS
+// policy when the client try to access other resources from google server.
+func GetTranslateScript(w http.ResponseWriter, r *http.Request) {
+	target, _ := url.Parse(GoogleTranslateServerProxy)
+	// Use a custom director so req.Host will be changed.
+	director := func(req *http.Request) {
+		req.URL.Scheme = target.Scheme
+		req.URL.Host = target.Host
+		req.Host = target.Host
+	}
+	proxy := &httputil.ReverseProxy{Director: director}
+	proxy.ServeHTTP(w, r)
+}
+
+// GetGoogleTranslateResource redirect the resource requests from google
+// translate server to brave's proxy.
+func GetGoogleTranslateResource(w http.ResponseWriter, r *http.Request) {
+	http.Redirect(w, r, GoogleTranslateServerProxy+r.URL.Path, http.StatusTemporaryRedirect)
+}
+
+// GetGStaticResource redirect the requests from gstatic to brave's proxy.
+func GetGStaticResource(w http.ResponseWriter, r *http.Request) {
+	http.Redirect(w, r, GStaticServerProxy+r.URL.Path, http.StatusTemporaryRedirect)
+}

go.mod

@@ -0,0 +1,23 @@
+module github.com/brave/go-translate
+
+go 1.12
+
+require (
+	github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f // indirect
+	github.com/brave-intl/bat-go v0.1.1
+	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
+	github.com/getsentry/raven-go v0.0.0-20190305115053-04157b81cbfb
+	github.com/go-chi/chi v3.3.4+incompatible
+	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/pressly/lg v1.1.1
+	github.com/prometheus/client_golang v0.9.2 // indirect
+	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
+	github.com/prometheus/common v0.2.0 // indirect
+	github.com/prometheus/procfs v0.0.0-20190315082738-e56f2e22fc76 // indirect
+	github.com/sirupsen/logrus v1.4.0
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a // indirect
+	golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca // indirect
+	golang.org/x/text v0.3.0 // indirect
+)

go.sum

@@ -0,0 +1,76 @@
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f h1:xHxhygLkJBQaXZ7H0JUpmqK/gfKO2DZXB7gAKT6bbBs=
+github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/brave-intl/bat-go v0.1.1 h1:LwETyhc3axI++UMdLT4qDuEwuZw08qMYKqtDRPs238k=
+github.com/brave-intl/bat-go v0.1.1/go.mod h1:ob0XhWyX3Tqu2j0fJEoXouwc63axNdPpyyg1PVC4y4k=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/getsentry/raven-go v0.0.0-20190305115053-04157b81cbfb h1:laolVQ/AAi3OHMsrnyWVGKpN7mZAL8vddcFrO7VVmng=
+github.com/getsentry/raven-go v0.0.0-20190305115053-04157b81cbfb/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/go-chi/chi v3.3.4+incompatible h1:X+OApYAmoQS6jr1WoUgW+t5Ry5RYGXq2A//WAL5xdAU=
+github.com/go-chi/chi v3.3.4+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pressly/lg v1.1.1 h1:MDJgZSm57Lw3S0wnfzFmorDfE3nHRAVcCRh4SUTMGxM=
+github.com/pressly/lg v1.1.1/go.mod h1:B/l4UikoXw0H/DXW1O0BJxa1vVU106gryElqMy1+NDw=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
+github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190315082738-e56f2e22fc76 h1:glJ8HXGaePtQl0vo01o6x8viTBt7BaxOyXbvC9XV6VY=
+github.com/prometheus/procfs v0.0.0-20190315082738-e56f2e22fc76/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a h1:YX8ljsm6wXlHZO+aRz9Exqr0evNhKRNe5K/gi+zKh4U=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca h1:o2TLx1bGN3W+Ei0EMU5fShLupLmTOU95KvJJmfYhAzM=
+golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=