Merge pull request #22 from martinroob/milestone-0.4

Milestone 0.4
martinroob · Jul 10, 2017 · a312146 · a312146
2 parents 5b0a73e + 220eb34
commit a312146
Show file tree

Hide file tree

Showing 16 changed files with 2,167 additions and 54 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,3 +1,18 @@
+<a name="0.4.0"></a>
+# [0.4.0](https://github.com/martinroob/tiny-translator/compare/v0.4.0...v0.3.0) (2017-07-10)
+
+### Features
+
+* **new languages** Added a *French* and a *Russian* version.
+Both are created by the new Autotranslate Feature, so do not expect to see perfect translation here.
+It is more like a design study. ([#21](https://github.com/martinroob/tiny-translator/issues/21)).
+
+* **auto translation** Google translate support for ICU messages ([#20](https://github.com/martinroob/tiny-translator/issues/20)).
+
+* **auto translation** Handle Google translate query limit ([#19](https://github.com/martinroob/tiny-translator/issues/19)).
+
+* **auto translation** Google translate support should ignore region codes ([#18](https://github.com/martinroob/tiny-translator/issues/18)).
+
 <a name="0.3.0"></a>
 # [0.3.0](https://github.com/martinroob/tiny-translator/compare/v0.3.0...v0.2.0) (2017-07-01)
 

diff --git a/README.md b/README.md
@@ -53,6 +53,8 @@ There is a preinstalled version on githubpages.
 Just start it by clicking on 
 - [Tiny Translator (English)](https://martinroob.github.io/tiny-translator/en)
 - or [Tiny Translator (Deutsch)](https://martinroob.github.io/tiny-translator/de)
+- or [Tiny Translator (Francaise, auto translated)](https://martinroob.github.io/tiny-translator/fr-google)
+- or [Tiny Translator (Russian, auto translated)](https://martinroob.github.io/tiny-translator/ru-google)
 
 ## Ready to run Docker image
 There are docker images available on [Docker Cloud](https://cloud.docker.com/swarm/martinroob/repository/docker/martinroob/tiny-translator/general).

diff --git a/package.json b/package.json
@@ -1,20 +1,24 @@
 {
   "name": "tiny-translator",
   "description": "A tiny web application to translate xliff files",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
     "build": "ng build",
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e",
-    "extract-i18n": "ng xi18n --output-path src/i18n --locale en && xliffmerge --profile xliffmerge.json en de",
+    "extract-i18n": "ng xi18n --output-path src/i18n --locale en && xliffmerge --profile xliffmerge.json en de fr-google ru-google",
     "start-en": "ng serve --aot --i18n-file=src/i18n/messages.en.xlf --locale=en --i18n-format=xlf",
     "start-de": "ng serve --aot --i18n-file=src/i18n/messages.de.xlf --locale=en --i18n-format=xlf",
+    "start-fr": "ng serve --aot --i18n-file=src/i18n/messages.fr-google.xlf --locale=fr --i18n-format=xlf",
+    "start-ru": "ng serve --aot --i18n-file=src/i18n/messages.ru-google.xlf --locale=ru --i18n-format=xlf",
     "build-prod-en": "ng build --prod --aot --i18n-file=src/i18n/messages.en.xlf --locale=en --i18n-format=xlf --base-href=/tiny-translator/en/ --output-path=dist/en",
     "build-prod-de": "ng build --prod --aot --i18n-file=src/i18n/messages.de.xlf --locale=de --i18n-format=xlf --base-href=/tiny-translator/de/ --output-path=dist/de",
-    "build-prod": "npm run build-prod-en && npm run build-prod-de && cpx ./src/ghpages/* ./dist",
+    "build-prod-fr": "ng build --prod --aot --i18n-file=src/i18n/messages.fr-google.xlf --locale=fr --i18n-format=xlf --base-href=/tiny-translator/fr-google/ --output-path=dist/fr-google",
+    "build-prod-ru": "ng build --prod --aot --i18n-file=src/i18n/messages.ru-google.xlf --locale=ru --i18n-format=xlf --base-href=/tiny-translator/ru-google/ --output-path=dist/ru-google",
+    "build-prod": "npm run build-prod-en && npm run build-prod-de && npm run build-prod-fr && npm run build-prod-ru && cpx ./src/ghpages/* ./dist",
     "publish2githubpages": "angular-cli-ghpages --no-silent",
     "dockerbuild": "docker build -t tiny-translator:0.3 .",
     "publish2dockerhub": "docker push"
@@ -53,6 +57,7 @@
     "hammerjs": "^2.0.8",
     "ngx-i18nsupport-lib": "^1.4.6",
     "rxjs": "^5.1.0",
+    "webpack": "^3.1.0",
     "zone.js": "^0.8.10"
   },
   "devDependencies": {
@@ -70,10 +75,10 @@
     "karma-coverage-istanbul-reporter": "^0.2.0",
     "karma-jasmine": "~1.1.0",
     "karma-jasmine-html-reporter": "^0.2.2",
-    "ngx-i18nsupport": "^0.6.2",
+    "ngx-i18nsupport": "^0.7.4",
     "protractor": "~5.1.0",
     "ts-node": "~2.0.0",
     "tslint": "~4.5.0",
-    "typescript": "^2.2.1"
+    "typescript": "2.3.3"
   }
 }
diff --git a/src/app/app.component.spec.ts b/src/app/app.component.spec.ts
@@ -22,7 +22,7 @@ describe('AppComponent', () => {
   it(`should have a build version`, async(() => {
     const fixture = TestBed.createComponent(AppComponent);
     const app: AppComponent = fixture.debugElement.componentInstance;
-    expect(app.buildversion()).toEqual('0.2.0'); // TODO
+    expect(app.buildversion()).toEqual('0.4.0'); // TODO
   }));
 
   it('should render md-toolbar', async(() => {

diff --git a/src/app/app.config.ts b/src/app/app.config.ts
@@ -17,8 +17,8 @@ export class AppConfig {
 
 export const APP_CONFIG_VALUE: AppConfig = {
   // set values here
-  'BUILDVERSION': '0.3',
-  'BUILDTIME': '2017-07-01', // TODO should be dynamic
+  'BUILDVERSION': '0.4.0',
+  'BUILDTIME': '2017-07-10', // TODO should be dynamic
   GOOGLETRANSLATE_API_ROOT_URL: 'https://translation.googleapis.com/',
   GOOGLETRANSLATE_API_KEY: environment.googletranslate_api_key
 };

diff --git a/src/app/model/auto-translate-google.service.spec.ts b/src/app/model/auto-translate-google.service.spec.ts
@@ -61,13 +61,31 @@ describe('AutoTranslateGoogleService', () => {
     });
   })));
 
+  it('should translate hello from english to german ignoring region codes', async(inject([AutoTranslateGoogleService], (service: AutoTranslateGoogleService) => {
+    service.translate('Hello', 'en-us', 'DE-DE').subscribe((translation) => {
+      expect(translation).toBe('Hallo');
+    });
+  })));
+
   it('should translate multiple string at once', async(inject([AutoTranslateGoogleService], (service: AutoTranslateGoogleService) => {
     service.translateMultipleStrings(['Hello', 'world'], 'en', 'de').subscribe((translations) => {
       expect(translations[0]).toBe('Hallo');
       expect(translations[1]).toBe('Welt');
     });
   })));
 
+  it('should translate more than 128 multiple strings at once (exceeding google limit)', async(inject([AutoTranslateGoogleService], (service: AutoTranslateGoogleService) => {
+    const NUM = 1000; // internal google limit is 128, so service has to split it...
+    const manyMessages: string[] = [];
+    for (let i = 0; i < NUM; i++) {
+      manyMessages.push('Hello world!');
+    }
+    service.translateMultipleStrings(manyMessages, 'en', 'de').subscribe((translations) => {
+      expect(translations[0]).toBe('Hallo Welt!');
+      expect(translations[NUM -1]).toBe('Hallo Welt!');
+    });
+  })));
+
   it('should return a list of languages supported', async(inject([AutoTranslateGoogleService], (service: AutoTranslateGoogleService) => {
     service.supportedLanguages('en').pairwise().subscribe((lists) => {
       const list = lists[1];

diff --git a/src/app/model/auto-translate-google.service.ts b/src/app/model/auto-translate-google.service.ts
@@ -7,11 +7,8 @@ import {APP_CONFIG, AppConfig} from '../app.config';
 import {Observable} from 'rxjs/Observable';
 import {Http, Response} from '@angular/http';
 import {isNullOrUndefined} from 'util';
-import {Subject} from 'rxjs/Subject';
 import {BehaviorSubject} from 'rxjs/BehaviorSubject';
 
-const projectId = 'tinytranslator-20170621';
-
 /**
  * Types form google translate api.
  */
@@ -47,6 +44,9 @@ interface TranslationsListResponse {
   translations: TranslationsResource[];
 }
 
+// maximum number of strings to translate in one request (a Google limit!)
+const MAX_SEGMENTS = 128;
+
 /**
  * Auto Translate Service using Google Translate.
  */
@@ -68,6 +68,25 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
    */
   private _permanentFailReason: AutoTranslateDisabledReason;
 
+  /**
+   * Strip region code and convert to lower
+   * @param lang
+   * @return {string} lang without region code and in lower case.
+   */
+  public static stripRegioncode(lang: string): string {
+    if (isNullOrUndefined(lang)) {
+      return null;
+    }
+    const langLower = lang.toLowerCase();
+    for(let i = 0; i < langLower.length; i++) {
+      const c = langLower.charAt(i);
+      if (c < 'a' || c > 'z') {
+        return langLower.substring(0, i);
+      }
+    }
+    return langLower;
+  }
+
   constructor(@Inject(APP_CONFIG) app_config: AppConfig, private httpService: Http) {
     super();
     this._rootUrl = app_config.GOOGLETRANSLATE_API_ROOT_URL;
@@ -97,10 +116,12 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
    */
   public canAutoTranslate(source: string, target: string): Observable<boolean> {
     return this.supportedLanguages().map((languages: Language[]) => {
-      if (languages.findIndex((lang) => lang.language === source) < 0) {
+      const s = AutoTranslateGoogleService.stripRegioncode(source);
+      const t = AutoTranslateGoogleService.stripRegioncode(target);
+      if (!s || languages.findIndex((lang) => lang.language === s) < 0) {
         return false;
       }
-      return (languages.findIndex((lang) => lang.language === target) >= 0);
+      return (!t || languages.findIndex((lang) => lang.language === t) >= 0);
     });
   }
 
@@ -115,10 +136,12 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
       if (languages.length === 0) {
         return this._permanentFailReason;
       }
-      if (!source || languages.findIndex((lang) => lang.language === source) < 0) {
+      const s = AutoTranslateGoogleService.stripRegioncode(source);
+      if (!s || languages.findIndex((lang) => lang.language === s) < 0) {
         return {reason: AutoTranslateDisabledReasonKey.SOURCE_LANG_NOT_SUPPORTED};
       }
-      if (!target || languages.findIndex((lang) => lang.language === target) < 0) {
+      const t = AutoTranslateGoogleService.stripRegioncode(target);
+      if (!t || languages.findIndex((lang) => lang.language === t) < 0) {
         return {reason: AutoTranslateDisabledReasonKey.TARGET_LANG_NOT_SUPPORTED};
       }
       return null;
@@ -133,6 +156,8 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
   supportedLanguages(target?: string): Observable<Language[]> {
     if (!target) {
       target = 'en';
+    } else {
+      target = AutoTranslateGoogleService.stripRegioncode(target);
     }
     if (!this._subjects[target]) {
       if (this._apiKey) {
@@ -185,6 +210,8 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
     if (!this._apiKey) {
       return Observable.throw('error, no api key');
     }
+    from = AutoTranslateGoogleService.stripRegioncode(from);
+    to = AutoTranslateGoogleService.stripRegioncode(to);
     const translateRequest: TranslateTextRequest = {
       q: [message],
       target: to,
@@ -209,6 +236,34 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
     if (!this._apiKey) {
       return Observable.throw('error, no api key');
     }
+    from = AutoTranslateGoogleService.stripRegioncode(from);
+    to = AutoTranslateGoogleService.stripRegioncode(to);
+    const allRequests: Observable<string[]>[] = this.splitMessagesToGoogleLimit(messages).map((partialMessages: string[]) => {
+      return this.limitedTranslateMultipleStrings(partialMessages, from, to);
+    });
+    return Observable.forkJoin(allRequests).map((allTranslations: string[][]) => {
+      let all = [];
+      for (let i = 0; i < allTranslations.length; i++) {
+        all = all.concat(allTranslations[i]);
+      }
+      return all;
+    })
+  }
+
+  /**
+   * Return translation request, but messages must be limited to google limits.
+   * Not more that 128 single messages.
+   * @param messages
+   * @param from
+   * @param to
+   * @return {Observable<string[]>} the translated strings
+   */
+  private limitedTranslateMultipleStrings(messages: string[], from: string, to: string): Observable<string[]> {
+    if (!this._apiKey) {
+      return Observable.throw('error, no api key');
+    }
+    from = AutoTranslateGoogleService.stripRegioncode(from);
+    to = AutoTranslateGoogleService.stripRegioncode(to);
     const translateRequest: TranslateTextRequest = {
       q: messages,
       target: to,
@@ -223,4 +278,33 @@ export class AutoTranslateGoogleService extends AutoTranslateServiceAPI {
       });
     });
   }
+
+  /**
+   * Splits one array of messages to n arrays, where each has at least 128 (const MAX_ENTRIES) entries.
+   * @param messages
+   * @return {any}
+   */
+  private splitMessagesToGoogleLimit(messages: string[]): string[][] {
+    if (messages.length <= MAX_SEGMENTS) {
+      return [messages];
+    }
+    const result = [];
+    let currentPackage = [];
+    let packageSize = 0;
+    for (let i = 0; i < messages.length; i++) {
+      currentPackage.push(messages[i]);
+      packageSize++;
+      if (packageSize >= MAX_SEGMENTS) {
+        result.push(currentPackage);
+        currentPackage = [];
+        packageSize = 0;
+      }
+    }
+    if (currentPackage.length > 0) {
+      result.push(currentPackage);
+    }
+    return result;
+  }
+
+
 }
diff --git a/src/app/model/auto-translate-summary-report.ts b/src/app/model/auto-translate-summary-report.ts
@@ -40,11 +40,39 @@ export class AutoTranslateSummaryReport {
     }
   }
 
+  /**
+   * Merge another summary into this one.
+   * @param anotherSummary
+   */
+  public merge(anotherSummary: AutoTranslateSummaryReport) {
+    this._total += anotherSummary.total();
+    this._ignored += anotherSummary.ignored();
+    this._success += anotherSummary.success();
+    this._failed += anotherSummary.failed();
+  }
+
+  public total(): number {
+    return this._total;
+  }
+
+  public ignored(): number {
+    return this._ignored;
+  }
+
+  public success(): number {
+    return this._success;
+  }
+
+  public failed(): number {
+    return this._failed;
+  }
+
   /**
    * Human readable version of report
    */
   public content(): string {
     let result = format('Total translated: %s\nIgnored: %s\nSuccesful: %s\nFailed: %s', this._total, this._ignored, this._success, this._failed);
     return result;
   }
+
 }
diff --git a/src/app/model/normalized-message.ts b/src/app/model/normalized-message.ts
@@ -120,18 +120,38 @@ export class NormalizedMessage {
    * @return new translated message (as Observable, it is an async call)
    */
   public autoTranslateUsingService(autoTranslateService: AutoTranslateServiceAPI, sourceLanguage: string, targetLanguage: string): Observable<NormalizedMessage> {
-    // TODO corner cases to be researched like special tags, ICU, ...
-    if (this.isICUMessage()) {
-      // TODO handling of ICU messages currently not supported
-      return Observable.of(this);
+    // TODO corner cases to be researched like special tags, ...
+    if (this.getICUMessage()) {
+      return this.autoTranslateICUMessageUsingService(autoTranslateService, sourceLanguage, targetLanguage);
+    } else {
+      return autoTranslateService.translate(this.dislayText(true), sourceLanguage, targetLanguage).map((translation: string) => {
+        if (!isNullOrUndefined(translation)) {
+          return this.translate(translation, true);
+        } else {
+          return null;
+        }
+      });
     }
-    return autoTranslateService.translate(this.dislayText(true), sourceLanguage, targetLanguage).map((translation: string) => {
-      if (!isNullOrUndefined(translation)) {
-        return this.translate(translation, true);
-      } else {
-        return null;
-      }
-    });
+  }
+
+  private autoTranslateICUMessageUsingService(autoTranslateService: AutoTranslateServiceAPI, sourceLanguage: string, targetLanguage: string): Observable<NormalizedMessage> {
+    const icuMessage: IICUMessage = this.getICUMessage();
+    const categories = icuMessage.getCategories();
+    // check for nested ICUs, we do not support that
+    if (categories.find((category) => !isNullOrUndefined(category.getMessageNormalized().getICUMessage()))) {
+      throw new Error('nested ICU message not supported');
+    }
+    const allMessages: string[] = categories.map((category) => category.getMessageNormalized().asDisplayString());
+    return autoTranslateService.translateMultipleStrings(allMessages, sourceLanguage, targetLanguage)
+      .map((translations: string[]) => {
+        const icuTranslation: IICUMessageTranslation = {};
+        for (let i = 0; i < translations.length; i++) {
+          const translationText = translations[i];
+          icuTranslation[categories[i].getCategory()] = translationText;
+        }
+        const result = this.translateICUMessage(icuTranslation);
+        return result;
+      });
   }
 
   public translateICUMessage(newValue: IICUMessageTranslation): NormalizedMessage {