From 89a351a0d156950071cc144a125f390f2ec038cd Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Thu, 6 Aug 2015 08:58:58 -0500 Subject: [PATCH 01/27] Add check for if string exists --- test/transifex/malformedStrings.test.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/transifex/malformedStrings.test.js b/test/transifex/malformedStrings.test.js index cd751cd752..11a41c88a9 100644 --- a/test/transifex/malformedStrings.test.js +++ b/test/transifex/malformedStrings.test.js @@ -29,10 +29,16 @@ _(languages).each(function(lang) { var englishString = parsedEnglishFile[key]; var translationString = parsedTranslationFile[key]; + it(key + ' exists in ' + file + ' for the ' + lang + ' language', function() { + expect(translationString).to.exist; + }); + + if(!translationString) return; + var EnglishOccurences = englishString.match(interpolationRegex); var TranslationOccurences = translationString.match(interpolationRegex); - xit(key + ' - contains an interpolation', function() { + it(key + ' - contains an interpolation', function() { if(!TranslationOccurences) console.log("English String:", englishString); expect(EnglishOccurences).to.exist; @@ -40,7 +46,7 @@ _(languages).each(function(lang) { }); if(EnglishOccurences.length > 1) { - it(key + ' - has the correct number of interpolations', function() { + xit(key + ' - has the correct number of interpolations', function() { if(!TranslationOccurences) console.log("English String:", englishString); var numberOfOccurences = TranslationOccurences && TranslationOccurences.length; expect(numberOfOccurences).to.eql(EnglishOccurences.length); From fd76baa6f8411c4b76d6392d771baf124df755ff Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Thu, 6 Aug 2015 22:42:24 -0500 Subject: [PATCH 02/27] Missing strings task --- tasks/gulp-transifex-test.js | 96 ++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tasks/gulp-transifex-test.js diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js new file mode 100644 index 0000000000..dc53284648 --- /dev/null +++ b/tasks/gulp-transifex-test.js @@ -0,0 +1,96 @@ +import fs from 'fs'; +import _ from 'lodash'; +import nconf from 'nconf'; +import gulp from 'gulp'; +import request from 'superagent'; + +nconf.argv().env().file({ file: 'config.json' }); + +const LOCALES = './common/locales/'; +const ENGLISH_LOCALE = `${LOCALES}en/`; + +const SLACK_URL = nconf.get('TRANSIFEX_SLACK:url'); +const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); +const SLACK_USERNAME = 'Transifex'; +const SLACK_EMOJI = ':transifex:'; + +gulp.task('transifex:exists', () => { + + var missingStrings = []; + + eachTranslationString((language, key, englishString, translationString) => { + if(!translationString) { + let errorString = `${language} - ${key} - ${englishString}`; + missingStrings.push(errorString); + } + }); + + if (!_.isEmpty(missingStrings)) { + post('The following strings are missing', missingStrings); + } +}); + +function eachTranslationFile(cb) { + let languages = fs.readdirSync(LOCALES); + languages.shift(); // Remove README.md from array of languages + + let jsonFiles = stripOutNonJsonFiles(fs.readdirSync(ENGLISH_LOCALE)); + + _(languages).each((lang) => { + _.each(jsonFiles, (file) => { + let translationFile = fs.readFileSync(LOCALES + lang + '/' + file); + let parsedTranslationFile = JSON.parse(translationFile); + + let englishFile = fs.readFileSync(ENGLISH_LOCALE + file); + let parsedEnglishFile = JSON.parse(englishFile); + + cb(lang, parsedEnglishFile, parsedTranslationFile) + }); + }); +} + +function eachTranslationString(cb) { + eachTranslationFile((language, englishJSON, translationJSON) => { + _.each(englishJSON, (string, key) => { + var translationString = translationJSON[key]; + cb(language, key, string, translationString); + }); + }); +} + +function checkForIssues(thingToCheck) { + let hasIssues = !_.isEmpty(thingToCheck); + return hasIssues; +} + +function post(message, items) { + let formattedMessage = formatMessageForPosting(message, items); + + request.post(SLACK_URL) + .send({ + channel: SLACK_CHANNEL, + username: SLACK_USERNAME, + text: formattedMessage, + icon_emoji: SLACK_EMOJI + }) + .end((err, res) => { + if (err) console.error('Unable to post to slack', err); + }); +} + +function formatMessageForPosting(msg, items) { + let body = `*Warning:* ${msg}`; + body += '\n\n```\n'; + body += items.join('\n'); + body += '\n```'; + + return body; +} + +function stripOutNonJsonFiles(collection) { + let onlyJson = _.filter(collection, (file) => { + return file.match(/[a-zA-Z]*\.json/); + }); + + return onlyJson; +} From 563072712baf60d426dd8301dd68be69721564d0 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Fri, 7 Aug 2015 09:00:17 -0500 Subject: [PATCH 03/27] Refactor untranslated strings task --- tasks/gulp-transifex-test.js | 44 +++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index dc53284648..f2dfd406bc 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -14,46 +14,64 @@ const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; -gulp.task('transifex:exists', () => { +gulp.task('transifex:untranslatedStrings', () => { - var missingStrings = []; + let missingStrings = []; + let languages = getArrayOfLanguages(); - eachTranslationString((language, key, englishString, translationString) => { + eachTranslationString(languages, (language, filename, key, englishString, translationString) => { if(!translationString) { - let errorString = `${language} - ${key} - ${englishString}`; + let errorString = `${language} - ${filename} - ${key} - ${englishString}`; missingStrings.push(errorString); } }); if (!_.isEmpty(missingStrings)) { - post('The following strings are missing', missingStrings); + let message = 'The following strings are not translated'; + let formattedMessage = formatMessageForPosting(message, missingStrings); + console.log(formattedMessage); } }); -function eachTranslationFile(cb) { +function getArrayOfLanguages() { let languages = fs.readdirSync(LOCALES); languages.shift(); // Remove README.md from array of languages + return languages; +} + +function getEnglishLanguages(cb) { + let allLanguages = getArrayOfLanguages(); + + let nonEnglishLanguages = _.filter(allLanguages, (lang) => { + return lang.indexOf('en') !== 0; + }); + + return nonEnglishLanguages; +} + + +function eachTranslationFile(languages, cb) { let jsonFiles = stripOutNonJsonFiles(fs.readdirSync(ENGLISH_LOCALE)); _(languages).each((lang) => { - _.each(jsonFiles, (file) => { - let translationFile = fs.readFileSync(LOCALES + lang + '/' + file); + _.each(jsonFiles, (filename) => { + let translationFile = fs.readFileSync(LOCALES + lang + '/' + filename); let parsedTranslationFile = JSON.parse(translationFile); - let englishFile = fs.readFileSync(ENGLISH_LOCALE + file); + let englishFile = fs.readFileSync(ENGLISH_LOCALE + filename); let parsedEnglishFile = JSON.parse(englishFile); - cb(lang, parsedEnglishFile, parsedTranslationFile) + cb(lang, filename, parsedEnglishFile, parsedTranslationFile) }); }); } -function eachTranslationString(cb) { - eachTranslationFile((language, englishJSON, translationJSON) => { +function eachTranslationString(languages, cb) { + eachTranslationFile(languages, (language, filename, englishJSON, translationJSON) => { _.each(englishJSON, (string, key) => { var translationString = translationJSON[key]; - cb(language, key, string, translationString); + cb(language, filename, key, string, translationString); }); }); } From 894611bc34a5398d92cc7365dda72372b88a9f45 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Fri, 7 Aug 2015 09:34:18 -0500 Subject: [PATCH 04/27] Create shell of task for looking up strings --- package.json | 1 + tasks/gulp-transifex-test.js | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/package.json b/package.json index b387c0f638..5f0b9ef6c8 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "karma-requirejs": "~0.2.0", "karma-script-launcher": "~0.1.0", "lcov-result-merger": "^1.0.2", + "md5": "^2.0.0", "mocha": "~1.12.1", "mongoskin": "~0.6.1", "phantomjssmith": "~0.5.4", diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index f2dfd406bc..c50562bfc2 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -1,6 +1,7 @@ import fs from 'fs'; import _ from 'lodash'; import nconf from 'nconf'; +import md5 from 'md5'; import gulp from 'gulp'; import request from 'superagent'; @@ -14,6 +15,16 @@ const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; +gulp.task('transifex:look', () => { + // curl -i -L --user username:password -X GET https://www.transifex.com/api/2/project/transifex/resource/core/translation/pt_BR/string/e9fbd679f07d178744bfa80344080962/ + // /project//resource//translation//string// + // request.get('https://www.transifex.com/api/2/project/habitrpg/resource/petsjson/translation/uk/string/7541ebdf41af9839f458f9afb1644882/') + // .end((err, res) => { + // if (err) console.log(":(", err); + // else console.log(res.body); + // }); +}); + gulp.task('transifex:untranslatedStrings', () => { let missingStrings = []; @@ -105,6 +116,11 @@ function formatMessageForPosting(msg, items) { return body; } +function getHash(string) { + let hash = md5(`${string}:`); + return hash; +} + function stripOutNonJsonFiles(collection) { let onlyJson = _.filter(collection, (file) => { return file.match(/[a-zA-Z]*\.json/); From 7ca785f33e75fb22b23aa9c9c21f76016da4eb64 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Fri, 7 Aug 2015 19:28:21 -0500 Subject: [PATCH 05/27] Remove unused callback --- tasks/gulp-transifex-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index c50562bfc2..559d48cdaa 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -51,7 +51,7 @@ function getArrayOfLanguages() { return languages; } -function getEnglishLanguages(cb) { +function getEnglishLanguages() { let allLanguages = getArrayOfLanguages(); let nonEnglishLanguages = _.filter(allLanguages, (lang) => { From f355e8e3af77845f0abaf3e6b55a1f994ffff611 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Fri, 7 Aug 2015 22:59:55 -0500 Subject: [PATCH 06/27] Wrote task that can look up english strings --- tasks/gulp-transifex-test.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 559d48cdaa..9fff15ae69 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -16,13 +16,18 @@ const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; gulp.task('transifex:look', () => { - // curl -i -L --user username:password -X GET https://www.transifex.com/api/2/project/transifex/resource/core/translation/pt_BR/string/e9fbd679f07d178744bfa80344080962/ - // /project//resource//translation//string// - // request.get('https://www.transifex.com/api/2/project/habitrpg/resource/petsjson/translation/uk/string/7541ebdf41af9839f458f9afb1644882/') - // .end((err, res) => { - // if (err) console.log(":(", err); - // else console.log(res.body); - // }); + + let equivalentStrings = []; + let nonEnglishLanguages = getNonEnglishLanguages(); + + eachTranslationString(nonEnglishLanguages, (language, filename, key, englishString, translationString) => { + if (englishString === translationString) { + let hash = getHash(key); + let json = filename.replace('.',''); + let url = `https://www.transifex.com/api/2/project/habitrpg/resource/${json}/translation/${language}/string/${hash}/` + console.log(url); + } + }); }); gulp.task('transifex:untranslatedStrings', () => { @@ -31,7 +36,7 @@ gulp.task('transifex:untranslatedStrings', () => { let languages = getArrayOfLanguages(); eachTranslationString(languages, (language, filename, key, englishString, translationString) => { - if(!translationString) { + if (!translationString) { let errorString = `${language} - ${filename} - ${key} - ${englishString}`; missingStrings.push(errorString); } @@ -51,7 +56,7 @@ function getArrayOfLanguages() { return languages; } -function getEnglishLanguages() { +function getNonEnglishLanguages() { let allLanguages = getArrayOfLanguages(); let nonEnglishLanguages = _.filter(allLanguages, (lang) => { From 9d3c40122ad193a03c3c1f875575e92c1788b51b Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sat, 8 Aug 2015 22:24:52 -0500 Subject: [PATCH 07/27] REmove unsued task --- tasks/gulp-transifex-test.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 9fff15ae69..ab93e89468 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -15,21 +15,6 @@ const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; -gulp.task('transifex:look', () => { - - let equivalentStrings = []; - let nonEnglishLanguages = getNonEnglishLanguages(); - - eachTranslationString(nonEnglishLanguages, (language, filename, key, englishString, translationString) => { - if (englishString === translationString) { - let hash = getHash(key); - let json = filename.replace('.',''); - let url = `https://www.transifex.com/api/2/project/habitrpg/resource/${json}/translation/${language}/string/${hash}/` - console.log(url); - } - }); -}); - gulp.task('transifex:untranslatedStrings', () => { let missingStrings = []; From f58c9e2192e735919007609e1440374ce96c6cdc Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sat, 8 Aug 2015 22:25:02 -0500 Subject: [PATCH 08/27] Rename task --- tasks/gulp-transifex-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index ab93e89468..c6e1e820e7 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -15,7 +15,7 @@ const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; -gulp.task('transifex:untranslatedStrings', () => { +gulp.task('transifex:missingStrings', () => { let missingStrings = []; let languages = getArrayOfLanguages(); From 092ba23ae4a4136396cf83157d158f8a5e246953 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sat, 8 Aug 2015 22:46:51 -0500 Subject: [PATCH 09/27] Adjust each file to handle errors --- tasks/gulp-transifex-test.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index c6e1e820e7..ef91ecf3c1 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -57,19 +57,24 @@ function eachTranslationFile(languages, cb) { _(languages).each((lang) => { _.each(jsonFiles, (filename) => { - let translationFile = fs.readFileSync(LOCALES + lang + '/' + filename); - let parsedTranslationFile = JSON.parse(translationFile); + try { + var translationFile = fs.readFileSync(LOCALES + lang + '/' + filename); + var parsedTranslationFile = JSON.parse(translationFile); + } catch (err) { + return cb(err); + } let englishFile = fs.readFileSync(ENGLISH_LOCALE + filename); let parsedEnglishFile = JSON.parse(englishFile); - cb(lang, filename, parsedEnglishFile, parsedTranslationFile) + cb(null, lang, filename, parsedEnglishFile, parsedTranslationFile) }); }); } function eachTranslationString(languages, cb) { - eachTranslationFile(languages, (language, filename, englishJSON, translationJSON) => { + eachTranslationFile(languages, (error, language, filename, englishJSON, translationJSON) => { + if (error) return; _.each(englishJSON, (string, key) => { var translationString = translationJSON[key]; cb(language, filename, key, string, translationString); From bfc5ac943583c29790079adcd52e015c3e7b3286 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sat, 8 Aug 2015 22:47:26 -0500 Subject: [PATCH 10/27] Add task to check for missing files --- tasks/gulp-transifex-test.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index ef91ecf3c1..2094f06c10 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -15,6 +15,21 @@ const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; +gulp.task('transifex:missingFiles', () => { + let missingStrings = []; + let languages = getArrayOfLanguages(); + eachTranslationFile(languages, (error) => { + if(error) { + missingStrings.push(error.path); + } + }); + + if (!_.isEmpty(missingStrings)) { + let message = 'the following files were missing from the translations folder'; + post(message, missingStrings); + } +}); + gulp.task('transifex:missingStrings', () => { let missingStrings = []; From 8b00a772cf60e38f5179f08b2480baf27852f3eb Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sat, 8 Aug 2015 22:50:18 -0500 Subject: [PATCH 11/27] Remove hash function --- tasks/gulp-transifex-test.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 2094f06c10..517a9b5fd5 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -66,7 +66,6 @@ function getNonEnglishLanguages() { return nonEnglishLanguages; } - function eachTranslationFile(languages, cb) { let jsonFiles = stripOutNonJsonFiles(fs.readdirSync(ENGLISH_LOCALE)); @@ -126,11 +125,6 @@ function formatMessageForPosting(msg, items) { return body; } -function getHash(string) { - let hash = md5(`${string}:`); - return hash; -} - function stripOutNonJsonFiles(collection) { let onlyJson = _.filter(collection, (file) => { return file.match(/[a-zA-Z]*\.json/); From cd15b115ad5f0ee9e34302eecf518d4ff33ffb04 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:13:17 -0500 Subject: [PATCH 12/27] Add constant for all languages array --- tasks/gulp-transifex-test.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 517a9b5fd5..bc280d224b 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -9,6 +9,7 @@ nconf.argv().env().file({ file: 'config.json' }); const LOCALES = './common/locales/'; const ENGLISH_LOCALE = `${LOCALES}en/`; +const ALL_LANGUAGES = getArrayOfLanguages(); const SLACK_URL = nconf.get('TRANSIFEX_SLACK:url'); const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); @@ -17,8 +18,8 @@ const SLACK_EMOJI = ':transifex:'; gulp.task('transifex:missingFiles', () => { let missingStrings = []; - let languages = getArrayOfLanguages(); - eachTranslationFile(languages, (error) => { + + eachTranslationFile(ALL_LANGUAGES, (error) => { if(error) { missingStrings.push(error.path); } @@ -33,9 +34,8 @@ gulp.task('transifex:missingFiles', () => { gulp.task('transifex:missingStrings', () => { let missingStrings = []; - let languages = getArrayOfLanguages(); - eachTranslationString(languages, (language, filename, key, englishString, translationString) => { + eachTranslationString(ALL_LANGUAGES, (language, filename, key, englishString, translationString) => { if (!translationString) { let errorString = `${language} - ${filename} - ${key} - ${englishString}`; missingStrings.push(errorString); @@ -57,9 +57,7 @@ function getArrayOfLanguages() { } function getNonEnglishLanguages() { - let allLanguages = getArrayOfLanguages(); - - let nonEnglishLanguages = _.filter(allLanguages, (lang) => { + let nonEnglishLanguages = _.filter(ALL_LANGUAGES, (lang) => { return lang.indexOf('en') !== 0; }); From c1f0fae456222c4ea48b091380fc34ecf04bea2f Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:14:09 -0500 Subject: [PATCH 13/27] Change format of reporting --- tasks/gulp-transifex-test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index bc280d224b..8c423fc45a 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -116,9 +116,8 @@ function post(message, items) { function formatMessageForPosting(msg, items) { let body = `*Warning:* ${msg}`; - body += '\n\n```\n'; + body += '\n\n>>>\n'; body += items.join('\n'); - body += '\n```'; return body; } From d0e4590a01c19a33e7a5bae82d510d30132bebb8 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:42:44 -0500 Subject: [PATCH 14/27] Change missingStrings to that it posts to slack --- tasks/gulp-transifex-test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 8c423fc45a..18e4dd94b0 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -37,15 +37,14 @@ gulp.task('transifex:missingStrings', () => { eachTranslationString(ALL_LANGUAGES, (language, filename, key, englishString, translationString) => { if (!translationString) { - let errorString = `${language} - ${filename} - ${key} - ${englishString}`; + let errorString = `${language} - ${filename} - ${key} - \`${englishString}\``; missingStrings.push(errorString); } }); if (!_.isEmpty(missingStrings)) { let message = 'The following strings are not translated'; - let formattedMessage = formatMessageForPosting(message, missingStrings); - console.log(formattedMessage); + post(message, missingStrings); } }); From 841532271783a7e645dd7c76f8549f441c5b1d3b Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:43:04 -0500 Subject: [PATCH 15/27] Create malformed string check --- tasks/gulp-transifex-test.js | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 18e4dd94b0..d9f1b6ae5a 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -48,6 +48,51 @@ gulp.task('transifex:missingStrings', () => { } }); +gulp.task('transifex:malformedStrings', () => { + + let jsonFiles = stripOutNonJsonFiles(fs.readdirSync(ENGLISH_LOCALE)); + let interpolationRegex = /<%= [a-zA-Z]* %>/g; + let stringsToLookFor = getStringsWith(jsonFiles, interpolationRegex); + + let stringsWithMalformedInterpolations = []; + let stringsWithIncorrectNumberOfInterpolations = []; + + let count = 0; + _(ALL_LANGUAGES).each(function(lang) { + + _.each(stringsToLookFor, function(strings, file) { + let translationFile = fs.readFileSync(LOCALES + lang + '/' + file); + let parsedTranslationFile = JSON.parse(translationFile); + + _.each(strings, function(value, key) { + let translationString = parsedTranslationFile[key]; + if (!translationString) return; + + let englishOccurences = stringsToLookFor[file][key]; + let translationOccurences = translationString.match(interpolationRegex); + + if (!translationOccurences) { + let malformedString = `${lang} - ${file} - ${key} - \`${translationString}\``; + stringsWithMalformedInterpolations.push(malformedString); + } else if (englishOccurences.length !== translationOccurences.length) { + let missingInterploationString = `${lang} - ${file} - ${key} - \`${translationString}\``; + stringsWithIncorrectNumberOfInterpolations.push(missingInterploationString); + } + }); + }); + }); + + if (!_.isEmpty(stringsWithMalformedInterpolations)) { + let message = '*Warning:* The following strings have malformed interpolations'; + post(message, stringsWithMalformedInterpolations); + } + + if (!_.isEmpty(stringsWithIncorrectNumberOfInterpolations)) { + let message = '*Warning:* The following strings have a different number of string interpolations'; + post(message, stringsWithIncorrectNumberOfInterpolations); + } +}); + function getArrayOfLanguages() { let languages = fs.readdirSync(LOCALES); languages.shift(); // Remove README.md from array of languages @@ -121,6 +166,23 @@ function formatMessageForPosting(msg, items) { return body; } +function getStringsWith(json, interpolationRegex) { + var strings = {}; + + _(json).each(function(file_name) { + var raw_file = fs.readFileSync(ENGLISH_LOCALE + file_name); + var parsed_json = JSON.parse(raw_file); + + strings[file_name] = {}; + _.each(parsed_json, function(value, key) { + var match = value.match(interpolationRegex); + if(match) strings[file_name][key] = match; + }); + }); + + return strings; +} + function stripOutNonJsonFiles(collection) { let onlyJson = _.filter(collection, (file) => { return file.match(/[a-zA-Z]*\.json/); From 6cd3ed3e075c069db902f338558061508755406b Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:44:34 -0500 Subject: [PATCH 16/27] Set default channel --- tasks/gulp-transifex-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index d9f1b6ae5a..d856ed4f25 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -12,7 +12,7 @@ const ENGLISH_LOCALE = `${LOCALES}en/`; const ALL_LANGUAGES = getArrayOfLanguages(); const SLACK_URL = nconf.get('TRANSIFEX_SLACK:url'); -const SLACK_CHANNEL = '#' + nconf.get('TRANSIFEX_SLACK:channel'); +const SLACK_CHANNEL = '#' + (nconf.get('TRANSIFEX_SLACK:channel') || 'general'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; From b35453cd4b470d280b9cc54d86e7939cf08e3f3c Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:52:00 -0500 Subject: [PATCH 17/27] Add transifex config to example config --- config.json.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.json.example b/config.json.example index a7b1660983..6836f38e74 100644 --- a/config.json.example +++ b/config.json.example @@ -40,6 +40,10 @@ "accessKeyId":"accessKeyId", "secretAccessKey":"secretAccessKey" }, + "transifex_slack": { + "url": "https://hooks.slack.com/services/some-url", + "channel": "general" + }, "PAYPAL":{ "billing_plans": { "basic_earned":"basic_earned", From 1ffad1997eb68146c925b3c8c45e3e6ca39c6141 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Sun, 9 Aug 2015 16:52:25 -0500 Subject: [PATCH 18/27] Removed mocha test for translations --- test/transifex/malformedStrings.test.js | 84 ------------------------- 1 file changed, 84 deletions(-) delete mode 100644 test/transifex/malformedStrings.test.js diff --git a/test/transifex/malformedStrings.test.js b/test/transifex/malformedStrings.test.js deleted file mode 100644 index 11a41c88a9..0000000000 --- a/test/transifex/malformedStrings.test.js +++ /dev/null @@ -1,84 +0,0 @@ -var chai = require("chai") -var expect = chai.expect - -var fs = require('fs'); -var _ = require('lodash'); - -var locales = './common/locales/'; -var english = locales + 'en/'; -var interpolationRegex = /<%= [a-zA-Z]* %>/g; - -var languages = fs.readdirSync(locales); -languages.shift(); // Remove README.md from array of languages - -var jsonFiles = stripOutNonJsonFiles(fs.readdirSync(english)); -var stringsToLookFor = stringsWithInterpolations(jsonFiles); - -_(languages).each(function(lang) { - describe('the ' + lang + ' language', function() { - - _.each(stringsToLookFor, function(strings, file) { - context('string in ' + file + ' -', function() { - var translationFile = fs.readFileSync(locales + lang + '/' + file); - var parsedTranslationFile = JSON.parse(translationFile); - - var englishFile = fs.readFileSync(english + file); - var parsedEnglishFile = JSON.parse(englishFile); - - _.each(strings, function(value, key) { - var englishString = parsedEnglishFile[key]; - var translationString = parsedTranslationFile[key]; - - it(key + ' exists in ' + file + ' for the ' + lang + ' language', function() { - expect(translationString).to.exist; - }); - - if(!translationString) return; - - var EnglishOccurences = englishString.match(interpolationRegex); - var TranslationOccurences = translationString.match(interpolationRegex); - - it(key + ' - contains an interpolation', function() { - - if(!TranslationOccurences) console.log("English String:", englishString); - expect(EnglishOccurences).to.exist; - expect(TranslationOccurences).to.exist; - }); - - if(EnglishOccurences.length > 1) { - xit(key + ' - has the correct number of interpolations', function() { - if(!TranslationOccurences) console.log("English String:", englishString); - var numberOfOccurences = TranslationOccurences && TranslationOccurences.length; - expect(numberOfOccurences).to.eql(EnglishOccurences.length); - }); - } - }); - }); - }); - }); -}); - -function stringsWithInterpolations(json) { - var strings = {}; - - _(json).each(function(file_name) { - var raw_file = fs.readFileSync(english + file_name); - var parsed_json = JSON.parse(raw_file); - - strings[file_name] = {}; - _.each(parsed_json, function(value, key) { - var match = value.match(interpolationRegex); - if(match) strings[file_name][key] = true; - }); - }); - - return strings; -} - -function stripOutNonJsonFiles(collection) { - var onlyJson = _.filter(collection, function(file) { - return file.match(/[a-zA-Z]*\.json/); - }); - - return onlyJson; -} From abb19d1d202235cbd24bac6338a9ebf170d6e6fb Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 07:33:14 -0500 Subject: [PATCH 19/27] Correct casing on config example --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index 6836f38e74..55adbe801f 100644 --- a/config.json.example +++ b/config.json.example @@ -40,7 +40,7 @@ "accessKeyId":"accessKeyId", "secretAccessKey":"secretAccessKey" }, - "transifex_slack": { + "TRANSIFEX_SLACK": { "url": "https://hooks.slack.com/services/some-url", "channel": "general" }, From c8af280ce4d5b43225d14e057e613302abe12b20 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 07:33:27 -0500 Subject: [PATCH 20/27] Revert formatting back to backticks --- tasks/gulp-transifex-test.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index d856ed4f25..157021ee3a 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -37,7 +37,7 @@ gulp.task('transifex:missingStrings', () => { eachTranslationString(ALL_LANGUAGES, (language, filename, key, englishString, translationString) => { if (!translationString) { - let errorString = `${language} - ${filename} - ${key} - \`${englishString}\``; + let errorString = `${language} - ${filename} - ${key} - ${englishString}`; missingStrings.push(errorString); } }); @@ -72,10 +72,10 @@ gulp.task('transifex:malformedStrings', () => { let translationOccurences = translationString.match(interpolationRegex); if (!translationOccurences) { - let malformedString = `${lang} - ${file} - ${key} - \`${translationString}\``; + let malformedString = `${lang} - ${file} - ${key} - ${translationString}`; stringsWithMalformedInterpolations.push(malformedString); } else if (englishOccurences.length !== translationOccurences.length) { - let missingInterploationString = `${lang} - ${file} - ${key} - \`${translationString}\``; + let missingInterploationString = `${lang} - ${file} - ${key} - ${translationString}`; stringsWithIncorrectNumberOfInterpolations.push(missingInterploationString); } }); @@ -160,8 +160,9 @@ function post(message, items) { function formatMessageForPosting(msg, items) { let body = `*Warning:* ${msg}`; - body += '\n\n>>>\n'; + body += '\n\n```\n'; body += items.join('\n'); + body += '\n```'; return body; } From cb89f07dcf4b1cfb1c6f0e5221ef5b9a2b006384 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 07:42:52 -0500 Subject: [PATCH 21/27] Add clarification about missing interpolations --- tasks/gulp-transifex-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 157021ee3a..94a5076874 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -83,7 +83,7 @@ gulp.task('transifex:malformedStrings', () => { }); if (!_.isEmpty(stringsWithMalformedInterpolations)) { - let message = '*Warning:* The following strings have malformed interpolations'; + let message = '*Warning:* The following strings have malformed or missing interpolations'; post(message, stringsWithMalformedInterpolations); } From 6ed3e77008f7a50b1bd01ed47673b776898fbe11 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 07:53:26 -0500 Subject: [PATCH 22/27] Remove double warning --- tasks/gulp-transifex-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 94a5076874..24657b7a09 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -83,12 +83,12 @@ gulp.task('transifex:malformedStrings', () => { }); if (!_.isEmpty(stringsWithMalformedInterpolations)) { - let message = '*Warning:* The following strings have malformed or missing interpolations'; + let message = 'The following strings have malformed or missing interpolations'; post(message, stringsWithMalformedInterpolations); } if (!_.isEmpty(stringsWithIncorrectNumberOfInterpolations)) { - let message = '*Warning:* The following strings have a different number of string interpolations'; + let message = 'The following strings have a different number of string interpolations'; post(message, stringsWithIncorrectNumberOfInterpolations); } }); From ee621b526e09bbe25d0bdfe2bbb8fbc5710634ad Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 17:14:46 -0500 Subject: [PATCH 23/27] Remove unused dependency --- package.json | 1 - tasks/gulp-transifex-test.js | 1 - 2 files changed, 2 deletions(-) diff --git a/package.json b/package.json index 5f0b9ef6c8..b387c0f638 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,6 @@ "karma-requirejs": "~0.2.0", "karma-script-launcher": "~0.1.0", "lcov-result-merger": "^1.0.2", - "md5": "^2.0.0", "mocha": "~1.12.1", "mongoskin": "~0.6.1", "phantomjssmith": "~0.5.4", diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 24657b7a09..7d1f8a6f26 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -1,7 +1,6 @@ import fs from 'fs'; import _ from 'lodash'; import nconf from 'nconf'; -import md5 from 'md5'; import gulp from 'gulp'; import request from 'superagent'; From ae2a9f57560c64363a090f356d337cae3879354b Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 17:16:50 -0500 Subject: [PATCH 24/27] remove unused functions --- tasks/gulp-transifex-test.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 7d1f8a6f26..37d4557609 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -99,14 +99,6 @@ function getArrayOfLanguages() { return languages; } -function getNonEnglishLanguages() { - let nonEnglishLanguages = _.filter(ALL_LANGUAGES, (lang) => { - return lang.indexOf('en') !== 0; - }); - - return nonEnglishLanguages; -} - function eachTranslationFile(languages, cb) { let jsonFiles = stripOutNonJsonFiles(fs.readdirSync(ENGLISH_LOCALE)); @@ -137,11 +129,6 @@ function eachTranslationString(languages, cb) { }); } -function checkForIssues(thingToCheck) { - let hasIssues = !_.isEmpty(thingToCheck); - return hasIssues; -} - function post(message, items) { let formattedMessage = formatMessageForPosting(message, items); From c74625a624be21920ef343ad0d4a9257a3faf9f1 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 17:17:27 -0500 Subject: [PATCH 25/27] Add line break for styling consistency --- tasks/gulp-transifex-test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 37d4557609..c4e52466ca 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -16,6 +16,7 @@ const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; gulp.task('transifex:missingFiles', () => { + let missingStrings = []; eachTranslationFile(ALL_LANGUAGES, (error) => { From c46659c6228a6f6cd27aa33a7a9cedcba21ab885 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Mon, 10 Aug 2015 17:31:43 -0500 Subject: [PATCH 26/27] Add top level transifex task --- tasks/gulp-transifex-test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index c4e52466ca..3c07fab67a 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -15,6 +15,8 @@ const SLACK_CHANNEL = '#' + (nconf.get('TRANSIFEX_SLACK:channel') || 'general'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; +gulp.task('transifex', ['transifex:missingFiles', 'transifex:missingStrings', 'transifex:malformedStrings']); + gulp.task('transifex:missingFiles', () => { let missingStrings = []; From 9d9e99045db801c041aad9fd257565a41dfcb0e6 Mon Sep 17 00:00:00 2001 From: Blade Barringer Date: Tue, 11 Aug 2015 08:12:03 -0500 Subject: [PATCH 27/27] Add exceptions to not check for malformities --- tasks/gulp-transifex-test.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tasks/gulp-transifex-test.js b/tasks/gulp-transifex-test.js index 3c07fab67a..72b51f54d6 100644 --- a/tasks/gulp-transifex-test.js +++ b/tasks/gulp-transifex-test.js @@ -15,6 +15,12 @@ const SLACK_CHANNEL = '#' + (nconf.get('TRANSIFEX_SLACK:channel') || 'general'); const SLACK_USERNAME = 'Transifex'; const SLACK_EMOJI = ':transifex:'; +const malformedStringExceptions = { + messageDropFood: true, + armoireFood: true, + feedPet: true +} + gulp.task('transifex', ['transifex:missingFiles', 'transifex:missingStrings', 'transifex:malformedStrings']); gulp.task('transifex:missingFiles', () => { @@ -76,7 +82,7 @@ gulp.task('transifex:malformedStrings', () => { if (!translationOccurences) { let malformedString = `${lang} - ${file} - ${key} - ${translationString}`; stringsWithMalformedInterpolations.push(malformedString); - } else if (englishOccurences.length !== translationOccurences.length) { + } else if (englishOccurences.length !== translationOccurences.length && !malformedStringExceptions[key]) { let missingInterploationString = `${lang} - ${file} - ${key} - ${translationString}`; stringsWithIncorrectNumberOfInterpolations.push(missingInterploationString); }