Compress and lazy load locales (#2603)

* Compress and lazy load locales

* Remove index.html when loading the locales on the web

* Fix locale output path for web build

_scripts/ProcessLocalesPlugin.js

@@ -0,0 +1,96 @@
+const { existsSync, readFileSync } = require('fs')
+const { brotliCompressSync, constants } = require('zlib')
+const { load: loadYaml } = require('js-yaml')
+
+class ProcessLocalesPlugin {
+  constructor(options = {}) {
+    this.compress = !!options.compress
+
+    if (typeof options.inputDir !== 'string') {
+      throw new Error('ProcessLocalesPlugin: no input directory `inputDir` specified.')
+    } else if (!existsSync(options.inputDir)) {
+      throw new Error('ProcessLocalesPlugin: the specified input directory does not exist.')
+    }
+    this.inputDir = options.inputDir
+
+    if (typeof options.outputDir !== 'string') {
+      throw new Error('ProcessLocalesPlugin: no output directory `outputDir` specified.')
+    }
+    this.outputDir = options.outputDir
+
+    this.localeNames = []
+
+    this.loadLocales()
+  }
+
+  apply(compiler) {
+    compiler.hooks.thisCompilation.tap('ProcessLocalesPlugin', (compilation) => {
+
+      const { RawSource } = compiler.webpack.sources;
+
+      compilation.hooks.processAssets.tapPromise({
+        name: 'process-locales-plugin',
+        stage: compiler.webpack.Compilation.PROCESS_ASSETS_STAGE_ADDITIONAL
+      },
+        async (_assets) => {
+          const promises = []
+
+          for (const { locale, data } of this.locales) {
+            promises.push(new Promise((resolve) => {
+              if (Object.prototype.hasOwnProperty.call(data, 'Locale Name')) {
+                delete data['Locale Name']
+              }
+
+              let filename = `${this.outputDir}/${locale}.json`
+              let output = JSON.stringify(data)
+
+              if (this.compress) {
+                filename += '.br'
+                output = this.compressLocale(output)
+              }
+
+              compilation.emitAsset(
+                filename,
+                new RawSource(output),
+                { minimized: true }
+              )
+
+              resolve()
+            }))
+          }
+
+          await Promise.all(promises)
+
+          this.locales = null
+        })
+    })
+  }
+
+  loadLocales() {
+    this.locales = []
+
+    const activeLocales = JSON.parse(readFileSync(`${this.inputDir}/activeLocales.json`))
+
+    for (const locale of activeLocales) {
+      const contents = readFileSync(`${this.inputDir}/${locale}.yaml`, 'utf-8')
+      const data = loadYaml(contents)
+
+      this.localeNames.push(data['Locale Name'] ?? locale)
+      this.locales.push({ locale, data })
+    }
+  }
+
+  compressLocale(data) {
+    const buffer = Buffer.from(data, 'utf-8')
+
+    return brotliCompressSync(buffer, {
+      params: {
+        [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
+        [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MAX_QUALITY,
+        [constants.BROTLI_PARAM_SIZE_HINT]: buffer.byteLength
+      }
+    })
+  }
+}
+
+module.exports = ProcessLocalesPlugin

_scripts/webpack.main.config.js

@@ -27,7 +27,9 @@ const config = {
   optimization: {
     minimizer: [
       '...', // extend webpack's list instead of overwriting it
-      new JsonMinimizerPlugin()
+      new JsonMinimizerPlugin({
+        exclude: /\/locales\/.*\.json/
+      })
     ]
   },
   node: {
@@ -70,7 +72,7 @@ if (isDevMode) {
             to: path.join(__dirname, '../dist/static'),
             globOptions: {
               dot: true,
-              ignore: ['**/.*', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'],
+              ignore: ['**/.*', '**/locales/**', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'],
             },
           },
       ]

_scripts/webpack.renderer.config.js

@@ -4,6 +4,7 @@ const HtmlWebpackPlugin = require('html-webpack-plugin')
 const VueLoaderPlugin = require('vue-loader/lib/plugin')
 const MiniCssExtractPlugin = require('mini-css-extract-plugin')
 const CssMinimizerPlugin = require('css-minimizer-webpack-plugin')
+const ProcessLocalesPlugin = require('./ProcessLocalesPlugin')
 
 const { productName } = require('../package.json')
 
@@ -151,6 +152,19 @@ if (isDevMode) {
       __static: `"${path.join(__dirname, '../static').replace(/\\/g, '\\\\')}"`,
     })
   )
+} else {
+  const processLocalesPlugin = new ProcessLocalesPlugin({
+    compress: true,
+    inputDir: path.join(__dirname, '../static/locales'),
+    outputDir: 'static/locales',
+  })
+
+  config.plugins.push(
+    processLocalesPlugin,
+    new webpack.DefinePlugin({
+      'process.env.LOCALE_NAMES': JSON.stringify(processLocalesPlugin.localeNames)
+    })
+  )
 }
 
 module.exports = config

_scripts/webpack.web.config.js

@@ -6,6 +6,7 @@ const CopyWebpackPlugin = require('copy-webpack-plugin')
 const MiniCssExtractPlugin = require('mini-css-extract-plugin')
 const JsonMinimizerPlugin = require('json-minimizer-webpack-plugin')
 const CssMinimizerPlugin = require('css-minimizer-webpack-plugin')
+const ProcessLocalesPlugin = require('./ProcessLocalesPlugin')
 
 const { productName } = require('../package.json')
 
@@ -105,7 +106,9 @@ const config = {
   optimization: {
     minimizer: [
       '...', // extend webpack's list instead of overwriting it
-      new JsonMinimizerPlugin(),
+      new JsonMinimizerPlugin({
+        exclude: /\/locales\/.*\.json/
+      }),
       new CssMinimizerPlugin()
     ]
   },
@@ -160,7 +163,17 @@ if (isDevMode) {
     })
   )
 } else {
+  const processLocalesPlugin = new ProcessLocalesPlugin({
+    compress: false,
+    inputDir: path.join(__dirname, '../static/locales'),
+    outputDir: 'static/locales',
+  })
+
   config.plugins.push(
+    processLocalesPlugin,
+    new webpack.DefinePlugin({
+      'process.env.LOCALE_NAMES': JSON.stringify(processLocalesPlugin.localeNames)
+    }),
     new CopyWebpackPlugin({
         patterns: [
           {
@@ -172,7 +185,7 @@ if (isDevMode) {
             to: path.join(__dirname, '../dist/web/static'),
             globOptions: {
               dot: true,
-              ignore: ['**/.*', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'],
+              ignore: ['**/.*', '**/locales/**', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'],
             },
           },
       ]

package.json

@@ -58,7 +58,6 @@
     "http-proxy-agent": "^4.0.1",
     "https-proxy-agent": "^5.0.0",
     "jquery": "^3.6.0",
-    "js-yaml": "^4.1.0",
     "lodash.debounce": "^4.0.8",
     "lodash.isequal": "^4.5.0",
     "marked": "^4.0.17",
@@ -105,6 +104,7 @@
     "eslint-plugin-standard": "^5.0.0",
     "eslint-plugin-vue": "^7.17.0",
     "html-webpack-plugin": "^5.3.2",
+    "js-yaml": "^4.1.0",
     "json-minimizer-webpack-plugin": "^4.0.0",
     "mini-css-extract-plugin": "^2.2.2",
     "npm-run-all": "^4.1.5",

src/renderer/components/ft-video-player/ft-video-player.js

@@ -138,7 +138,7 @@ export default Vue.extend({
   },
   computed: {
     currentLocale: function () {
-      return this.$store.getters.getCurrentLocale
+      return this.$i18n.locale
     },
 
     defaultPlayback: function () {

src/renderer/components/general-settings/general-settings.js

@@ -111,7 +111,7 @@ export default Vue.extend({
     },
 
     localeOptions: function () {
-      return ['system'].concat(Object.keys(this.$i18n.messages))
+      return ['system'].concat(this.$i18n.allLocales)
     },
 
     localeNames: function () {
@@ -119,6 +119,7 @@ export default Vue.extend({
         this.$t('Settings.General Settings.System Default')
       ]
 
+      if (process.env.NODE_ENV === 'development') {
         Object.entries(this.$i18n.messages).forEach(([locale, localeData]) => {
           const localeName = localeData['Locale Name']
           if (typeof localeName !== 'undefined') {
@@ -127,6 +128,9 @@ export default Vue.extend({
             names.push(locale)
           }
         })
+      } else {
+        names.push(...process.env.LOCALE_NAMES)
+      }
 
       return names
     },

src/renderer/i18n/index.js

@@ -1,31 +1,88 @@
 import Vue from 'vue'
 import VueI18n from 'vue-i18n'
-import yaml from 'js-yaml'
 import fs from 'fs'
 
+// List of locales approved for use
+import activeLocales from '../../../static/locales/activeLocales.json'
+
 const isDev = process.env.NODE_ENV === 'development'
 
-Vue.use(VueI18n)
-
-// List of locales approved for use
-const activeLocales = ['en-US', 'en_GB', 'ar', 'bg', 'ca', 'cs', 'da', 'de-DE', 'el', 'es', 'es_AR', 'es-MX', 'et', 'eu', 'fi', 'fr-FR', 'gl', 'he', 'hu', 'hr', 'id', 'is', 'it', 'ja', 'ko', 'lt', 'nb_NO', 'nl', 'nn', 'pl', 'pt', 'pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'tr', 'uk', 'vi', 'zh-CN', 'zh-TW']
 const messages = {}
-/* eslint-disable-next-line */
-const fileLocation = isDev ? 'static/locales/' : `${__dirname}/static/locales/`
 
-// Take active locales and load respective YAML file
+if (isDev) {
-activeLocales.forEach((locale) => {
+  const { load } = require('js-yaml')
+  // Take active locales and load respective YAML file
+  activeLocales.forEach((locale) => {
     try {
       // File location when running in dev
-    const doc = yaml.load(fs.readFileSync(`${fileLocation}${locale}.yaml`))
+      const doc = load(fs.readFileSync(`static/locales/${locale}.yaml`))
       messages[locale] = doc
     } catch (e) {
       console.error(e)
     }
-})
+  })
+}
 
-export default new VueI18n({
+class CustomVueI18n extends VueI18n {
+  constructor(options) {
+    super(options)
+    this.allLocales = activeLocales
+  }
+
+  async loadLocale(locale) {
+    // we only lazy load locales for producation builds
+    if (!isDev) {
+      // don't need to load it if it's already loaded
+      if (this.availableLocales.includes(locale)) {
+        return
+      }
+      if (!this.allLocales.includes(locale)) {
+        console.error(`Unable to load unknown locale: "${locale}"`)
+      }
+
+      if (process.env.IS_ELECTRON) {
+        const { brotliDecompressSync } = require('zlib')
+        // locales are only compressed in our production Electron builds
+        try {
+          // decompress brotli compressed json file and then load it
+          // eslint-disable-next-line node/no-path-concat
+          const compressed = fs.readFileSync(`${__dirname}/static/locales/${locale}.json.br`)
+          const data = JSON.parse(brotliDecompressSync(compressed).toString())
+          this.setLocaleMessage(locale, data)
+        } catch (err) {
+          console.error(err)
+        }
+      } else {
+        const url = new URL(window.location.href)
+        url.hash = ''
+        if (url.pathname.endsWith('index.html')) {
+          url.pathname = url.pathname.replace(/index\.html$/, '')
+        }
+
+        if (url.pathname) {
+          url.pathname += `/static/locales/${locale}.json`
+        } else {
+          url.pathname = `/static/locales/${locale}.json`
+        }
+
+        const response = await fetch(url)
+        const data = await response.json()
+        this.setLocaleMessage(locale, data)
+      }
+    }
+  }
+}
+
+Vue.use(CustomVueI18n)
+
+const i18n = new CustomVueI18n({
   locale: 'en-US',
   fallbackLocale: { default: 'en-US' },
-  messages: messages
+  messages
 })
+
+if (!isDev) {
+  i18n.loadLocale('en-US')
+}
+
+export default i18n

src/renderer/store/modules/settings.js

@@ -281,7 +281,7 @@ const stateWithSideEffects = {
       if (value === 'system') {
         const systemLocaleName = (await dispatch('getSystemLocale')).replace('-', '_') // ex: en_US
         const systemLocaleLang = systemLocaleName.split('_')[0] // ex: en
-        const targetLocaleOptions = Object.keys(i18n.messages).filter((locale) => { // filter out other languages
+        const targetLocaleOptions = i18n.allLocales.filter((locale) => { // filter out other languages
           const localeLang = locale.replace('-', '_').split('_')[0]
           return localeLang.includes(systemLocaleLang)
         }).sort((a, b) => {
@@ -317,6 +317,10 @@ const stateWithSideEffects = {
         }
       }
 
+      if (process.env.NODE_ENV !== 'development') {
+        await i18n.loadLocale(targetLocale)
+      }
+
       i18n.locale = targetLocale
       dispatch('getRegionData', {
         isDev: process.env.NODE_ENV === 'development',

src/renderer/views/SubscribedChannels/SubscribedChannels.js

@@ -60,7 +60,7 @@ export default Vue.extend({
     },
 
     locale: function () {
-      return this.$store.getters.getCurrentLocale.replace('_', '-')
+      return this.$i18n.locale.replace('_', '-')
     },
 
     backendPreference: function () {

static/locales/activeLocales.json

@@ -0,0 +1,46 @@
+[
+  "en-US",
+  "en_GB",
+  "ar",
+  "bg",
+  "ca",
+  "cs",
+  "da",
+  "de-DE",
+  "el",
+  "es",
+  "es_AR",
+  "es-MX",
+  "et",
+  "eu",
+  "fi",
+  "fr-FR",
+  "gl",
+  "he",
+  "hu",
+  "hr",
+  "id",
+  "is",
+  "it",
+  "ja",
+  "ko",
+  "lt",
+  "nb_NO",
+  "nl",
+  "nn",
+  "pl",
+  "pt",
+  "pt-BR",
+  "pt-PT",
+  "ro",
+  "ru",
+  "sk",
+  "sl",
+  "sr",
+  "sv",
+  "tr",
+  "uk",
+  "vi",
+  "zh-CN",
+  "zh-TW"
+]