From 20dbd50ad6c7c56e5b86572757ac920778fee3be Mon Sep 17 00:00:00 2001 From: Motin Date: Thu, 10 Jun 2021 23:58:23 +0300 Subject: [PATCH 01/31] Updates to reflect the recent repo rename (#233) * Update repo url across docs and source files * Updated npm package metadata * Updated mean readme title --- README.md | 4 +- docs/INSTALL.md | 6 +- package.json | 4 +- src/core/package.json | 2 +- .../background.js/telemetry/Telemetry.ts | 2 +- .../telemetry/generated/metrics.md | 2 +- .../background.js/telemetry/metrics.yaml | 96 +++++++++---------- .../background.js/telemetry/pings.yaml | 2 +- src/cross-browser-ui/package.json | 2 +- src/firefox-infobar-ui/package.json | 2 +- 10 files changed, 61 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index c1bfd5ad..ab991ad3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Bergamot Translate Browser Extension +# Firefox Translations Web Extension -[![CircleCI badge](https://img.shields.io/circleci/project/github/mozilla-extensions/bergamot-browser-extension/main.svg?label=CircleCI)](https://circleci.com/gh/mozilla-extensions/bergamot-browser-extension/) +[![CircleCI badge](https://img.shields.io/circleci/project/github/mozilla-extensions/firefox-translations/main.svg?label=CircleCI)](https://circleci.com/gh/mozilla-extensions/firefox-translations/) For more information see [The Bergamot Project website](https://browser.mt/). diff --git a/docs/INSTALL.md b/docs/INSTALL.md index d46e302f..eab8dd47 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -15,7 +15,7 @@ This document outlines how to get the v0.4.2 pre-release version of the extension running locally. -Before installing, please take a minute to read [the release notes](https://github.com/mozilla-extensions/bergamot-browser-extension/releases/tag/v0.4.2). +Before installing, please take a minute to read [the release notes](https://github.com/mozilla-extensions/firefox-translations/releases/tag/v0.4.2). - - -**Contents** - -- [Developing this extension](#developing-this-extension) - - [First time setup](#first-time-setup) - - [Using known-to-work Bergamot Translator WASM artifacts and importing them to the extension](#using-known-to-work-bergamot-translator-wasm-artifacts-and-importing-them-to-the-extension) - - [Building Bergamot Translator WASM artifacts and importing them to the extension](#building-bergamot-translator-wasm-artifacts-and-importing-them-to-the-extension) - - [Creating extension builds for distribution](#creating-extension-builds-for-distribution) - - [Development mode](#development-mode) - - [Firefox - Infobar UI](#firefox---infobar-ui) - - [Chrome - Cross-browser UI](#chrome---cross-browser-ui) - - [Firefox - Cross-browser UI](#firefox---cross-browser-ui) - - [Creating a signed build of the extension for self-distribution](#creating-a-signed-build-of-the-extension-for-self-distribution) - - [Run end-to-end tests](#run-end-to-end-tests) - - [Locally](#locally) - - [Continuous Integration](#continuous-integration) - - [Troubleshooting functional tests](#troubleshooting-functional-tests) - - [Troubleshooting the extension when it is running](#troubleshooting-the-extension-when-it-is-running) - - [Firefox - Infobar UI](#firefox---infobar-ui-1) - - [Firefox - Cross-browser UI](#firefox---cross-browser-ui-1) - - [Upgrading Glean.js](#upgrading-gleanjs) - - [Analyze webpack bundle size](#analyze-webpack-bundle-size) - - [Firefox - Infobar UI](#firefox---infobar-ui-2) - - [Chrome - Cross-browser UI](#chrome---cross-browser-ui-1) - - [Firefox - Cross-browser UI](#firefox---cross-browser-ui-2) - - [Opening up specific extension pages](#opening-up-specific-extension-pages) - - [Enabling error reporting via Sentry](#enabling-error-reporting-via-sentry) - - - -# Developing this extension - -The commands in these instructions are meant to be run in the root folder in this repo. - -## First time setup - -Install dependencies using [yarn v1](https://classic.yarnpkg.com/en/docs/install/): - -```bash -yarn install -``` - -## Using known-to-work Bergamot Translator WASM artifacts and importing them to the extension - -To use artifacts that are known to work (built by bergamot-translator's CI): - -```bash -yarn bergamot-translator:download-and-import -``` - -Note: Once this command has run, it will create a folder called `downloaded-bergamot-translator-wasm-artifacts` and if this folder exists, it will not re-download the artifacts again. Thus, to make sure that you are using the most up to date WASM artifacts, remove the `downloaded-bergamot-translator-wasm-artifacts` folder, then re-run the command. - -## Building Bergamot Translator WASM artifacts and importing them to the extension - -If you want to try out custom changes to [bergamot-translator](../bergamot-translator/README.md), first make sure that you have checked out the submodules: - -``` -git submodule update --init --recursive -``` - -Then, follow the upstream instructions ([bergamot-translator/README](../bergamot-translator/README.md) for setting up an environment that successfully builds bergamot-translator. - -When all is properly set-up, you should be able to run the following to build and import the custom WASM artifacts into the extension: - -```bash -./bergamot-translator/build-wasm.sh -./import-bergamot-translator.sh ./bergamot-translator/build-wasm/ -``` - -Re-run this command any time there has been an update in the bergamot-translator submodule. - -## Creating extension builds for distribution - -To build for Firefox: - -```bash -yarn build:cross-browser-ui:firefox -``` - -The build artifact will be created under `dist/production/firefox/cross-browser-ui`. - -For Chrome: - -```bash -yarn build:cross-browser-ui:chrome -``` - -The build artifact will be created under `dist/production/chrome/cross-browser-ui`. - -To build the Firefox native UI variant: - -```bash -yarn build:firefox-infobar-ui -``` - -The build artifact will be created under `dist/production/firefox/firefox-infobar-ui`. - -## Development mode - -First, import bergamot models locally, since the extension downloads the models from a local endpoint when running in development mode: - -```bash -yarn bergamot-models:import -``` - -If you haven't already, download and install Firefox Nightly from [here](https://www.mozilla.org/en-US/firefox/channel/desktop/) before running the below commands. - -Finally, use the commands below to build the extension, launch the browser, install the extension and start Webpack in watch mode, which repeats the build process and reloads the extension when source files are changed. - -### Firefox - Infobar UI - -```bash -yarn watch:firefox-infobar-ui -``` - -### Chrome - Cross-browser UI - -```bash -yarn watch:cross-browser-ui:chrome -``` - -### Firefox - Cross-browser UI - -```bash -yarn watch:cross-browser-ui:firefox -``` - -## Creating a signed build of the extension for self-distribution - -After version bumping and setting the API_KEY and API_SECRET env vars: - -```bash -yarn build:cross-browser-ui:firefox && npx web-ext sign --api-key $API_KEY --api-secret $API_SECRET -``` - -Note: This is for Firefox version with the cross-browser UI only. Chrome Web Store does not offer signed builds for self-distribution and the Firefox infobar UI version gets signed via a separate process. - -## Run end-to-end tests - -### Locally - -Before running the tests locally for the first time, install [mitmproxy](https://mitmproxy.org). Example installation command for Mac: - -```bash -brew install mitmproxy -``` - -To run the end-to-end tests, run: - -```bash -yarn e2e-tests -``` - -Note, to modify the `test/e2e/intercept-telemetry-requests.py` script, it may be useful to have the Python deps installed: - -```bash -test/e2e/setup-python-venv.sh -``` - -### Continuous Integration - -End-to-end tests are run against each new commits/PRs. Read more about the current CI setup [here](./CI.md). - -### Troubleshooting functional tests - -**Basic principles** - -End-to-end tests are run using the built extension artifacts found in `dist/production/`. To test new non-test-related code changes, remember to re-run the relevant build command. - -**Intervening** - -If you want to intervene in a test (eg. to double-check something), follow this pattern: - -1. Add a long delay, eg `await driver.sleep(60 * 60 * 1000);` to the test at the place you want to intervene. -2. Make sure to temporarily also increase the timeout for the test you are running. -3. Run the tests and intervene manually as desired. - -**Obtaining Geckodriver logs** - -To troubleshoot issues with failing tests when only cryptic error messages are available, check the geckodriver logs, located in `test/e2e/results/logs/`. - -## Troubleshooting the extension when it is running - -Hint: To produce a clean log output for forwarding to developers / attaching to issues, first click the Trash can icon in the top left before repeating the steps that leads to the erroneous behavior. - -### Firefox - Infobar UI - -1. Go to `about:devtools-toolbox?type=extension&id=firefox-translations%40mozilla.org` -2. Click Console - -### Firefox - Cross-browser UI - -1. Go to `about:devtools-toolbox?type=extension&id=bergamot-browser-extension%40browser.mt` -2. Click Console - -## Upgrading Glean.js - -1. Bump the `@mozilla/glean` version in `package.json` -2. `yarn` -3. `yarn generate-glean` -4. Commit all changes. Use a commit message in the format of `Upgrade Glean.js to vx.y.z` - -## Analyze webpack bundle size - -### Firefox - Infobar UI - -```bash -yarn build:firefox-infobar-ui -npx webpack-bundle-analyzer build/production/firefox/firefox-infobar-ui.stats.json build/production/firefox/firefox-infobar-ui -``` - -### Chrome - Cross-browser UI - -```bash -yarn build:cross-browser-ui:chrome -npx webpack-bundle-analyzer build/production/chrome/cross-browser-ui.stats.json build/production/chrome/cross-browser-ui -``` - -### Firefox - Cross-browser UI - -```bash -yarn build:cross-browser-ui:firefox -npx webpack-bundle-analyzer build/production/firefox/cross-browser-ui.stats.json build/production/firefox/cross-browser-ui -``` - -## Opening up specific extension pages - -(Only relevant when developing the Cross-browser UI) - -From the background context: - -```javascript -(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( - `main-interface/popup.html`, -); -``` - -```javascript -(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( - `main-interface/popup.html#/components`, -); -``` - -```javascript -(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( - `get-started/get-started.html`, -); -``` - -```javascript -(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( - `options-ui/options-ui.html`, -); -``` - -## Enabling error reporting via Sentry - -Configure `.env.development` and `.env.production` with values as per https://docs.sentry.io/cli/configuration/. - -Hint: You can verify that the Sentry configuration is working using: - -```bash -cp .env.development .env -npx sentry-cli info -cp .env.production .env -npx sentry-cli info -``` diff --git a/docs/bin/build-docs.sh b/docs/bin/build-docs.sh new file mode 100755 index 00000000..7ee66312 --- /dev/null +++ b/docs/bin/build-docs.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Build all docs with one command +# Documentation will be placed in `build/docs`. + +set -xe + +DEST_PATH=../build/docs + +# To cleanly apply some preprocessing, we copy all src files to an +# intermediate path before running mdbook +INTERMEDIATE_SRC_PATH=../build/tmp-docs + +# Clean +rm -rf $INTERMEDIATE_SRC_PATH +mkdir -p $INTERMEDIATE_SRC_PATH +rm -rf $DEST_PATH +mkdir -p $DEST_PATH + +# Copy src files to the intermediate source dir +cp -r dev shared user $INTERMEDIATE_SRC_PATH/ + +# Add npm assets necessary at build time +cp ../node_modules/mermaid/dist/mermaid.min.js $INTERMEDIATE_SRC_PATH/shared/ + +# TODO: Add telemetry docs +# TODO: Add data-review docs? + +# Build the docs +cat dev/SUMMARY.tpl.md dev/api-docs-summary.md > $INTERMEDIATE_SRC_PATH/dev/SUMMARY.md +cd $INTERMEDIATE_SRC_PATH +output=$(mdbook build user/ 2>&1) +if echo "$output" | grep -q "\[ERROR\]" ; then + exit 1 +fi +output=$(mdbook build dev/ 2>&1) +if echo "$output" | grep -q "\[ERROR\]" ; then + exit 1 +fi +cd - + +# Clean up intermediate src files +rm -rf $INTERMEDIATE_SRC_PATH + +# Add redirect from the root page to the user docs +echo '' > $DEST_PATH/index.html diff --git a/docs/bin/generate-api-docs.sh b/docs/bin/generate-api-docs.sh new file mode 100755 index 00000000..c576a934 --- /dev/null +++ b/docs/bin/generate-api-docs.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Build API docs +# Documentation will be placed in `dev/api-docs` for inclusion in the +# developer documentation. + +set -e + +REPO_PACKAGE_VERSION=$(node -p "require('../package.json').version") + +typedoc --options typedoc.json --gitRevision "v$REPO_PACKAGE_VERSION" + +cd dev + +# Remove the first line (includes breadcrumbs) from the individual pages +# since it is superfluous when results are rendered in mdBook and the +# links in the breadcrumbs cause some linkchecking warnings +for FILE in api-docs/**/*.md; do + tail -n +2 "$FILE" > "$FILE.tmp" && mv "$FILE.tmp" "$FILE" +done + +# Create links to all the individual pages in a file that eventually +# ends up in SUMMARY.md (an mdBook requirement is that all files +# are linked in SUMMARY.md) +rm api-docs-summary.md || true +touch api-docs-summary.md +for f in api-docs/**/*.md; do + b=$(basename $f) + echo " - [$b](./$f)" >> api-docs-summary.md +done diff --git a/docs/bin/spellcheck.sh b/docs/bin/spellcheck.sh new file mode 100755 index 00000000..caf02b46 --- /dev/null +++ b/docs/bin/spellcheck.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# Copied, and modified, from the Rust Cookbook. +# https://github.com/rust-lang-nursery/rust-cookbook/blob/HEAD/ci/spellcheck.sh + +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +aspell --version + +# Checks project markdown files for spell errors + +# Notes: + +# This script needs dictionary file ($dict_filename) with project-specific +# valid words. If this file is missing, first invocation of a script generates +# a file of words considered typos at the moment. User should remove real typos +# from this file and leave only valid words. When script generates false +# positive after source modification, new valid word should be added +# to dictionary file. + +# Default mode of this script is interactive. Each source file is scanned for +# typos. aspell opens window, suggesting fixes for each found typo. Original +# files with errors will be backed up to files with format "filename.md.bak". + +# When running in CI, this script should be run in "list" mode (pass "list" +# as first argument). In this mode script scans all files and reports found +# errors. Exit code in this case depends on scan result: +# 1 if any errors found, +# 0 if all is clear. + +# Script skips words with length less than or equal to 3. This helps to avoid +# some false positives. + +# We can consider skipping source code in markdown files (```code```) to reduce +# rate of false positives, but then we lose ability to detect typos in code +# comments/strings etc. + +shopt -s nullglob + +dict_filename=./.dictionary +markdown_sources="$(git ls-files | grep '\.md$')" +mode="check" +aspell_args="--lang en_US --mode=markdown" + +# aspell repeatedly modifies personal dictionary for some purpose, +# so we should use a copy of our dictionary +mkdir -p "/tmp/ci" +dict_path="/tmp/$dict_filename" + +if [[ "$1" == "list" ]]; then + mode="list" +fi + +if [[ ! -f "$dict_filename" ]]; then + # Pre-check mode: generates dictionary of words aspell consider typos. + # After user validates that this file contains only valid words, we can + # look for typos using this dictionary and some default aspell dictionary. + echo "Scanning files to generate dictionary file '$dict_filename'." + echo "Please check that it doesn't contain any misspellings." + + echo "personal_ws-1.1 en 0 utf-8" > "$dict_filename" + cat ${markdown_sources} | aspell ${aspell_args} list | sort -u >> "$dict_filename" +elif [[ "$mode" == "list" ]]; then + # List (default) mode: scan all files, report errors + declare -i retval=0 + + cp "$dict_filename" "$dict_path" + + if [ ! -f $dict_path ]; then + retval=1 + exit "$retval" + fi + + for fname in $markdown_sources; do + command=$(aspell ${aspell_args} --personal="$dict_path" "$mode" < "$fname") + if [[ -n "$command" ]]; then + for error in $command; do + # FIXME: Find more correct way to get line number + # (ideally from aspell). Now it can make some false positives, + # because it is just a grep + grep --with-filename --line-number -o --color=always "$error" "$fname" + done + retval=1 + fi + done + exit "$retval" +elif [[ "$mode" == "check" ]]; then + for fname in $markdown_sources; do + aspell --mode=markdown ${aspell_args} --dont-backup --personal="$dict_filename" "$mode" "$fname" + done +fi diff --git a/docs/dev/README.md b/docs/dev/README.md new file mode 100644 index 00000000..2cdb7ec5 --- /dev/null +++ b/docs/dev/README.md @@ -0,0 +1,45 @@ +# Introduction + +The **Firefox Translations Web Extension** ties together the various components that together form the **Firefox Translations** feature. + +The source code is available at [https://github.com/mozilla-extensions/firefox-translations/](). + +## About this guide + +This guide is meant for developers of the **Firefox Translations Web Extension**. + +For developer documentation related to any other part than the Web Extension (and what those parts may be), check the section called `Related repositories` below. + +For user documentation on the **Firefox Translations** feature, refer to [Firefox Translations User Guide](../user/index.html). + +## The main sections of this guide + +- **[Engineering](./engineering/overview.md)** - Details how to get your hands dirty and contribute to the codebase +- **[Maintenance](./maintenance/overview.md)** - Shorter tutorials on how to achieve specific maintenance tasks + +## Reporting bugs/issues + +To report engineering-related issues or request changes specific to the **Firefox Translations Web Extension**, [file a bug on GitHub](https://github.com/mozilla-extensions/firefox-translations/issues/). + +## Related repositories + +The other repositories that together build up the **Firefox Translations** feature are: + +- **Bergamot Translator** - The translation engine ([Source code](https://github.com/mozilla/bergamot-translator)) +- **Language Models** - The files that the translation engine uses to translate between languages ([Source code](https://github.com/mozilla-applied-ml/bergamot-models)) +- **Firefox** - Contains UI elements and optimizations specific to the translations feature ([Firefox source code documentation](https://firefox-source-docs.mozilla.org/)) + +### The relationship between the repositories + +
erDiagram
+    BROWSER ||--|| THIS-WEB-EXTENSION : bundles-or-installs
+    THIS-WEB-EXTENSION ||..|{ LANGUAGE-MODELS : downloads
+    THIS-WEB-EXTENSION ||..|| BERGAMOT-TRANSLATOR : downloads
+    THIS-WEB-EXTENSION ||--|| BERGAMOT-TRANSLATOR : consumes
+    BERGAMOT-TRANSLATOR ||--|{ LANGUAGE-MODELS : loads
+
+ +## License + +Firefox Translations Source Code is subject to the terms of the Mozilla Public License v2.0. +You can obtain a copy of the MPL at . diff --git a/docs/dev/SUMMARY.tpl.md b/docs/dev/SUMMARY.tpl.md new file mode 100644 index 00000000..e2a39c24 --- /dev/null +++ b/docs/dev/SUMMARY.tpl.md @@ -0,0 +1,22 @@ +[Introduction](README.md) + +# Developer's Guide + +- [Engineering](./engineering/overview.md) + + - [Getting started](./engineering/getting-started.md) + - [Local development](./engineering/local-development.md) + - [Building and installing the extension](./engineering/building-and-installing-the-extension.md) + - [Debugging](./engineering/debugging.md) + - [Improving the documentation](./engineering/improving-the-documentation.md) + - [Automated tests](./engineering/automated-web-extension-testing.md) + - [Continuous integration](./engineering/ci.md) + - [Contributing](./engineering/contributing.md) + +- [Maintenance](./maintenance/overview.md) + + - [Upgrading Glean.js](./maintenance/upgrading-glean-js.md) + +# Appendix + +- [Generated API Docs](./api-docs/README.md) diff --git a/docs/dev/architecture/overview.md b/docs/dev/architecture/overview.md new file mode 100644 index 00000000..e484b6e7 --- /dev/null +++ b/docs/dev/architecture/overview.md @@ -0,0 +1,3 @@ +# Architecture + +This section contains a description of the main components and how they interact with each other. diff --git a/docs/dev/book.toml b/docs/dev/book.toml new file mode 100644 index 00000000..2cf9bc43 --- /dev/null +++ b/docs/dev/book.toml @@ -0,0 +1,25 @@ +[book] +title = "Firefox Translations Web Extension - Developer's Guide" +authors = ["Bergamot Team @ Mozilla"] +src = "." + +[build] +build-dir = "../../docs/dev" +create-missing = true + +[output.html] +additional-css = ["../shared/custom.css"] +additional-js = ["../shared/tabs.js", "../shared/mermaid.min.js", "../shared/mermaid-init.js"] +git-repository-url = "https://github.com/mozilla-extensions/firefox-translations" +git-branch = "main" +mathjax-support = true + +[output.html.fold] +enable = true +level = 0 + +[preprocessor.open-on-gh] +command = "mdbook-open-on-gh" + +[preprocessor.mermaid] +command = "mdbook-mermaid" diff --git a/docs/dev/engineering/automated-web-extension-testing.md b/docs/dev/engineering/automated-web-extension-testing.md new file mode 100644 index 00000000..4befca46 --- /dev/null +++ b/docs/dev/engineering/automated-web-extension-testing.md @@ -0,0 +1,57 @@ +# Automated web extension testing + +## In-browser tests + +These tests run inside an extension page to be able to have access to the same privileges that web extension background script has. + +## End-to-end (e2e) tests + +These are based on Selenium because of only that allows access to the privileged parts of the UI. + +## Run end-to-end tests + +### Locally + +Before running the tests locally for the first time, install [mitmproxy](https://mitmproxy.org). Example installation command for Mac: + +```bash +brew install mitmproxy +``` + +To run the end-to-end tests, run: + +```bash +yarn e2e-tests +``` + +Note, to modify the `test/e2e/intercept-telemetry-requests.py` script, it may be useful to have the Python deps installed: + +```bash +test/e2e/setup-python-venv.sh +``` + +### Troubleshooting functional tests + +**Basic principles** + +End-to-end tests are run using the built extension artifacts found in `dist/production/`. To test new non-test-related code changes, remember to re-run the relevant build command. + +**Intervening** + +If you want to intervene in a test (eg. to double-check something), follow this pattern: + +1. Add a long delay, eg `await driver.sleep(60 * 60 * 1000);` to the test at the place you want to intervene. +2. Make sure to temporarily also increase the timeout for the test you are running. +3. Run the tests and intervene manually as desired. + +**Obtaining Geckodriver logs** + +To troubleshoot issues with failing tests when only cryptic error messages are available, check the geckodriver logs, located in `test/e2e/results/logs/`. + +## Relevant engineering background + +Maintenance and development of this component requires experience in: + +- Web extension regression tests +- Selenium +- CircleCI diff --git a/docs/dev/engineering/building-and-installing-the-extension.md b/docs/dev/engineering/building-and-installing-the-extension.md new file mode 100644 index 00000000..3ec52d06 --- /dev/null +++ b/docs/dev/engineering/building-and-installing-the-extension.md @@ -0,0 +1,28 @@ +# Building and installing the extension + +## Creating extension builds + +```bash +yarn build:firefox-infobar-ui +``` + +The build artifact will be created under `dist/production/firefox/firefox-infobar-ui`. + +### Installing locally built versions of the extension + +Artifacts built locally or via CircleCI are unsigned, and additional config preferences are necessary to get them to work as expected. + +- Make sure that the following preferences are set to `true` in `about:config`: + - `extensions.experiments.enabled` + - `javascript.options.wasm_simd` + - `javascript.options.wasm_simd_wormhole` + - `dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled` +- Make sure that the following preferences are set to `false` in `about:config`: + - `xpinstall.signatures.required` + +## Analyze webpack bundle size + +```bash +yarn build:firefox-infobar-ui +npx webpack-bundle-analyzer build/production/firefox/firefox-infobar-ui.stats.json build/production/firefox/firefox-infobar-ui +``` diff --git a/docs/CI.md b/docs/dev/engineering/ci.md similarity index 56% rename from docs/CI.md rename to docs/dev/engineering/ci.md index 4c430d47..e695a225 100644 --- a/docs/CI.md +++ b/docs/dev/engineering/ci.md @@ -1,15 +1,3 @@ - - - -**Contents** - -- [Continuous Integration](#continuous-integration) - - [CircleCI](#circleci) - - [Run Circle CI locally (requires Docker)](#run-circle-ci-locally-requires-docker) - - [Installing artifacts created by CircleCI](#installing-artifacts-created-by-circleci) - - - # Continuous Integration [Circle CI](https://circleci.com/) is used for continuous integration. @@ -54,15 +42,3 @@ To forward the GUI to you local Mac workstation, start XQuartz and replace the d xhost + 127.0.0.1 docker run -v "$PWD:/home/circleci/checkout" -e DISPLAY=host.docker.internal:0 -it circleci/node:latest-browsers /bin/bash ``` - -### Installing artifacts created by CircleCI - -Artifacts built via CircleCI are unsigned (just like developer-created local builds), and additional config preferences are necessary to get them to work as expected. First enable the preferences outlined in the [general installation instructions](./INSTALL.md), then: - -- Make sure that the following preferences are set to `true` in `about:config`: - - `extensions.experiments.enabled` - - `javascript.options.wasm_simd` - - `javascript.options.wasm_simd_wormhole` - - `dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled` -- Make sure that the following preferences are set to `false` in `about:config`: - - `xpinstall.signatures.required` diff --git a/docs/dev/engineering/contributing.md b/docs/dev/engineering/contributing.md new file mode 100644 index 00000000..0ead0537 --- /dev/null +++ b/docs/dev/engineering/contributing.md @@ -0,0 +1,5 @@ +# Contributing + +We follow the ordinary Github workflow. Clone the repo, create a PR and get approval from a team member to get it merged. + +End-to-end tests are run against each new commits/PRs. Read more about the current CI setup [here](./ci.md). diff --git a/docs/dev/engineering/cross-browser-development.md b/docs/dev/engineering/cross-browser-development.md new file mode 100644 index 00000000..3f15efe1 --- /dev/null +++ b/docs/dev/engineering/cross-browser-development.md @@ -0,0 +1,107 @@ +## Start development mode + +### Chrome - Cross-browser UI + +```bash +yarn watch:cross-browser-ui:chrome +``` + +### Firefox - Cross-browser UI + +```bash +yarn watch:cross-browser-ui:firefox +``` + +## Creating extension builds + +To build for Firefox: + +```bash +yarn build:cross-browser-ui:firefox +``` + +The build artifact will be created under `dist/production/firefox/cross-browser-ui`. + +For Chrome: + +```bash +yarn build:cross-browser-ui:chrome +``` + +The build artifact will be created under `dist/production/chrome/cross-browser-ui`. + +## Creating a signed build of the extension for self-distribution + +After version bumping and setting the API_KEY and API_SECRET env vars: + +```bash +yarn build:cross-browser-ui:firefox && npx web-ext sign --api-key $API_KEY --api-secret $API_SECRET +``` + +Note: This is for Firefox version with the cross-browser UI only. Chrome Web Store does not offer signed builds for self-distribution and the Firefox infobar UI version gets signed via a separate process. + +## Troubleshooting the extension when it is running + +1. Go to `about:devtools-toolbox?type=extension&id=bergamot-browser-extension%40browser.mt` +2. Click Console + +## Analyze webpack bundle size + +### Chrome - Cross-browser UI + +```bash +yarn build:cross-browser-ui:chrome +npx webpack-bundle-analyzer build/production/chrome/cross-browser-ui.stats.json build/production/chrome/cross-browser-ui +``` + +### Firefox - Cross-browser UI + +```bash +yarn build:cross-browser-ui:firefox +npx webpack-bundle-analyzer build/production/firefox/cross-browser-ui.stats.json build/production/firefox/cross-browser-ui +``` + +## Opening up specific extension pages + +From the background context: + +```javascript +(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( + `main-interface/popup.html`, +); +``` + +```javascript +(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( + `main-interface/popup.html#/components`, +); +``` + +```javascript +(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( + `get-started/get-started.html`, +); +``` + +```javascript +(typeof browser !== "undefined" ? browser : chrome).runtime.getURL( + `options-ui/options-ui.html`, +); +``` + + diff --git a/docs/dev/engineering/debugging.md b/docs/dev/engineering/debugging.md new file mode 100644 index 00000000..6791a5a7 --- /dev/null +++ b/docs/dev/engineering/debugging.md @@ -0,0 +1,13 @@ +## Troubleshooting the extension when it is running + +Hint: To produce a clean log output for forwarding to developers / attaching to issues, first click the Trash can icon in the top left before repeating the steps that leads to the erroneous behavior. + +### Bundled version + +1. Go to `about:devtools-toolbox?type=extension&id=firefox-translations%40mozilla.org` +2. Click Console + +### Self-built version + +1. Go to `about:devtools-toolbox?type=extension&id=TODO` +2. Click Console diff --git a/docs/dev/engineering/getting-started.md b/docs/dev/engineering/getting-started.md new file mode 100644 index 00000000..3289348c --- /dev/null +++ b/docs/dev/engineering/getting-started.md @@ -0,0 +1,13 @@ +# Getting started + +## First time setup + +If you haven't already, download and install Firefox Nightly from [here](https://www.mozilla.org/en-US/firefox/channel/desktop/) before running the below commands. + +Install dependencies using [yarn v1](https://classic.yarnpkg.com/en/docs/install/): + +```bash +yarn install +``` + +Now either continue with [Local development](./local-development.md) or [Building and installing the extension](./building-and-installing-the-extension.md) diff --git a/docs/dev/engineering/improving-the-documentation.md b/docs/dev/engineering/improving-the-documentation.md new file mode 100644 index 00000000..8dd43de8 --- /dev/null +++ b/docs/dev/engineering/improving-the-documentation.md @@ -0,0 +1,85 @@ +# Improving the documentation + +The documentation in this repository pertains to the client-side code for the Firefox Translations feature. + +The human-written documentation is written in Markdown and converted to static HTML using [mdBook](https://rust-lang.github.io/mdBook/). + +API docs are generated using [TypeDoc](https://typedoc.org/) using the [typedoc-plugin-markdown](https://www.npmjs.com/package/typedoc-plugin-markdown). + +## Building documentation + +### Generating the API documentation + +```sh +yarn workspace docs generate-api-docs +``` + +### Building the human-written documentation + +Rust and [mdBook](https://rust-lang.github.io/mdBook/) is required in order to build the documentation. Installation instructions for Mac: + +```sh +xcode-select --install +brew install rust +cargo install mdbook mdbook-mermaid mdbook-open-on-gh +``` + +Add `~/.cargo/bin` to your PATH, eg: + +```sh +echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.bash_profile +``` + +Open up a new terminal. + +Now the docs can be built with: + +```sh +yarn workspace docs build +``` + +The built documentation is saved in `build/docs/`. + +## Developer/watch mode + +```sh +yarn workspace docs watch +``` + +This makes the user documentation available locally on http://0.0.0.0:3000/user/index.html and http://0.0.0.0:3000/dev/index.html. Changes made in the `docs/` directory will result in a rebuild and client reload. + +### Checking links + +Internal links within the documentation can be checked using the [`link-checker`](https://www.npmjs.com/package/link-checker) tool. +External links are currently not checked, since this takes considerable time and frequently fails in CI due to networking restrictions or issues. + +Link checking runs automatically in CI, and requires building the human-written documentation as well the generated API documentation first. To run link checking locally, run: + +```sh +yarn workspace docs linkcheck +``` + +### Spell checking + +The human-written documentation (but not the generated API documentation) is spell checked using [aspell](http://aspell.net/). + +On Unix-like platforms, it can be installed using your system's package manager: + +```sh +sudo apt install aspell-en +# ...or... +sudo dnf install aspell-en +# ...or... +brew install aspell +``` + +Note that aspell 0.60.8 or later is required, as that is the first release with markdown support. + +Spell checking runs automatically in CI. You can the spell check the documentation using the following: + +```sh +yarn workspace docs spellcheck +``` + +This will bring up an interactive spell-checking environment in the console. +Pressing `a` to add a word will add it to the project's local `.dictionary` file, and the changes should be committed to `git`. diff --git a/docs/dev/engineering/local-development.md b/docs/dev/engineering/local-development.md new file mode 100644 index 00000000..eadf7612 --- /dev/null +++ b/docs/dev/engineering/local-development.md @@ -0,0 +1,54 @@ +# Local development + +The commands in these instructions are meant to be run in the root folder in this repo. + +## First time setup + +1. Follow the [Getting started](./getting-started.md) instructions + +2. Import bergamot models locally, since the extension downloads the models from a local endpoint when running in development mode: + +```bash +yarn bergamot-models:import +``` + +3. Import Bergamot translator using one of the two methods below. + +### Using known-to-work Bergamot Translator WASM artifacts and importing them to the extension + +To use artifacts that are known to work (built by bergamot-translator's CI): + +```bash +yarn bergamot-translator:download-and-import +``` + +Note: Once this command has run, it will create a folder called `downloaded-bergamot-translator-wasm-artifacts` and if this folder exists, it will not re-download the artifacts again. Thus, to make sure that you are using the most up to date WASM artifacts, remove the `downloaded-bergamot-translator-wasm-artifacts` folder, then re-run the command. + +### Building Bergamot Translator WASM artifacts and importing them to the extension + +If you want to try out custom changes to [bergamot-translator](https://github.com/mozilla/bergamot-translator/README.md), first make sure that you have checked out the submodules: + +``` +git submodule update --init --recursive +``` + +Then, follow the upstream instructions ([bergamot-translator/README](https://github.com/mozilla/bergamot-translator/README.md) for setting up an environment that successfully builds bergamot-translator. + +When all is properly set-up, you should be able to run the following to build and import the custom WASM artifacts into the extension: + +```bash +./bergamot-translator/build-wasm.sh +./import-bergamot-translator.sh ./bergamot-translator/build-wasm/ +``` + +Re-run this command any time there has been an update in the bergamot-translator submodule. + +## Start development/watch mode + +When the above commands have finished, run: + +```bash +yarn watch:firefox-infobar-ui +``` + +This will build the extension, launch the browser, install the extension and start Webpack in watch mode, which repeats the build process and reloads the extension when source files are changed: diff --git a/docs/dev/engineering/overview.md b/docs/dev/engineering/overview.md new file mode 100644 index 00000000..f51b312f --- /dev/null +++ b/docs/dev/engineering/overview.md @@ -0,0 +1,5 @@ +# Engineering + +This section details how to get your hands dirty and contribute to the codebase. + +Use the navigation to the left to continue. diff --git a/docs/dev/maintenance/overview.md b/docs/dev/maintenance/overview.md new file mode 100644 index 00000000..a7f84b00 --- /dev/null +++ b/docs/dev/maintenance/overview.md @@ -0,0 +1,5 @@ +# Maintenance + +This section contains shorter tutorials on how to achieve specific maintenance tasks. + +Use the navigation to the left to continue. diff --git a/docs/dev/maintenance/upgrading-glean-js.md b/docs/dev/maintenance/upgrading-glean-js.md new file mode 100644 index 00000000..558b20af --- /dev/null +++ b/docs/dev/maintenance/upgrading-glean-js.md @@ -0,0 +1,6 @@ +## Upgrading Glean.js + +1. Bump the `@mozilla/glean` version in `package.json` +2. `yarn` +3. `yarn generate-glean` +4. Commit all changes. Use a commit message in the format of `Upgrade Glean.js to vx.y.z` diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 00000000..84704c03 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,27 @@ +{ + "name": "docs", + "description": "docs", + "version": "0.0.0", + "dependencies": { + "core": "0.0.0", + "cross-browser-ui": "0.0.0", + "firefox-infobar-ui": "0.0.0", + "mermaid": "^8.10.2" + }, + "devDependencies": { + "light-server": "^2.9.1", + "link-checker": "^1.4.2", + "typedoc": "^0.20.36", + "typedoc-plugin-markdown": "^3.9.0" + }, + "private": true, + "scripts": { + "build": "bin/build-docs.sh", + "generate-api-docs": "bin/generate-api-docs.sh", + "format": "prettier '**/*.{md,css,js,json,html}' --ignore-path=.eslintignore --write", + "linkcheck": "link-checker ../build/docs --disable-external true --allow-hash-href true", + "spellcheck": "bin/spellcheck.sh check", + "serve": "light-server -s ../build/docs -p 3000", + "watch": "yarn build && yarn serve -w \"** # yarn build\"" + } +} diff --git a/docs/shared/blockquote-info.html b/docs/shared/blockquote-info.html new file mode 100644 index 00000000..8eb68c47 --- /dev/null +++ b/docs/shared/blockquote-info.html @@ -0,0 +1 @@ +
diff --git a/docs/shared/blockquote-stop.html b/docs/shared/blockquote-stop.html new file mode 100644 index 00000000..dfa47986 --- /dev/null +++ b/docs/shared/blockquote-stop.html @@ -0,0 +1 @@ +
diff --git a/docs/shared/blockquote-warning.html b/docs/shared/blockquote-warning.html new file mode 100644 index 00000000..0726c5bc --- /dev/null +++ b/docs/shared/blockquote-warning.html @@ -0,0 +1,3 @@ +
diff --git a/docs/shared/custom.css b/docs/shared/custom.css new file mode 100644 index 00000000..99d44e6f --- /dev/null +++ b/docs/shared/custom.css @@ -0,0 +1,334 @@ +/* API Reference Titles */ + +a.header code { + background-color: rgba(230, 230, 230, 0.8); + display: inline-block; + padding: 0 5px; + border-radius: 3px; + color: black; + font-weight: bold; + transition: all 150ms ease-in-out; +} + +a.header code:hover { + background-color: rgba(230, 230, 230, 1); +} + +/* Style the tab */ +.tabs { + display: flex; + flex-direction: column; + margin-bottom: 40px; +} + +.tabbar { + border: 1px solid #ccc; + background-color: #f1f1f1; + width: 100%; + box-sizing: border-box; +} + +/* Style the buttons that are used to open the tab content */ +.tabbar button { + background-color: inherit; + float: left; + border: none; + outline: none; + cursor: pointer; + padding: 14px 16px; + transition: 0.3s; +} + +/* Change background color of buttons on hover */ +.tabbar button:hover { + background-color: #ddd; +} + +.tabbar button.disabled:hover { + background-color: inherit; +} + +/* Create an active/current tablink class */ +.tabbar button.active { + background-color: #ccc; +} + +.tabbar button.disabled { + color: #ccc; + cursor: default; +} + +/* The container that holds all of the tab contents */ +.tabcontents { + display: flex; +} + +/* The container for each individual language */ +.tab { + display: none; + width: 100%; + border: 1px solid #ccc; + border-top: none; + padding: 6px 12px; + box-sizing: border-box; +} + +.tab.active { + display: block; +} + +/* The footer with the "Open on GitHub" link */ +footer#open-on-gh { + font-size: 0.8em; + text-align: center; + border-top: 1px solid black; + padding: 5px 0; +} + +/* Distribution simulator styles */ + +#simulator-container { + display: flex; + flex-wrap: wrap; + overflow: hidden; +} + +#simulator-container h3 { + margin-top: 20px; +} + +#simulator-container .input-group { + width: 100%; +} + +#simulator-container .input-group label { + display: block; + font-weight: bold; + font-size: 14px; + margin-bottom: 7px; +} + +#simulator-container .input-group input, +#simulator-container .input-group select, +#custom-data-modal textarea { + display: block; + width: 100%; + padding: 5px; + margin-bottom: 10px; + border-radius: 3px; + border: 1px solid #e0e0e0; + box-sizing: border-box; +} + +#histogram-props, +#data-options { + width: 50%; + box-sizing: border-box; +} + +#data-options { + padding-right: 50px; +} + +#data-options .input-group { + margin-bottom: 10px; +} + +#data-options .input-group:first-of-type { + margin-top: 20px; +} + +#data-options .input-group:last-of-type { + margin-bottom: 0; +} + +#data-options .input-group label { + display: inline-block; +} + +#data-options .input-group input { + display: inline; + width: auto; +} + +#histogram-chart-container { + width: 100%; + padding: 30px; + border: 1px solid #e0e0e0; + margin: 30px 0; + overflow: hidden; + position: relative; +} + +#histogram-chart { + margin-top: 50px; + width: 100%; +} + +#histogram-chart-legend { + font-size: 14px; + text-align: center; + width: 100%; +} + +#histogram-functional-props, +#histogram-non-functional-props { + display: none; +} + +#custom-data-modal-overlay { + background-color: rgba(0, 0, 0, 0.5); + position: fixed; + width: 100vw; + height: 100vh; + top: 0; + left: 0; + z-index: 999; + display: none; +} + +#custom-data-modal { + width: 50%; + background-color: white; + border-radius: 5px; + position: relative; + top: 15%; + left: 25%; + padding: 50px; +} + +.hide { + display: none !important; +} + +/* Special block quotes */ + +.special-quote { + font-size: 20px; + padding: 20px 20px 20px; + border-top: 3px solid rgba(200, 200, 200, 0.3); + border-left: 3px solid rgba(200, 200, 200, 0.3); + border-right: 3px solid rgba(200, 200, 200, 0.3); + display: block; + border-top-left-radius: 5px; + border-top-right-radius: 5px; + margin-top: 40px; +} + +.special-quote + h1, +.special-quote + h2, +.special-quote + h3, +.special-quote + h4, +.special-quote + h5 { + margin: -40px 0 17px; + margin-top: -40px; + padding-left: 55px; + font-size: 1.6rem; +} + +.special-quote + h1::before, +.special-quote + h2::before, +.special-quote + h3::before, +.special-quote + h4::before, +.special-quote + h5::before { + display: none; +} + +.special-quote + blockquote, +.special-quote + h1 + blockquote, +.special-quote + h2 + blockquote, +.special-quote + h3 + blockquote, +.special-quote + h4 + blockquote, +.special-quote + h5 + blockquote { + margin-top: 0; + background-color: transparent !important; + border-bottom: 3px solid rgba(200, 200, 200, 0.3) !important; + border-left: 3px solid rgba(200, 200, 200, 0.3) !important; + border-right: 3px solid rgba(200, 200, 200, 0.3) !important; + border-top: none !important; + border-bottom-left-radius: 5px; + border-bottom-right-radius: 5px; + padding-bottom: 7px; + margin-bottom: 40px; +} + +.special-quote + blockquote *:first-child, +.special-quote + h1 + blockquote *:first-child, +.special-quote + h2 + blockquote *:first-child, +.special-quote + h3 + blockquote *:first-child, +.special-quote + h4 + blockquote *:first-child, +.special-quote + h5 + blockquote *:first-child { + margin-top: 0 !important; +} + +.special-quote + h3 + blockquote { + margin-top: 17px; +} + +.special-quote.special-quote-info { + color: #005aa8; +} + +.special-quote.special-quote-warning { + color: #fdc008; +} + +.special-quote.special-quote-stop { + color: #fd0808; +} + +.special-quote.special-quote-stop, +.special-quote.special-quote-stop + blockquote, +.special-quote.special-quote-stop + h1 + blockquote, +.special-quote.special-quote-stop + h2 + blockquote, +.special-quote.special-quote-stop + h3 + blockquote, +.special-quote.special-quote-stop + h4 + blockquote, +.special-quote.special-quote-stop + h5 + blockquote { + border-color: #fd0808 !important; +} + +/* Tab tooltips */ + +.tablinks { + position: relative; +} + +.tablinks .tooltip { + display: none; + position: absolute; + padding: 7px; + border-radius: 3px; + z-index: 999; + background-color: #333; + color: #eee; + font-size: 0.9em; + width: 200px; + top: 40px; + left: calc(50% - 100px); + box-sizing: border-box; + user-select: text; +} + +.tablinks .tooltip a { + color: #048dc2 !important; +} + +.tablinks .tooltip::after { + bottom: 100%; + left: 50%; + border: solid transparent; + content: ""; + height: 0; + width: 0; + position: absolute; + pointer-events: none; + border-color: rgba(240, 240, 240, 0); + border-bottom-color: #333; + border-width: 7px; + margin-left: -7px; +} + +.tablinks:hover .tooltip, +.tablinks .tooltip:hover { + display: block; +} diff --git a/docs/shared/mermaid-init.js b/docs/shared/mermaid-init.js new file mode 100644 index 00000000..5ce9923b --- /dev/null +++ b/docs/shared/mermaid-init.js @@ -0,0 +1,4 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +mermaid.initialize({ startOnLoad: true }); diff --git a/docs/shared/tab_footer.md b/docs/shared/tab_footer.md new file mode 100644 index 00000000..bdd6bc96 --- /dev/null +++ b/docs/shared/tab_footer.md @@ -0,0 +1,2 @@ + + diff --git a/docs/shared/tab_header.md b/docs/shared/tab_header.md new file mode 100644 index 00000000..3adf4ed9 --- /dev/null +++ b/docs/shared/tab_header.md @@ -0,0 +1,3 @@ +
+
+
diff --git a/docs/shared/tabs.js b/docs/shared/tabs.js new file mode 100644 index 00000000..ef5b34f2 --- /dev/null +++ b/docs/shared/tabs.js @@ -0,0 +1,111 @@ +/** + * Returns true if browser supports HTML5 localStorage. + */ +function supportsHTML5Storage() { + try { + return "localStorage" in window && window.localStorage !== null; + } catch (e) { + return false; + } +} + +/** + * Event handler for when a tab is clicked. + */ +function onClickTab(event) { + const target = event.currentTarget; + const language = target.dataset.lang; + + const initialTargetOffset = target.offsetTop; + const initialScrollPosition = window.scrollY; + switchAllTabs(language); + + // Keep initial perceived scroll position after resizing + // that may happen due to activation of multiple tabs in the same page. + const finalTargetOffset = target.offsetTop; + window.scrollTo({ + top: initialScrollPosition + (finalTargetOffset - initialTargetOffset), + }); +} + +/** + * Switches the displayed tab for the given container. + * + * :param container: The div containing both the tab bar and the individual tabs + * as direct children. + * :param language: The language to switch to. + */ +function switchTab(container, language) { + const previouslyActiveTab = container.querySelector(".tabcontents .active"); + previouslyActiveTab && previouslyActiveTab.classList.remove("active"); + const tab = container.querySelector(`.tabcontents [data-lang="${language}"]`); + tab && tab.innerText && tab.classList.add("active"); + + const previouslyActiveButton = container.querySelector(".tabbar .active"); + previouslyActiveButton && previouslyActiveButton.classList.remove("active"); + const button = container.querySelector(`.tabbar [data-lang="${language}"]`); + button && tab.innerText && button.classList.add("active"); +} + +/** + * Switches all tabs on the page to the given language. + * + * :param language: The language to switch to. + */ +function switchAllTabs(language) { + const containers = document.getElementsByClassName("tabs"); + for (let i = 0; i < containers.length; ++i) { + switchTab(containers[i], language); + } + + if (supportsHTML5Storage()) { + localStorage.setItem("glean-preferred-language", language); + } +} + +/** + * Opens all tabs on the page to the given language on page load. + */ +(function openTabs() { + if (!supportsHTML5Storage()) { + return; + } + + const containers = document.getElementsByClassName("tabs"); + for (let i = 0; i < containers.length; ++i) { + // Create tabs for every language that has content + const tabs = containers[i].children[0]; + const tabcontents = containers[i].children[1]; + for (const tabcontent of tabcontents.children) { + const button = document.createElement("button"); + button.dataset.lang = tabcontent.dataset.lang; + button.innerText = tabcontent.dataset.lang; + button.classList.add("tablinks"); + if (!tabcontent.innerHTML) { + button.classList.add("disabled"); + const tooltip = document.createElement("span"); + tooltip.classList.add("tooltip"); + + if (tabcontent.dataset.bug) { + tooltip.innerHTML += `${tabcontent.dataset.lang} does not provide this API yet. \nFollow Bug ${tabcontent.dataset.bug} for updates.`; + } else if (tabcontent.dataset.info) { + // Note: it is safe to use .innerHTML since we have full control over `tabcontent.dataset.info`. + tooltip.innerHTML += `${tabcontent.dataset.info}`; + } else { + tooltip.innerHTML += `${tabcontent.dataset.lang} does not provide this API.`; + } + button.appendChild(tooltip); + } else { + button.onclick = onClickTab; + } + tabs.appendChild(button); + } + } + + let language = localStorage.getItem("glean-preferred-language"); + if (language == null) { + language = "Kotlin"; + } + + switchAllTabs(language); +})(); diff --git a/docs/typedoc.json b/docs/typedoc.json new file mode 100644 index 00000000..dfda0c12 --- /dev/null +++ b/docs/typedoc.json @@ -0,0 +1,13 @@ +{ + "entryPoints": [ + "../src/core/ts", + "../src/cross-browser-ui/ts", + "../src/firefox-infobar-ui/ts" + ], + "theme": "markdown", + "exclude": ["**/*.tsx", "**/*+(index|.spec).ts"], + "excludeExternals": true, + "readme": "none", + "externalPattern": ["**/node_modules/**"], + "out": "dev/api-docs" +} diff --git a/docs/user/README.md b/docs/user/README.md new file mode 100644 index 00000000..6cd03298 --- /dev/null +++ b/docs/user/README.md @@ -0,0 +1,42 @@ +# Introduction + +Firefox Translations allows neural machine translation to happen directly in the browser, without having any content sent to an external server. + +The current version allows translation from Spanish, Estonian and Czech in and out of English, as well as English into German. + +Installation instructions are found [here](./install.md). + +> **Note** Firefox Translations is an experimental feature. + +## Release notes + +The release notes can be found on Github: [https://github.com/mozilla-extensions/firefox-translations/releases]() + +## Reporting bugs/issues + +To report issues or request changes on Firefox Translations, file a bug in [Bugzilla in Firefox :: Translation](https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox&component=Translation). Check current list of reported bugs [here](https://bugzilla.mozilla.org/buglist.cgi?product=Firefox&component=Translation&bug_status=__open__&list_id=15731344). + +## About this guide + +This is the user documentation for the **Firefox Translations** feature. + +For developer documentation on the **Firefox Translations Web Extension**, refer to [Firefox Translations Developer's Guide](../dev/index.html). + +## Acknowledgments + +### The Bergamot project + +This project was made possible through the combined effort of [all researchers and partners in the Bergamot project](https://browser.mt/partners/). The translation models are prepared as part of the Bergamot project ([https://github.com/browsermt/students]()). The translation engine used is [Bergamot Translator](https://github.com/browsermt/bergamot-translator) which is based on [Marian](https://github.com/marian-nmt/marian-dev). + +The Bergamot project adds and improves client-side machine translation in the web browser. + +Unlike current cloud-based options, running directly on users’ machines empowers citizens to preserve their privacy and increases the uptake of language technologies in Europe in various sectors that require confidentiality. Free software integrated with open-source web browsers, such as Mozilla Firefox, enables bottom-up adoption by non-experts, resulting in cost savings for private and public sector users who would otherwise procure translation or operate monolingually. + + + +The Bergamot project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 825303. + +## License + +Firefox Translations and Firefox Translations Source Code is subject to the terms of the Mozilla Public License v2.0. +You can obtain a copy of the MPL at . diff --git a/docs/user/SUMMARY.md b/docs/user/SUMMARY.md new file mode 100644 index 00000000..fcd77640 --- /dev/null +++ b/docs/user/SUMMARY.md @@ -0,0 +1,5 @@ +[Introduction](README.md) + +- [Installation instructions](install.md) +- [Usage instructions](usage.md) +- [Cross-browser version](cross-browser-version.md) diff --git a/docs/user/book.toml b/docs/user/book.toml new file mode 100644 index 00000000..23ed4016 --- /dev/null +++ b/docs/user/book.toml @@ -0,0 +1,25 @@ +[book] +title = "Firefox Translations Web Extension - User Guide" +authors = ["Bergamot Team @ Mozilla"] +src = "." + +[build] +build-dir = "../../docs/user" +create-missing = true + +[output.html] +additional-css = ["../shared/custom.css"] +additional-js = ["../shared/tabs.js", "../shared/mermaid.min.js", "../shared/mermaid-init.js"] +git-repository-url = "https://github.com/mozilla-extensions/firefox-translations" +git-branch = "main" +mathjax-support = true + +[output.html.fold] +enable = true +level = 0 + +[preprocessor.open-on-gh] +command = "mdbook-open-on-gh" + +[preprocessor.mermaid] +command = "mdbook-mermaid" diff --git a/docs/INSTALL.md b/docs/user/cross-browser-version.md similarity index 52% rename from docs/INSTALL.md rename to docs/user/cross-browser-version.md index 4898068c..6e50bb88 100644 --- a/docs/INSTALL.md +++ b/docs/user/cross-browser-version.md @@ -1,53 +1,11 @@ - - +# Cross-browser version -**Contents** +Firefox Translations is currently a Firefox-only feature since it uses native browser UI elements and takes advantage of Firefox-specific optimizations. -- [Installation instructions](#installation-instructions) - - [Firefox - Infobar UI](#firefox---infobar-ui) - - [Preparations](#preparations) - - [Configuring Nightly to enable the extension](#configuring-nightly-to-enable-the-extension) - - [Demo](#demo) - - - -# Installation instructions - -This document outlines how to get the v0.4.3 pre-release version of the extension running locally. - -Before installing, please take a minute to read [the release notes](https://github.com/mozilla-extensions/firefox-translations/releases/tag/v0.4.3). - - - -## Firefox - Infobar UI - -### Preparations - -- The current release requires bleeding edge browser capabilities. If you haven't already, download and install Firefox Nightly (English or German edition) from [https://www.mozilla.org/en-US/firefox/all/#product-desktop-nightly](https://www.mozilla.org/en-US/firefox/all/#product-desktop-nightly). -- If you have tried any previous version of the Firefox extension, please make sure to remove/uninstall it before installing the new version. You will find it named "Firefox Translations" or "Bergamot Translate" in `about:addons`. - -### Configuring Nightly to enable the extension - -- Change the following preferences `false` in `about:config`: - - `extensions.translations.disabled` -- The extension is now enabled - -### Demo - -- To try it out, visit a page in a language combination that is supported for translation, eg one of: - - With Firefox Nightly English edition, visit a page in either Spanish or Estonian, eg https://www.mozilla.org/es-ES/ or https://www.mozilla.org/et/ - - With Firefox Nightly German edition, visit a page in English, eg https://www.mozilla.org/en-US/ -- Press Translate in the translation infobar that pops up - - Note: For non-english editions of Nightly, one must eliminate EN and EN-US from the accepted languages (from the Language section of about:preferences, more precisely from the Webpage Language Settings) for the infobar to show +In the past, and possibly sometime in the future, there was/will be releases of cross-browser versions of the feature. The cross-browser version uses no Firefox-specific UI or optimizations. + diff --git a/src/core/ts/background-scripts/background.js/telemetry/generated/performance.ts b/src/core/ts/background-scripts/background.js/telemetry/generated/performance.ts index 02e77176..9d0d38fa 100644 --- a/src/core/ts/background-scripts/background.js/telemetry/generated/performance.ts +++ b/src/core/ts/background-scripts/background.js/telemetry/generated/performance.ts @@ -4,9 +4,9 @@ // AUTOGENERATED BY glean_parser. DO NOT EDIT. DO NOT COMMIT. -import StringMetricType from "@mozilla/glean/webext/private/metrics/string"; -import TimespanMetricType from "@mozilla/glean/webext/private/metrics/timespan"; import QuantityMetricType from "@mozilla/glean/webext/private/metrics/quantity"; +import TimespanMetricType from "@mozilla/glean/webext/private/metrics/timespan"; +import StringMetricType from "@mozilla/glean/webext/private/metrics/string"; /** * Timing from "translation button pressed" * to "full page is translated". diff --git a/src/core/ts/background-scripts/background.js/telemetry/generated/test.ts b/src/core/ts/background-scripts/background.js/telemetry/generated/test.ts index ed92a49a..2101b104 100644 --- a/src/core/ts/background-scripts/background.js/telemetry/generated/test.ts +++ b/src/core/ts/background-scripts/background.js/telemetry/generated/test.ts @@ -5,8 +5,8 @@ // AUTOGENERATED BY glean_parser. DO NOT EDIT. DO NOT COMMIT. import EventMetricType from "@mozilla/glean/webext/private/metrics/event"; -import StringMetricType from "@mozilla/glean/webext/private/metrics/string"; import CounterMetricType from "@mozilla/glean/webext/private/metrics/counter"; +import StringMetricType from "@mozilla/glean/webext/private/metrics/string"; /** * Test counter. * diff --git a/src/core/ts/background-scripts/background.js/telemetry/metrics.yaml b/src/core/ts/background-scripts/background.js/telemetry/metrics.yaml index d9dbcc0f..6bccdfc3 100644 --- a/src/core/ts/background-scripts/background.js/telemetry/metrics.yaml +++ b/src/core/ts/background-scripts/background.js/telemetry/metrics.yaml @@ -29,7 +29,7 @@ metadata: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 to_lang: type: string lifetime: ping @@ -45,7 +45,7 @@ metadata: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 firefox_client_id: type: string lifetime: ping @@ -61,7 +61,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 extension_version: type: string lifetime: ping @@ -77,7 +77,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 extension_build_id: type: string lifetime: ping @@ -94,7 +94,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 bergamot_translator_version: type: string lifetime: ping @@ -110,7 +110,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 system_memory: type: quantity unit: mb @@ -127,7 +127,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_count: type: quantity unit: cpus @@ -144,7 +144,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_cores_count: type: quantity unit: cores @@ -161,7 +161,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_vendor: type: string lifetime: ping @@ -177,7 +177,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_family: type: quantity unit: enum @@ -194,7 +194,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_model: type: quantity unit: enum @@ -211,7 +211,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_stepping: type: quantity unit: enum @@ -228,7 +228,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_l2_cache: type: quantity unit: kb @@ -245,7 +245,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_l3_cache: type: quantity unit: kb @@ -262,7 +262,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_speed: type: quantity unit: mhz @@ -279,7 +279,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 cpu_extensions: type: string lifetime: ping @@ -295,7 +295,7 @@ metadata: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 performance: full_page_translated_time: @@ -314,7 +314,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 full_page_translated_wps: type: quantity unit: words per second @@ -332,7 +332,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 model_download_time_num: type: timespan lifetime: ping @@ -351,7 +351,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 model_load_time_num: type: timespan lifetime: ping @@ -370,7 +370,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 translation_engine_time: type: timespan lifetime: ping @@ -387,7 +387,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 translation_engine_wps: type: quantity unit: words per second @@ -405,7 +405,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 word_count: type: quantity unit: words @@ -423,7 +423,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 word_count_visible_in_viewport: type: quantity unit: words @@ -442,7 +442,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 translation_quality: type: string lifetime: ping @@ -458,7 +458,7 @@ performance: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 infobar: displayed: @@ -476,7 +476,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 closed: type: event lifetime: ping @@ -492,7 +492,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 translate: type: event lifetime: ping @@ -508,7 +508,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 never_translate_lang: type: event lifetime: ping @@ -524,7 +524,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 never_translate_site: type: event lifetime: ping @@ -540,7 +540,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 not_now: type: event lifetime: ping @@ -556,7 +556,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 change_lang: type: event lifetime: ping @@ -572,7 +572,7 @@ infobar: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 service: not_supported: @@ -590,7 +590,7 @@ service: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 lang_mismatch: type: counter lifetime: ping @@ -606,7 +606,7 @@ service: - interaction notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 errors: translation: @@ -624,7 +624,7 @@ errors: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 marian: type: counter lifetime: ping @@ -640,7 +640,7 @@ errors: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 memory: type: counter lifetime: ping @@ -656,7 +656,7 @@ errors: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 model_download: type: counter lifetime: ping @@ -672,7 +672,7 @@ errors: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 test: counter_test: @@ -690,7 +690,7 @@ test: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 event_test: type: event lifetime: ping @@ -706,7 +706,7 @@ test: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 string_test: type: string lifetime: ping @@ -722,4 +722,4 @@ test: - technical notification_emails: - anatal-all@mozilla.com - expires: 2021-08-10 + expires: 2022-02-10 From c4361f0d42afc406c76f3be87290bfb0c8c9dd4a Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 18 Aug 2021 12:50:54 -0700 Subject: [PATCH 22/31] Update models (#278) --- import-models.sh | 2 +- src/core/ts/config.ts | 3 ++- src/core/ts/modelRegistry.ts | 39 ++++++++++++++++++++++++++++-------- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/import-models.sh b/import-models.sh index 87d44762..7a45ccd4 100755 --- a/import-models.sh +++ b/import-models.sh @@ -7,7 +7,7 @@ set -e echo "* Checkout out the relevant revision of the firefox-translations-models repo" MODELS_UPDATED=0 -MODELS_GIT_REV="a8158d24c78fc9b8bea6b3e94c99ee370210bd84" # v0.2.5 +MODELS_GIT_REV="2473ea093215446a76e8ae3915b2bd947c5659eb" # v0.2.6 if [ ! -d "firefox-translations-models" ]; then git clone --branch main --single-branch https://github.com/mozilla/firefox-translations-models MODELS_UPDATED=1 diff --git a/src/core/ts/config.ts b/src/core/ts/config.ts index 5d6c482d..b64e7ef7 100644 --- a/src/core/ts/config.ts +++ b/src/core/ts/config.ts @@ -5,7 +5,7 @@ export const config = { sentryDsn: process.env.SENTRY_DSN, bergamotModelsBaseUrl: developmentBuild ? "http://0.0.0.0:4000/models" - : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.5", + : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.6", wasmBinariesBaseUrl: developmentBuild ? "http://0.0.0.0:4000/wasm" : "https://storage.googleapis.com/bergamot-models-sandbox/wasm/1", @@ -31,6 +31,7 @@ export const config = { ["ru", "en"], ["en", "ru"], ["pt", "en"], + ["it", "en"], ], privacyNoticeUrl: "https://example.com/privacy-notice", feedbackSurveyUrl: diff --git a/src/core/ts/modelRegistry.ts b/src/core/ts/modelRegistry.ts index c4a8001c..ac37227b 100644 --- a/src/core/ts/modelRegistry.ts +++ b/src/core/ts/modelRegistry.ts @@ -33,6 +33,29 @@ export const modelRegistry: ModelRegistry = { "b88c646db5af7cf2b8be78450a47e78e55f4f4d2fb14611eada5dcd3e324c212", }, }, + iten: { + vocab: { + name: "vocab.iten.spm", + size: 814151, + estimatedCompressedSize: 405416, + expectedSha256Hash: + "22d5ce6973be5360a921103acbe984a9bfca952a1f6c55c9cb5ef7de4fd58266", + }, + lex: { + name: "lex.50.50.iten.s2t.bin", + size: 5238420, + estimatedCompressedSize: 2860178, + expectedSha256Hash: + "357d362373022b029ee9965975a133e6f36fdb0fed749202ff578365cf0111f8", + }, + model: { + name: "model.iten.intgemm.alphas.bin", + size: 17140836, + estimatedCompressedSize: 13423308, + expectedSha256Hash: + "1fae546faeb9046f80b1b7e940b37b660974ce72902778181d6cd1c30b717f35", + }, + }, pten: { vocab: { name: "vocab.pten.spm", @@ -59,24 +82,24 @@ export const modelRegistry: ModelRegistry = { ruen: { vocab: { name: "vocab.ruen.spm", - size: 952700, - estimatedCompressedSize: 439416, + size: 936576, + estimatedCompressedSize: 435801, expectedSha256Hash: - "52166910903332384a723561dc5a5913d3c2348e6f4b9c8846869f24a7cadce5", + "aaf9a325c0a988c507d0312cb6ba1a02bac7a370bcd879aedee626a40bfbda78", }, lex: { name: "lex.50.50.ruen.s2t.bin", - size: 4370648, - estimatedCompressedSize: 2284561, + size: 5090836, + estimatedCompressedSize: 2684919, expectedSha256Hash: - "5bbea0f3bd641f675eaa3ef325f623e3427b29eb291805f37e3efd2e5abc928b", + "e6667e22f5f86be4872e3768b7184727f5dd8c9f2ccfb0639baabcb1176f5d11", }, model: { name: "model.ruen.intgemm.alphas.bin", size: 17140836, - estimatedCompressedSize: 13189875, + estimatedCompressedSize: 13108893, expectedSha256Hash: - "a045a53520d5792aa1761198e3508c3c698870d47510e2c08fa77a1a4a2b83c9", + "3b6a0305e3d232fadd54f5a765365b7b96ad6d8f2e818cba594b02fbd8fadb3d", }, }, csen: { From c3220adfbc0af4e41602bbb804e8269635c98b5b Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Mon, 30 Aug 2021 21:18:11 -0700 Subject: [PATCH 23/31] Add en-pt model (#279) --- import-models.sh | 2 +- src/core/ts/config.ts | 3 ++- src/core/ts/modelRegistry.ts | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/import-models.sh b/import-models.sh index 7a45ccd4..cb96e747 100755 --- a/import-models.sh +++ b/import-models.sh @@ -7,7 +7,7 @@ set -e echo "* Checkout out the relevant revision of the firefox-translations-models repo" MODELS_UPDATED=0 -MODELS_GIT_REV="2473ea093215446a76e8ae3915b2bd947c5659eb" # v0.2.6 +MODELS_GIT_REV="5ff404f0af67654bcd971639c45cc4f864f9d9f9" # v0.2.7 if [ ! -d "firefox-translations-models" ]; then git clone --branch main --single-branch https://github.com/mozilla/firefox-translations-models MODELS_UPDATED=1 diff --git a/src/core/ts/config.ts b/src/core/ts/config.ts index b64e7ef7..827780a5 100644 --- a/src/core/ts/config.ts +++ b/src/core/ts/config.ts @@ -5,7 +5,7 @@ export const config = { sentryDsn: process.env.SENTRY_DSN, bergamotModelsBaseUrl: developmentBuild ? "http://0.0.0.0:4000/models" - : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.6", + : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.7", wasmBinariesBaseUrl: developmentBuild ? "http://0.0.0.0:4000/wasm" : "https://storage.googleapis.com/bergamot-models-sandbox/wasm/1", @@ -31,6 +31,7 @@ export const config = { ["ru", "en"], ["en", "ru"], ["pt", "en"], + ["en", "pt"], ["it", "en"], ], privacyNoticeUrl: "https://example.com/privacy-notice", diff --git a/src/core/ts/modelRegistry.ts b/src/core/ts/modelRegistry.ts index ac37227b..c99f168b 100644 --- a/src/core/ts/modelRegistry.ts +++ b/src/core/ts/modelRegistry.ts @@ -10,6 +10,29 @@ export interface ModelRegistry { } export const modelRegistry: ModelRegistry = { + enpt: { + vocab: { + name: "vocab.enpt.spm", + size: 812781, + estimatedCompressedSize: 406524, + expectedSha256Hash: + "633a3d782c79f7d5e4b94ab96848f47c2fdf8ba82dd99efd1742b8a696bbd0cc", + }, + lex: { + name: "lex.50.50.enpt.s2t.bin", + size: 4472528, + estimatedCompressedSize: 2411984, + expectedSha256Hash: + "1e96599123d275afa37353dfe84677a4070f013494fbdc9c52a28445cc9bc38d", + }, + model: { + name: "model.enpt.intgemm.alphas.bin", + size: 17140836, + estimatedCompressedSize: 13429592, + expectedSha256Hash: + "d968735704c75e33c2e183b9241f14c0b2a560d01d88a2728e5c0119a4d7fb22", + }, + }, enru: { vocab: { name: "vocab.enru.spm", From 5266b2f8d63a005a8796cc3059278ee00bbd3eb2 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 1 Sep 2021 09:53:31 -0700 Subject: [PATCH 24/31] Add en-it model (#280) --- import-models.sh | 2 +- src/core/ts/config.ts | 2 +- src/core/ts/modelRegistry.ts | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/import-models.sh b/import-models.sh index cb96e747..ea4740fd 100755 --- a/import-models.sh +++ b/import-models.sh @@ -7,7 +7,7 @@ set -e echo "* Checkout out the relevant revision of the firefox-translations-models repo" MODELS_UPDATED=0 -MODELS_GIT_REV="5ff404f0af67654bcd971639c45cc4f864f9d9f9" # v0.2.7 +MODELS_GIT_REV="48b0e0471461778694646166374352a9f68c88f4" # v0.2.8 if [ ! -d "firefox-translations-models" ]; then git clone --branch main --single-branch https://github.com/mozilla/firefox-translations-models MODELS_UPDATED=1 diff --git a/src/core/ts/config.ts b/src/core/ts/config.ts index 827780a5..fb4a3717 100644 --- a/src/core/ts/config.ts +++ b/src/core/ts/config.ts @@ -5,7 +5,7 @@ export const config = { sentryDsn: process.env.SENTRY_DSN, bergamotModelsBaseUrl: developmentBuild ? "http://0.0.0.0:4000/models" - : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.7", + : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.8", wasmBinariesBaseUrl: developmentBuild ? "http://0.0.0.0:4000/wasm" : "https://storage.googleapis.com/bergamot-models-sandbox/wasm/1", diff --git a/src/core/ts/modelRegistry.ts b/src/core/ts/modelRegistry.ts index c99f168b..4ecfbc5f 100644 --- a/src/core/ts/modelRegistry.ts +++ b/src/core/ts/modelRegistry.ts @@ -10,6 +10,29 @@ export interface ModelRegistry { } export const modelRegistry: ModelRegistry = { + enit: { + vocab: { + name: "vocab.enit.spm", + size: 814128, + estimatedCompressedSize: 405338, + expectedSha256Hash: + "de8cbeb79e0139304bfa47e8559f2447016bf9906225a97d3df1baed4de8f3a3", + }, + lex: { + name: "lex.50.50.enit.s2t.bin", + size: 4489920, + estimatedCompressedSize: 2409986, + expectedSha256Hash: + "bb1fad3b3f6a13ebce1698cf7f39ca736c4dea4525f3dab5e1a78436f07445e6", + }, + model: { + name: "model.enit.intgemm.alphas.bin", + size: 17140836, + estimatedCompressedSize: 13283223, + expectedSha256Hash: + "a5ce3723f62ead92a0e0373b6df0ad8e3e6d22963adb1333984206e33b8b6c61", + }, + }, enpt: { vocab: { name: "vocab.enpt.spm", From 6ef068f2b3ca2d1144022d8b2eda0066cfed394d Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Thu, 2 Sep 2021 12:20:55 -0700 Subject: [PATCH 25/31] Add de-en model (#282) --- import-models.sh | 2 +- src/core/ts/config.ts | 4 ++-- src/core/ts/modelRegistry.ts | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/import-models.sh b/import-models.sh index ea4740fd..4d10e21d 100755 --- a/import-models.sh +++ b/import-models.sh @@ -7,7 +7,7 @@ set -e echo "* Checkout out the relevant revision of the firefox-translations-models repo" MODELS_UPDATED=0 -MODELS_GIT_REV="48b0e0471461778694646166374352a9f68c88f4" # v0.2.8 +MODELS_GIT_REV="139fbf4538e44591613f424248d038c3f5c7b955" # v0.2.9 if [ ! -d "firefox-translations-models" ]; then git clone --branch main --single-branch https://github.com/mozilla/firefox-translations-models MODELS_UPDATED=1 diff --git a/src/core/ts/config.ts b/src/core/ts/config.ts index fb4a3717..9a1effc0 100644 --- a/src/core/ts/config.ts +++ b/src/core/ts/config.ts @@ -5,7 +5,7 @@ export const config = { sentryDsn: process.env.SENTRY_DSN, bergamotModelsBaseUrl: developmentBuild ? "http://0.0.0.0:4000/models" - : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.8", + : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.9", wasmBinariesBaseUrl: developmentBuild ? "http://0.0.0.0:4000/wasm" : "https://storage.googleapis.com/bergamot-models-sandbox/wasm/1", @@ -16,7 +16,7 @@ export const config = { // "German, French, Spanish, Polish, Czech, and Estonian in and out of English" // ISO 639-1 codes // Language pairs that are not available are commented out - // ["de","en"], + ["de", "en"], // ["fr","en"], ["es", "en"], // ["pl","en"], diff --git a/src/core/ts/modelRegistry.ts b/src/core/ts/modelRegistry.ts index 4ecfbc5f..68691c6d 100644 --- a/src/core/ts/modelRegistry.ts +++ b/src/core/ts/modelRegistry.ts @@ -171,6 +171,29 @@ export const modelRegistry: ModelRegistry = { "5b16661e2864dc50b2f4091a16bdd4ec8d8283e04271e602159ba348df5d6e2d", }, }, + deen: { + vocab: { + name: "vocab.deen.spm", + size: 784269, + estimatedCompressedSize: 410738, + expectedSha256Hash: + "417668f2ed297970febafb5b079a9d5ebc4ed0b3550ac8386d67a90473a09bd7", + }, + lex: { + name: "lex.50.50.deen.s2t.bin", + size: 5047568, + estimatedCompressedSize: 2657472, + expectedSha256Hash: + "2f7c0f7bbce97ae5b52454074a892ba7b7610fb98e3c5d341e4ca79f0850c4de", + }, + model: { + name: "model.deen.intgemm.alphas.bin", + size: 17140837, + estimatedCompressedSize: 13091214, + expectedSha256Hash: + "dda44d87ab0d8ad3b3871122fd3ee385f37878183a8b4ec139cd909531ec5009", + }, + }, encs: { vocab: { name: "vocab.csen.spm", From 0208e5dd7eabb6ef264216b5fea53eb64a168bf6 Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Tue, 7 Sep 2021 13:28:06 -0700 Subject: [PATCH 26/31] Update en-ru model (#283) --- import-models.sh | 2 +- src/core/ts/config.ts | 2 +- src/core/ts/modelRegistry.ts | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/import-models.sh b/import-models.sh index 4d10e21d..87bb2a60 100755 --- a/import-models.sh +++ b/import-models.sh @@ -7,7 +7,7 @@ set -e echo "* Checkout out the relevant revision of the firefox-translations-models repo" MODELS_UPDATED=0 -MODELS_GIT_REV="139fbf4538e44591613f424248d038c3f5c7b955" # v0.2.9 +MODELS_GIT_REV="b22ca725bb102c034dabf3871d7349f2aca8d73d" # v0.2.10 if [ ! -d "firefox-translations-models" ]; then git clone --branch main --single-branch https://github.com/mozilla/firefox-translations-models MODELS_UPDATED=1 diff --git a/src/core/ts/config.ts b/src/core/ts/config.ts index 9a1effc0..07c31016 100644 --- a/src/core/ts/config.ts +++ b/src/core/ts/config.ts @@ -5,7 +5,7 @@ export const config = { sentryDsn: process.env.SENTRY_DSN, bergamotModelsBaseUrl: developmentBuild ? "http://0.0.0.0:4000/models" - : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.9", + : "https://storage.googleapis.com/bergamot-models-sandbox/0.2.10", wasmBinariesBaseUrl: developmentBuild ? "http://0.0.0.0:4000/wasm" : "https://storage.googleapis.com/bergamot-models-sandbox/wasm/1", diff --git a/src/core/ts/modelRegistry.ts b/src/core/ts/modelRegistry.ts index 68691c6d..0b4c30ee 100644 --- a/src/core/ts/modelRegistry.ts +++ b/src/core/ts/modelRegistry.ts @@ -59,24 +59,24 @@ export const modelRegistry: ModelRegistry = { enru: { vocab: { name: "vocab.enru.spm", - size: 954844, - estimatedCompressedSize: 439863, + size: 937157, + estimatedCompressedSize: 435776, expectedSha256Hash: - "4bd81da3657acbdf12d22a04d0076bec7ca2db70b978791723113b595b6a831e", + "feca2d44f01b946c85faba3b15b5eb53344bec84cd14a1a4d4a82ddd774c5edd", }, lex: { name: "lex.50.50.enru.s2t.bin", - size: 2981280, - estimatedCompressedSize: 1532704, + size: 3049096, + estimatedCompressedSize: 1579779, expectedSha256Hash: - "2e28a40f9146db615acfe4e836b6fc61d6933417a1ea9640cc015fd7b4546798", + "7bd3e2c0a72286fe1f3da65c56c49a7cd77efa5f1d1a444e2a9e769480b96ff3", }, model: { name: "model.enru.intgemm.alphas.bin", size: 17140836, - estimatedCompressedSize: 12868710, + estimatedCompressedSize: 12853987, expectedSha256Hash: - "b88c646db5af7cf2b8be78450a47e78e55f4f4d2fb14611eada5dcd3e324c212", + "4a45186a93b8a2dd9301c66a3b3dad580b1bcfa74aadda583ca383f9fe0dea93", }, }, iten: { From 7d7d5ee1d96fa43de93560c60c90429f2327bbf8 Mon Sep 17 00:00:00 2001 From: Andre Natal Date: Tue, 7 Sep 2021 13:28:17 -0700 Subject: [PATCH 27/31] Update README.md (#284) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5a927637..efecf41a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +``` +This repo has been archived and the work on Firefox Translations will continue here: https://github.com/mozilla/firefox-translations +``` # Firefox Translations Web Extension [![CircleCI badge](https://img.shields.io/circleci/project/github/mozilla-extensions/firefox-translations/main.svg?label=CircleCI)](https://circleci.com/gh/mozilla-extensions/firefox-translations/) From 2afd6c500391e5dca292d618f4e6c1750f3bae96 Mon Sep 17 00:00:00 2001 From: Andre Natal Date: Tue, 7 Sep 2021 13:31:22 -0700 Subject: [PATCH 28/31] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index efecf41a..14973432 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ ``` -This repo has been archived and the work on Firefox Translations will continue here: https://github.com/mozilla/firefox-translations +This repo has been archived and the work on Firefox Translations will continue here: +https://github.com/mozilla/firefox-translations ``` # Firefox Translations Web Extension From 891a30fe941206658fb01ae5e14eddd304a64f75 Mon Sep 17 00:00:00 2001 From: Motin Date: Fri, 29 Oct 2021 09:08:26 +0300 Subject: [PATCH 29/31] Add instruction to install git-lfs to docs --- docs/dev/engineering/local-development.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/dev/engineering/local-development.md b/docs/dev/engineering/local-development.md index 7dd63d41..227e591d 100644 --- a/docs/dev/engineering/local-development.md +++ b/docs/dev/engineering/local-development.md @@ -6,13 +6,15 @@ The commands in these instructions are meant to be run in the root folder in thi 1. Follow the [Getting started](./getting-started.md) instructions -2. Import bergamot models locally, since the extension downloads the models from a local endpoint when running in development mode: +2. Install [Git Large File Storage](https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage) + +3. Import bergamot models locally, since the extension downloads the models from a local endpoint when running in development mode: ```bash yarn models:import ``` -3. Import Bergamot translator using one of the two methods below. +4. Import Bergamot translator using one of the two methods below. ### Using known-to-work Bergamot Translator WASM artifacts and importing them to the extension From 236f72ee051fae7e996354f0a05271499af1bdc9 Mon Sep 17 00:00:00 2001 From: Motin Date: Fri, 29 Oct 2021 09:11:22 +0300 Subject: [PATCH 30/31] Applied the changeset from https://phabricator.services.mozilla.com/D107755#change-ejYef3i1CbQL that was intended to update the infobar invocation to the changed notification interface in tree --- .../translateUi/TranslationBrowserChromeUi.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js b/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js index 9ddec7b4..a1e1cc61 100644 --- a/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js +++ b/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js @@ -302,15 +302,11 @@ class TranslationBrowserChromeUi { this.translationInfoBarShown = true; const notificationBox = this.notificationBox; const chromeWin = this.browser.ownerGlobal; - const notif = notificationBox.appendNotification( - "", - "translation", - null, - notificationBox.PRIORITY_INFO_HIGH, - null, - null, - `translation-notification-${chromeWin.now}`, - ); + const notif = notificationBox.appendNotification("translation", { + label: "", + priority: notificationBox.PRIORITY_INFO_HIGH, + is: `translation-notification-${chromeWin.now}`, + }); notif.init(this.translationBrowserChromeUiNotificationManager); this.translationBrowserChromeUiNotificationManager.infobarDisplayed( notif._getSourceLang(), From ca7bbd0c866f9161affe9f87a8a5c0006346ff9d Mon Sep 17 00:00:00 2001 From: Motin Date: Fri, 29 Oct 2021 09:14:22 +0300 Subject: [PATCH 31/31] Fixed typo in the changeset from https://phabricator.services.mozilla.com/D107755#change-ejYef3i1CbQL that was intended to update the infobar invocation to the changed notification interface in tree --- .../experiment-apis/translateUi/TranslationBrowserChromeUi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js b/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js index a1e1cc61..05f5bb21 100644 --- a/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js +++ b/src/firefox-infobar-ui/static/experiment-apis/translateUi/TranslationBrowserChromeUi.js @@ -305,7 +305,7 @@ class TranslationBrowserChromeUi { const notif = notificationBox.appendNotification("translation", { label: "", priority: notificationBox.PRIORITY_INFO_HIGH, - is: `translation-notification-${chromeWin.now}`, + notificationIs: `translation-notification-${chromeWin.now}`, }); notif.init(this.translationBrowserChromeUiNotificationManager); this.translationBrowserChromeUiNotificationManager.infobarDisplayed(