From 5a5a29e614a7629a47d97fe4b689e245cf4a2b84 Mon Sep 17 00:00:00 2001 From: Daosheng Mu Date: Tue, 18 Sep 2018 11:00:28 -0700 Subject: [PATCH] Add telemetry for uri count and loading time. --- app/build.gradle | 2 +- .../org/mozilla/vrbrowser/SessionStore.java | 14 ++ .../vrbrowser/telemetry/TelemetryWrapper.java | 96 ++++++++++- .../org/mozilla/vrbrowser/utils/UrlUtils.java | 160 ++++++++++++++++++ 4 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 app/src/common/shared/org/mozilla/vrbrowser/utils/UrlUtils.java diff --git a/app/build.gradle b/app/build.gradle index 6bc8111d53..8b5d95fbd8 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -187,7 +187,7 @@ dependencies { svrImplementation fileTree(dir: "${project.rootDir}/third_party/svr/", include: ['*.jar']) implementation 'com.android.support:design:27.1.1' implementation 'com.google.vr:sdk-audio:1.170.0' - implementation "org.mozilla.components:telemetry:0.10" + implementation "org.mozilla.components:telemetry:0.23" implementation "com.github.mozilla:mozillaspeechlibrary:1.0.4" } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/SessionStore.java b/app/src/common/shared/org/mozilla/vrbrowser/SessionStore.java index c84674ded6..7837c932ef 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/SessionStore.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/SessionStore.java @@ -7,6 +7,7 @@ import android.content.Context; import android.graphics.Rect; +import android.os.SystemClock; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.util.Log; @@ -16,6 +17,8 @@ import org.mozilla.gecko.GeckoAppShell; import org.mozilla.gecko.GeckoProfile; import org.mozilla.geckoview.*; +import org.mozilla.vrbrowser.telemetry.TelemetryWrapper; +import org.mozilla.vrbrowser.utils.UrlUtils; import java.io.File; import java.io.FileNotFoundException; @@ -45,6 +48,8 @@ public static SessionStore get() { private LinkedList mSessionChangeListeners; private LinkedList mTextInputListeners; private LinkedList mPromptListeners; + private final long MIN_LOAD_TIME = 40; + private long startLoadTime = 0; public interface SessionChangeListener { void onNewSession(GeckoSession aSession, int aId); @@ -823,6 +828,7 @@ public void onPageStart(GeckoSession aSession, String aUri) { return; } state.mIsLoading = true; + startLoadTime = SystemClock.elapsedRealtime(); for (GeckoSession.ProgressDelegate listener: mProgressListeners) { listener.onPageStart(aSession, aUri); } @@ -837,6 +843,14 @@ public void onPageStop(GeckoSession aSession, boolean b) { } state.mIsLoading = false; + long elapsedLoad = SystemClock.elapsedRealtime() - startLoadTime; + if (elapsedLoad > MIN_LOAD_TIME + && !state.mUri.equals(getHomeUri()) + && !state.mUri.equals(HOME_WITHOUT_REGION_ORIGIN) + && !UrlUtils.isLocalizedContent(state.mUri)) { + Log.i(LOGTAG, "Sent load to histogram"); + TelemetryWrapper.addLoadToHistogram(state.mUri, elapsedLoad); + } for (GeckoSession.ProgressDelegate listener: mProgressListeners) { listener.onPageStop(aSession, b); } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/telemetry/TelemetryWrapper.java b/app/src/common/shared/org/mozilla/vrbrowser/telemetry/TelemetryWrapper.java index c3407d6222..2e312e089e 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/telemetry/TelemetryWrapper.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/telemetry/TelemetryWrapper.java @@ -4,6 +4,8 @@ import android.content.res.Resources; import android.os.StrictMode; import android.support.annotation.UiThread; +import android.util.Log; + import org.mozilla.telemetry.Telemetry; import org.mozilla.telemetry.TelemetryHolder; import org.mozilla.telemetry.config.TelemetryConfiguration; @@ -19,31 +21,51 @@ import org.mozilla.vrbrowser.R; import org.mozilla.vrbrowser.SettingsStore; import org.mozilla.vrbrowser.search.SearchEngine; +import org.mozilla.vrbrowser.utils.UrlUtils; + +import java.net.URI; +import java.util.HashSet; +import static java.lang.Math.toIntExact; public class TelemetryWrapper { private final static String APP_NAME = "FirefoxReality"; + private final static String LOGTAG = "VRB"; + private final static int HISTOGRAM_SIZE = 200; + private final static int BUCKET_SIZE_MS = 100; + private final static int HISTOGRAM_MIN_INDEX = 0; + + private static HashSet domainMap = new HashSet(); + private static int[] histogram = new int[HISTOGRAM_SIZE]; + private static int numUri = 0; private class Category { private static final String ACTION = "action"; + private static final String HISTOGRAM = "histogram"; } private class Method { private static final String FOREGROUND = "foreground"; private static final String BACKGROUND = "background"; + private static final String OPEN = "open"; private static final String TYPE_URL = "type_url"; private static final String TYPE_QUERY = "type_query"; // TODO: Support "select_query" after providing search suggestion. private static final String VOICE_QUERY = "voice_query"; - } private class Object { private static final String APP = "app"; + private static final String BROWSER = "browser"; private static final String SEARCH_BAR = "search_bar"; private static final String VOICE_INPUT = "voice_input"; } + private class Extra { + private static final String TOTAL_URI_COUNT = "total_uri_count"; + private static final String UNIQUE_DOMAINS_COUNT = "unique_domains_count"; + } + // We should call this at the application initial stage. Instead, // it would be called when users turn on/off the setting of telemetry. // e.g., SettingsStore.getInstance(context).setTelemetryEnabled(); @@ -84,6 +106,29 @@ public static void start() { @UiThread public static void stop() { + TelemetryEvent histogramEvent = TelemetryEvent.create(Category.HISTOGRAM, Method.FOREGROUND, Object.BROWSER); + for (int bucketIndex = 0; bucketIndex < histogram.length; ++bucketIndex) { + histogramEvent.extra(Integer.toString(bucketIndex * BUCKET_SIZE_MS), Integer.toString(histogram[bucketIndex])); + } + histogramEvent.queue(); + + // Clear histogram array after queueing it + histogram = new int[HISTOGRAM_SIZE]; + + // We only upload the domain and URI counts to the probes without including + // users' URI info. + TelemetryEvent.create(Category.ACTION, Method.OPEN, Object.BROWSER).extra( + Extra.UNIQUE_DOMAINS_COUNT, + Integer.toString(domainMap.size()) + ).queue(); + domainMap.clear(); + + TelemetryEvent.create(Category.ACTION, Method.OPEN, Object.BROWSER).extra( + Extra.TOTAL_URI_COUNT, + Integer.toString(numUri) + ).queue(); + numUri = 0; + TelemetryEvent.create(Category.ACTION, Method.BACKGROUND, Object.APP).queue(); TelemetryHolder.get().recordSessionEnd(); @@ -93,6 +138,36 @@ public static void stop() { .scheduleUpload(); } + @UiThread + public static void stopSession() { + TelemetryHolder.get().recordSessionEnd(); + + TelemetryEvent histogramEvent = TelemetryEvent.create(Category.HISTOGRAM, Method.FOREGROUND, Object.BROWSER); + for (int bucketIndex = 0; bucketIndex < histogram.length; ++bucketIndex) { + histogramEvent.extra(Integer.toString(bucketIndex * BUCKET_SIZE_MS), Integer.toString(histogram[bucketIndex])); + } + histogramEvent.queue(); + + // Clear histogram array after queueing it + histogram = new int[HISTOGRAM_SIZE]; + + // We only upload the domain and URI counts to the probes without including + // users' URI info. + TelemetryEvent.create(Category.ACTION, Method.OPEN, Object.BROWSER).extra( + Extra.UNIQUE_DOMAINS_COUNT, + Integer.toString(domainMap.size()) + ).queue(); + domainMap.clear(); + + TelemetryEvent.create(Category.ACTION, Method.OPEN, Object.BROWSER).extra( + Extra.TOTAL_URI_COUNT, + Integer.toString(numUri) + ).queue(); + numUri = 0; + + TelemetryEvent.create(Category.ACTION, Method.BACKGROUND, Object.APP).queue(); + } + @UiThread public static void urlBarEvent(boolean aIsUrl) { if (aIsUrl) { @@ -129,5 +204,24 @@ private static void browseEvent() { // TODO: Working on autocomplete result. event.queue(); } + + @UiThread + public static void addLoadToHistogram(String uri, Long newLoadTime) { + domainMap.add(UrlUtils.stripCommonSubdomains(URI.create(uri).getHost())); + numUri++; + int histogramLoadIndex = toIntExact(newLoadTime / BUCKET_SIZE_MS); + + if (histogramLoadIndex > (HISTOGRAM_SIZE - 2)) { + histogramLoadIndex = HISTOGRAM_SIZE - 1; + } else if (histogramLoadIndex < HISTOGRAM_MIN_INDEX) { + histogramLoadIndex = HISTOGRAM_MIN_INDEX; + } + + if (histogramLoadIndex >= histogram.length) { + Log.e(LOGTAG, "the histogram size is overflow."); + histogramLoadIndex = histogram.length - 1; + } + histogram[histogramLoadIndex]++; + } } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/utils/UrlUtils.java b/app/src/common/shared/org/mozilla/vrbrowser/utils/UrlUtils.java new file mode 100644 index 0000000000..2d6c28f0fb --- /dev/null +++ b/app/src/common/shared/org/mozilla/vrbrowser/utils/UrlUtils.java @@ -0,0 +1,160 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.vrbrowser.utils; + +import android.net.Uri; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.text.TextUtils; +import android.webkit.URLUtil; + +import java.net.URI; +import java.net.URISyntaxException; + + +// This class refers from mozilla-mobile/focus-android +public class UrlUtils { + public static String normalize(@NonNull String input) { + String trimmedInput = input.trim(); + Uri uri = Uri.parse(trimmedInput); + + if (TextUtils.isEmpty(uri.getScheme())) { + uri = Uri.parse("http://" + trimmedInput); + } + + return uri.toString(); + } + + /** + * Is the given string a URL or should we perform a search? + * + * TODO: This is a super simple and probably stupid implementation. + */ + public static boolean isUrl(String url) { + String trimmedUrl = url.trim(); + if (trimmedUrl.contains(" ")) { + return false; + } + + return trimmedUrl.contains(".") || trimmedUrl.contains(":"); + } + + public static boolean isValidSearchQueryUrl(String url) { + String trimmedUrl = url.trim(); + if (!trimmedUrl.matches("^.+?://.+?")) { + // UI hint url doesn't have http scheme, so add it if necessary + trimmedUrl = "http://" + trimmedUrl; + } + + final boolean isNetworkUrl = URLUtil.isNetworkUrl(trimmedUrl); + final boolean containsToken = trimmedUrl.contains("%s"); + + return isNetworkUrl && containsToken; + } + + public static boolean isHttpOrHttps(String url) { + if (TextUtils.isEmpty(url)) { + return false; + } + + return url.startsWith("http:") || url.startsWith("https:"); + } + + public static String stripUserInfo(@Nullable String url) { + if (TextUtils.isEmpty(url)) { + return ""; + } + + try { + URI uri = new URI(url); + + final String userInfo = uri.getUserInfo(); + if (userInfo == null) { + return url; + } + + // Strip the userInfo to minimise spoofing ability. This only affects what's shown + // during browsing, this information isn't used when we start editing the URL: + uri = new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); + + return uri.toString(); + } catch (URISyntaxException e) { + // We might be trying to display a user-entered URL (which could plausibly contain errors), + // in this case its safe to just return the raw input. + // There are also some special cases that URI can't handle, such as "http:" by itself. + return url; + } + } + + public static boolean isPermittedResourceProtocol(@Nullable final String scheme) { + return scheme != null && ( + scheme.startsWith("http") || + scheme.startsWith("https") || + scheme.startsWith("file") || + scheme.startsWith("data") || + scheme.startsWith("javascript") || + scheme.startsWith("about")); + } + + public static boolean isSupportedProtocol(@Nullable final String scheme) { + return scheme != null && (isPermittedResourceProtocol(scheme) || scheme.startsWith("error")); + } + + public static boolean isInternalErrorURL(final String url) { + return "data:text/html;charset=utf-8;base64,".equals(url); + } + + /** + * Checks that urls are non-null and are the same aside from a trailing slash. + * + * @return true if urls are the same except for trailing slash, or if either url is null. + */ + public static boolean urlsMatchExceptForTrailingSlash(final String url1, final String url2) { + // This is a hack to catch a NPE in issue #26. + if (url1 == null || url2 == null) { + return false; + } + int lengthDifference = url1.length() - url2.length(); + + if (lengthDifference == 0) { + // The simplest case: + return url1.equalsIgnoreCase(url2); + } else if (lengthDifference == 1) { + // url1 is longer: + return url1.charAt(url1.length() - 1) == '/' && + url1.regionMatches(true, 0, url2, 0, url2.length()); + } else if (lengthDifference == -1) { + return url2.charAt(url2.length() - 1) == '/' && + url2.regionMatches(true, 0, url1, 0, url1.length()); + } + + return false; + } + + public static String stripCommonSubdomains(@Nullable String host) { + if (host == null) { + return null; + } + + // In contrast to desktop, we also strip mobile subdomains, + // since its unlikely users are intentionally typing them + int start = 0; + + if (host.startsWith("www.")) { + start = 4; + } else if (host.startsWith("mobile.")) { + start = 7; + } else if (host.startsWith("m.")) { + start = 2; + } + + return host.substring(start); + } + + public static boolean isLocalizedContent(@Nullable String url) { + return url != null && (url.equals("about:blank")); + } +}