Skip to content

Commit

Permalink
Add or improve analytics for audit logging, form updates, servers, ex…
Browse files Browse the repository at this point in the history 
…ternal apps and external CSV features (getodk#2545)

* Log analytics event when audit logging enabled

* Log analytics event when form update settings change

* Add known hosts to analytics event logged on server change

* Add form identifier hash to external app analytics event

This is to differentiate between a form with many enumerators that uses an external app vs many forms that use external apps.

* Log analytics events for external CSV features
  • Loading branch information
@lognaturel @grzesiek2010
lognaturel authored and grzesiek2010 committed Sep 19, 2018
1 parent 008c9c5 commit 7e0fc9b
Show file tree
Hide file tree
Showing 8 changed files with 123 additions and 33 deletions.
22 changes: 22 additions & 0 deletions collect_app/src/main/java/org/odk/collect/android/application/Collect.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@
import org.odk.collect.android.preferences.AutoSendPreferenceMigrator;
import org.odk.collect.android.preferences.FormMetadataMigrator;
import org.odk.collect.android.preferences.GeneralSharedPreferences;
import org.odk.collect.android.utilities.FileUtils;
import org.odk.collect.android.utilities.LocaleHelper;
import org.odk.collect.android.utilities.PRNGFixes;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.util.Locale;

Expand Down Expand Up @@ -345,6 +347,26 @@ public void setComponent(AppComponent applicationComponent) {
this.applicationComponent = applicationComponent;
}

/**
* Gets a unique, privacy-preserving identifier for the current form.
*
* @return md5 hash of the form title, a space, the form ID
*/
public static String getCurrentFormIdentifierHash() {
String formIdentifier = "";
FormController formController = getInstance().getFormController();
if (formController != null) {
if (formController.getFormDef() != null) {
String formID = formController.getFormDef().getMainInstance()
.getRoot().getAttributeValue("", "id");
formIdentifier = formController.getFormTitle() + " " + formID;
}
}

return FileUtils.getMd5Hash(
new ByteArrayInputStream(formIdentifier.getBytes()));
}

@Override
public DispatchingAndroidInjector<Activity> activityInjector() {
return androidInjector;
Expand Down
10 changes: 10 additions & 0 deletions collect_app/src/main/java/org/odk/collect/android/external/ExternalDataUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
package org.odk.collect.android.external;

import android.widget.Toast;

import com.google.android.gms.analytics.HitBuilders;

import org.javarosa.core.model.SelectChoice;
import org.javarosa.core.model.condition.EvaluationContext;
import org.javarosa.core.model.instance.FormInstance;
Expand Down Expand Up @@ -107,6 +110,13 @@ public static XPathFuncExpr getSearchXPathExpression(String appearance) {

Matcher matcher = SEARCH_FUNCTION_REGEX.matcher(appearance);
if (matcher.find()) {
Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("ExternalData")
.setAction("search()")
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());

String function = matcher.group(0);
try {
XPathExpression xpathExpression = XPathParseTool.parseXPath(function);
Expand Down
10 changes: 10 additions & 0 deletions ...t_app/src/main/java/org/odk/collect/android/external/handler/ExternalDataHandlerPull.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteException;

import com.google.android.gms.analytics.HitBuilders;

import org.javarosa.core.model.condition.EvaluationContext;
import org.javarosa.xpath.expr.XPathFuncExpr;
import org.odk.collect.android.application.Collect;
import org.odk.collect.android.external.ExternalDataManager;
import org.odk.collect.android.external.ExternalDataUtil;
import org.odk.collect.android.external.ExternalSQLiteOpenHelper;
Expand Down Expand Up @@ -68,6 +71,13 @@ public boolean realTime() {

@Override
public Object eval(Object[] args, EvaluationContext ec) {
Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("ExternalData")
.setAction("pulldata()")
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());

if (args.length != 4) {
Timber.e("4 arguments are needed to evaluate the %s function", HANDLER_NAME);
return "";
Expand Down
10 changes: 10 additions & 0 deletions collect_app/src/main/java/org/odk/collect/android/logic/FormController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

import android.support.annotation.Nullable;

import com.google.android.gms.analytics.HitBuilders;

import org.javarosa.core.model.CoreModelModule;
import org.javarosa.core.model.FormDef;
import org.javarosa.core.model.FormIndex;
Expand Down Expand Up @@ -45,6 +47,7 @@
import org.javarosa.xform.parse.XFormParser;
import org.javarosa.xpath.XPathParseTool;
import org.javarosa.xpath.expr.XPathExpression;
import org.odk.collect.android.application.Collect;
import org.odk.collect.android.exception.JavaRosaException;
import org.odk.collect.android.utilities.TimerLogger;
import org.odk.collect.android.views.ODKView;
Expand Down Expand Up @@ -1142,6 +1145,13 @@ public InstanceMetadata getSubmissionMetadata() {
// timing element...
v = e.getChildrenWithName(AUDIT);
if (v.size() == 1) {
Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("AuditLogging")
.setAction("Enabled")
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());

audit = true;
IAnswerData answerData = new StringData();
answerData.setValue(AUDIT_FILE_NAME);
Expand Down
29 changes: 28 additions & 1 deletion collect_app/src/main/java/org/odk/collect/android/preferences/FormManagementPreferences.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
import android.support.annotation.Nullable;
import android.view.View;

import com.google.android.gms.analytics.HitBuilders;

import org.odk.collect.android.R;
import org.odk.collect.android.application.Collect;
import org.odk.collect.android.tasks.ServerPollingJob;

import static org.odk.collect.android.preferences.AdminKeys.ALLOW_OTHER_WAYS_OF_EDITING_FORM;
Expand Down Expand Up @@ -69,8 +72,17 @@ private void initListPref(String key) {
int index = ((ListPreference) preference).findIndexOfValue(newValue.toString());
CharSequence entry = ((ListPreference) preference).getEntries()[index];
preference.setSummary(entry);

if (key.equals(KEY_PERIODIC_FORM_UPDATES_CHECK)) {
ServerPollingJob.schedulePeriodicJob((String) newValue);

Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("PreferenceChange")
.setAction("Periodic form updates check")
.setLabel((String) newValue)
.build());

if (newValue.equals(getString(R.string.never_value))) {
Preference automaticUpdatePreference = findPreference(KEY_AUTOMATIC_UPDATE);
if (automaticUpdatePreference != null) {
Expand All @@ -92,7 +104,22 @@ private void initPref(String key) {

if (pref != null) {
if (key.equals(KEY_AUTOMATIC_UPDATE)) {
pref.setEnabled(!GeneralSharedPreferences.getInstance().get(KEY_PERIODIC_FORM_UPDATES_CHECK).equals(getString(R.string.never_value)));
String formUpdateCheckPeriod = (String) GeneralSharedPreferences.getInstance()
.get(KEY_PERIODIC_FORM_UPDATES_CHECK);

// Only enable automatic form updates if periodic updates are set
pref.setEnabled(!formUpdateCheckPeriod.equals(getString(R.string.never_value)));

pref.setOnPreferenceChangeListener((preference, newValue) -> {
Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("PreferenceChange")
.setAction("Automatic form updates")
.setLabel(newValue + " " + formUpdateCheckPeriod)
.build());

return true;
});
}
}
}
Expand Down
42 changes: 33 additions & 9 deletions collect_app/src/main/java/org/odk/collect/android/preferences/ServerPreferencesFragment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,15 +307,7 @@ private Preference.OnPreferenceChangeListener createChangeListener() {
}

if (Validator.isUrlValid(url)) {
String prefix = url.split(":")[0].toUpperCase(Locale.ENGLISH);
String urlHash = FileUtils.getMd5Hash(
new ByteArrayInputStream(url.getBytes()));
Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("SetServer")
.setAction(prefix)
.setLabel(urlHash)
.build());
sendAnalyticsEvent(url);

preference.setSummary(newValue.toString());
SharedPreferences prefs = PreferenceManager
Expand Down Expand Up @@ -402,6 +394,38 @@ private Preference.OnPreferenceChangeListener createChangeListener() {
};
}

/**
* Remotely log the URL scheme, whether the URL is on one of 3 common hosts, and a URL hash.
* This will help inform decisions on whether or not to allow insecure server configurations
* (HTTP) and on which hosts to strengthen support for.
*
* @param url the URL that the server setting has just been set to
*/
private void sendAnalyticsEvent(String url) {
String upperCaseURL = url.toUpperCase(Locale.ENGLISH);
String scheme = upperCaseURL.split(":")[0];

String host = "Other";
if (upperCaseURL.contains("APPSPOT")) {
host = "Appspot";
} else if (upperCaseURL.contains("KOBOTOOLBOX.ORG") ||
upperCaseURL.contains("HUMANITARIANRESPONSE.INFO")) {
host = "Kobo";
} else if (upperCaseURL.contains("ONA.IO")) {
host = "Ona";
}

String urlHash = FileUtils.getMd5Hash(
new ByteArrayInputStream(url.getBytes()));

Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("SetServer")
.setAction(scheme + " " + host)
.setLabel(urlHash)
.build());
}

private void maskPasswordSummary(String password) {
passwordPreference.setSummary(password != null && password.length() > 0
? "********"
Expand Down
1 change: 1 addition & 0 deletions collect_app/src/main/java/org/odk/collect/android/widgets/ExStringWidget.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ public ExStringWidget(Context context, FormEntryPrompt prompt) {
.send(new HitBuilders.EventBuilder()
.setCategory("WidgetType")
.setAction("ExternalApp")
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());

}
Expand Down
32 changes: 9 additions & 23 deletions collect_app/src/main/java/org/odk/collect/android/widgets/WidgetFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,7 @@
import org.odk.collect.android.R;
import org.odk.collect.android.activities.WebViewActivity;
import org.odk.collect.android.application.Collect;
import org.odk.collect.android.logic.FormController;
import org.odk.collect.android.utilities.FileUtils;

import java.io.ByteArrayInputStream;
import java.util.List;
import java.util.Locale;

Expand Down Expand Up @@ -136,6 +133,13 @@ public static QuestionWidget createWidgetFromPrompt(FormEntryPrompt fep, Context
String query = fep.getQuestion().getAdditionalAttribute(null, "query");
if (query != null) {
questionWidget = new ItemsetWidget(context, fep, appearance.startsWith("quick"));

Collect.getInstance().getDefaultTracker()
.send(new HitBuilders.EventBuilder()
.setCategory("ExternalData")
.setAction("External itemset")
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());
} else if (appearance.startsWith("printer")) {
questionWidget = new ExPrinterWidget(context, fep);
} else if (appearance.startsWith("ex:")) {
Expand Down Expand Up @@ -317,7 +321,7 @@ private static boolean logChoiceFilterAnalytics(QuestionDef question) {
.send(new HitBuilders.EventBuilder()
.setCategory("Itemset")
.setAction(actionName)
.setLabel(getFormIdentifierHash())
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());

if (predicate.toString().contains("current")) {
Expand All @@ -331,24 +335,6 @@ private static boolean logChoiceFilterAnalytics(QuestionDef question) {
return false;
}

/**
* Gets a unique, privacy-preserving identifier for the current form.
*
* @return hash of the form title, a space, the form ID
*/
private static String getFormIdentifierHash() {
String formIdentifier = "";
FormController formController = Collect.getInstance().getFormController();
if (formController != null) {
String formID = formController.getFormDef().getMainInstance()
.getRoot().getAttributeValue("", "id");
formIdentifier = formController.getFormTitle() + " " + formID;
}

return FileUtils.getMd5Hash(
new ByteArrayInputStream(formIdentifier.getBytes()));
}

/**
* Show an alert explaining the upcoming change in current() predicates.
*/
Expand All @@ -366,7 +352,7 @@ private static void showCurrentPredicateAlert(Context context) {
.send(new HitBuilders.EventBuilder()
.setCategory("Itemset")
.setAction("CurrentChangeViewed")
.setLabel(getFormIdentifierHash())
.setLabel(Collect.getCurrentFormIdentifierHash())
.build());
};

Expand Down

0 comments on commit 7e0fc9b

Please sign in to comment.