add FDroidMetricsWorker to gather data into JSON reports

app/src/androidTest/java/org/fdroid/fdroid/work/FDroidMetricsWorkerTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2021  Hans-Christoph Steiner <hans@eds.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 3
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.fdroid.fdroid.work;
+
+import androidx.arch.core.executor.testing.InstantTaskExecutorRule;
+import androidx.test.filters.LargeTest;
+import androidx.test.platform.app.InstrumentationRegistry;
+import androidx.work.OneTimeWorkRequest;
+import androidx.work.WorkInfo;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This actually runs {@link FDroidMetricsWorker} on a device/emulator and
+ * submits a report to https://metrics.cleaninsights.org
+ * <p>
+ * This is marked with {@link LargeTest} to exclude it from running on GitLab CI
+ * because it always fails on the emulator tests there.  Also, it actually submits
+ * a report.
+ */
+@LargeTest
+public class FDroidMetricsWorkerTest {
+    public static final String TAG = "FDroidMetricsWorkerTest";
+
+    @Rule
+    public InstantTaskExecutorRule instantTaskExecutorRule = new InstantTaskExecutorRule();
+
+    @Rule
+    public WorkManagerTestRule workManagerTestRule = new WorkManagerTestRule();
+
+    /**
+     * A test for easy manual testing.
+     */
+    @Ignore
+    @Test
+    public void testGenerateReport() throws IOException {
+        String json = FDroidMetricsWorker.generateReport(
+                InstrumentationRegistry.getInstrumentation().getTargetContext());
+        System.out.println(json);
+    }
+
+    @Test
+    public void testWorkRequest() throws ExecutionException, InterruptedException {
+        OneTimeWorkRequest request = new OneTimeWorkRequest.Builder(FDroidMetricsWorker.class).build();
+        workManagerTestRule.workManager.enqueue(request).getResult();
+        ListenableFuture<WorkInfo> workInfo = workManagerTestRule.workManager.getWorkInfoById(request.getId());
+        assertEquals(WorkInfo.State.SUCCEEDED, workInfo.get().getState());
+    }
+}

app/src/main/java/org/fdroid/fdroid/FDroidApp.java

@@ -677,7 +677,7 @@ public class FDroidApp extends Application implements androidx.work.Configuratio
      * Set up WorkManager on demand to avoid slowing down starts.
      *
      * @see CleanCacheWorker
-     * @see org.fdroid.fdroid.work.PopularityContestWorker
+     * @see org.fdroid.fdroid.work.FDroidMetricsWorker
      * @see org.fdroid.fdroid.work.UpdateWorker
      * @see <a href="https://developer.android.com/codelabs/android-adv-workmanager#3">example</a>
      */

app/src/main/java/org/fdroid/fdroid/Utils.java

@@ -811,6 +811,10 @@ public final class Utils {
         return versionName;
     }
 
+    public static String getUserAgent() {
+        return "F-Droid " + BuildConfig.VERSION_NAME;
+    }
+
     /**
      * Try to get the {@link PackageInfo} for the {@code packageName} provided.
      *

app/src/main/java/org/fdroid/fdroid/installer/InstallHistoryService.java

@@ -26,8 +26,8 @@ import android.content.Intent;
 import android.content.IntentFilter;
 import android.net.Uri;
 import android.os.Process;
-import androidx.localbroadcastmanager.content.LocalBroadcastManager;
 import android.text.TextUtils;
+import androidx.localbroadcastmanager.content.LocalBroadcastManager;
 import org.fdroid.fdroid.Utils;
 import org.fdroid.fdroid.data.Apk;
 
@@ -89,6 +89,12 @@ public class InstallHistoryService extends IntentService {
         context.startService(intent);
     }
 
+    public static File getInstallHistoryFile(Context context) {
+        File installHistoryDir = new File(context.getCacheDir(), "install_history");
+        installHistoryDir.mkdir();
+        return new File(installHistoryDir, "all");
+    }
+
     public InstallHistoryService() {
         super("InstallHistoryService");
     }
@@ -112,9 +118,7 @@ public class InstallHistoryService extends IntentService {
         values.add(String.valueOf(versionCode));
         values.add(intent.getAction());
 
-        File installHistoryDir = new File(getCacheDir(), "install_history");
+        File logFile = getInstallHistoryFile(this);
-        installHistoryDir.mkdir();
-        File logFile = new File(installHistoryDir, "all");
         FileWriter fw = null;
         PrintWriter out = null;
         try {

app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java

@@ -28,7 +28,6 @@ import android.text.TextUtils;
 import android.util.Base64;
 import info.guardianproject.netcipher.NetCipher;
 import org.apache.commons.io.FileUtils;
-import org.fdroid.fdroid.BuildConfig;
 import org.fdroid.fdroid.FDroidApp;
 import org.fdroid.fdroid.Utils;
 
@@ -195,7 +194,7 @@ public class HttpDownloader extends Downloader {
                 && FDroidApp.subnetInfo.isInRange(host); // on the same subnet as we are
     }
 
-    private HttpURLConnection getConnection() throws SocketTimeoutException, IOException {
+    HttpURLConnection getConnection() throws SocketTimeoutException, IOException {
         HttpURLConnection connection;
         if (isSwapUrl(sourceUrl)) {
             // swap never works with a proxy, its unrouted IP on the same subnet
@@ -209,7 +208,7 @@ public class HttpDownloader extends Downloader {
             }
         }
 
-        connection.setRequestProperty("User-Agent", "F-Droid " + BuildConfig.VERSION_NAME);
+        connection.setRequestProperty("User-Agent", Utils.getUserAgent());
         connection.setConnectTimeout(getTimeout());
         connection.setReadTimeout(getTimeout());
 

app/src/main/java/org/fdroid/fdroid/net/HttpPoster.java

@@ -0,0 +1,47 @@
+package org.fdroid.fdroid.net;
+
+import android.net.Uri;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+
+/**
+ * HTTP POST a JSON string to the URL configured in the constructor.
+ */
+public class HttpPoster extends HttpDownloader {
+
+    public HttpPoster(String url) throws FileNotFoundException, MalformedURLException {
+        this(Uri.parse(url), null);
+    }
+
+    HttpPoster(Uri uri, File destFile) throws FileNotFoundException, MalformedURLException {
+        super(uri, destFile);
+    }
+
+    /**
+     * @return The HTTP Status Code
+     */
+    public void post(String json) throws IOException {
+        HttpURLConnection connection = getConnection();
+        connection.setRequestMethod("POST");
+        connection.setRequestProperty("Content-Type", "application/json; utf-8");
+        connection.setDoOutput(true);
+        OutputStream os = connection.getOutputStream();
+        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
+        writer.write(json, 0, json.length());
+        writer.flush();
+        writer.close();
+        os.close();
+        connection.connect();
+        int statusCode = connection.getResponseCode();
+        if (statusCode < 200 || statusCode >= 300) {
+            throw new IOException("HTTP POST failed with " + statusCode + " " + connection.getResponseMessage());
+        }
+    }
+}

app/src/main/java/org/fdroid/fdroid/work/FDroidMetricsWorker.java

@@ -0,0 +1,447 @@
+/*
+ * Copyright (C) 2021  Hans-Christoph Steiner <hans@eds.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 3
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.fdroid.fdroid.work;
+
+import android.content.Context;
+import android.content.pm.PackageInfo;
+import android.content.pm.PackageManager;
+import android.os.Build;
+import android.system.Os;
+import android.text.format.DateUtils;
+import androidx.annotation.NonNull;
+import androidx.work.Constraints;
+import androidx.work.ExistingPeriodicWorkPolicy;
+import androidx.work.ListenableWorker;
+import androidx.work.PeriodicWorkRequest;
+import androidx.work.WorkManager;
+import androidx.work.Worker;
+import androidx.work.WorkerParameters;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import org.apache.commons.io.FileUtils;
+import org.fdroid.fdroid.Preferences;
+import org.fdroid.fdroid.Utils;
+import org.fdroid.fdroid.data.App;
+import org.fdroid.fdroid.data.InstalledAppProvider;
+import org.fdroid.fdroid.installer.InstallHistoryService;
+import org.fdroid.fdroid.net.HttpPoster;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This gathers all the information needed for F-Droid Metrics, aka the
+ * "Popularity Contest", and submits it to the Clean Insights Matomo.  This
+ * should <b>never</b> include any Personally Identifiable Information (PII)
+ * like telephone numbers, IP Addresses, MAC, SSID, IMSI, IMEI, user accounts,
+ * etc.
+ * <p>
+ * This uses static methods so that they can easily be tested in Robolectric
+ * rather than painful, slow, flaky emulator tests.
+ */
+public class FDroidMetricsWorker extends Worker {
+
+    public static final String TAG = "FDroidMetricsWorker";
+
+    static SimpleDateFormat weekFormatter = new SimpleDateFormat("yyyy ww", Locale.ENGLISH);
+
+    public FDroidMetricsWorker(@NonNull Context context, @NonNull WorkerParameters workerParams) {
+        super(context, workerParams);
+    }
+
+    /**
+     * Schedule or cancel a work request to update the app index, according to the
+     * current preferences.  It is meant to run weekly, so it will schedule one week
+     * from the last run.  If it has never been run, it will run as soon as possible.
+     * <p>
+     * Although {@link Constraints.Builder#setRequiresDeviceIdle(boolean)} is available
+     * down to {@link Build.VERSION_CODES#M}, it will cause {@code UpdateService} to
+     * rarely run, if ever on some devices.  So {@link Constraints.Builder#setRequiresDeviceIdle(boolean)}
+     * should only be used in conjunction with
+     * {@link Constraints.Builder#setTriggerContentMaxDelay(long, TimeUnit)} to ensure
+     * that updates actually happen regularly.
+     */
+    public static void schedule(final Context context) {
+        final WorkManager workManager = WorkManager.getInstance(context);
+        long interval = TimeUnit.DAYS.toMillis(7);
+
+        final Constraints.Builder constraintsBuilder = new Constraints.Builder()
+                .setRequiresCharging(true)
+                .setRequiresBatteryNotLow(true);
+        // TODO use the Data/WiFi preferences here
+        if (Build.VERSION.SDK_INT >= 24) {
+            constraintsBuilder.setTriggerContentMaxDelay(interval, TimeUnit.MILLISECONDS);
+            constraintsBuilder.setRequiresDeviceIdle(true);
+        }
+        final PeriodicWorkRequest cleanCache =
+                new PeriodicWorkRequest.Builder(FDroidMetricsWorker.class, interval, TimeUnit.MILLISECONDS)
+                        .setConstraints(constraintsBuilder.build())
+                        .build();
+        workManager.enqueueUniquePeriodicWork(TAG, ExistingPeriodicWorkPolicy.REPLACE, cleanCache);
+        Utils.debugLog(TAG, "Scheduled periodic work");
+    }
+
+    @NonNull
+    @Override
+    public Result doWork() {
+        // TODO check useTor preference and force-submit over Tor.
+        String json = generateReport(getApplicationContext());
+        try {
+            HttpPoster httpPoster = new HttpPoster("https://metrics.cleaninsights.org/cleaninsights.php");
+            httpPoster.post(json);
+            return ListenableWorker.Result.success();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+        return ListenableWorker.Result.retry();
+    }
+
+    /**
+     * Convert a Java timestamp in milliseconds to a CleanInsights/Matomo timestamp
+     * normalized to the week and in UNIX epoch seconds format.
+     */
+    static long toCleanInsightsTimestamp(long timestamp) {
+        return toCleanInsightsTimestamp(timestamp, timestamp);
+    }
+
+    /**
+     * Convert a Java timestamp in milliseconds to a CleanInsights/Matomo timestamp
+     * normalized to the week and in UNIX epoch seconds format, plus the time
+     * difference between {@code relativeTo} and {@code timestamp}.
+     */
+    static long toCleanInsightsTimestamp(long relativeTo, long timestamp) {
+        long diff = timestamp - relativeTo;
+        long weekNumber = timestamp / DateUtils.WEEK_IN_MILLIS;
+        return ((weekNumber * DateUtils.WEEK_IN_MILLIS) + diff) / 1000L;
+    }
+
+    static boolean isTimestampInReportingWeek(long timestamp) {
+        return isTimestampInReportingWeek(getReportingWeekStart(), timestamp);
+    }
+
+    static boolean isTimestampInReportingWeek(long weekStart, long timestamp) {
+        long weekEnd = weekStart + DateUtils.WEEK_IN_MILLIS;
+        return weekStart < timestamp && timestamp < weekEnd;
+    }
+
+    static long getVersionCode(PackageInfo packageInfo) {
+        if (Build.VERSION.SDK_INT < 28) {
+            return packageInfo.versionCode;
+        } else {
+            return packageInfo.getLongVersionCode();
+        }
+    }
+
+    /**
+     * Gets the most recent week that is over based on the current time.
+     *
+     * @return start timestamp or 0 on parsing error
+     */
+    static long getReportingWeekStart() {
+        return getReportingWeekStart(System.currentTimeMillis());
+    }
+
+    /**
+     * Gets the most recent week that is over based on {@code timestamp}. This
+     * is the testable version of {@link #getReportingWeekStart()}
+     *
+     * @return start timestamp or 0 on parsing error
+     */
+    static long getReportingWeekStart(long timestamp) {
+        try {
+            Date start = new Date(timestamp - DateUtils.WEEK_IN_MILLIS);
+            return weekFormatter.parse(weekFormatter.format(start)).getTime();
+        } catch (ParseException e) {
+            // ignored
+        }
+        return 0;
+    }
+
+    /**
+     * Reads the {@link InstallHistoryService} CSV log, debounces the duplicate events,
+     * then converts it to {@link MatomoEvent} instances to be gathered.
+     */
+    static Collection<? extends MatomoEvent> parseInstallHistoryCsv(Context context, long weekStart) {
+        try {
+            File csv = InstallHistoryService.getInstallHistoryFile(context);
+            List<String> lines = FileUtils.readLines(csv, Charset.defaultCharset());
+            List<RawEvent> events = new ArrayList<>(lines.size());
+            for (String line : lines) {
+                RawEvent event = new RawEvent(line.split(","));
+                if (isTimestampInReportingWeek(weekStart, event.timestamp)) {
+                    events.add(event);
+                }
+            }
+            Collections.sort(events, new Comparator<RawEvent>() {
+                @Override
+                public int compare(RawEvent e0, RawEvent e1) {
+                    int applicationIdComparison = e0.applicationId.compareTo(e1.applicationId);
+                    if (applicationIdComparison != 0) {
+                        return applicationIdComparison;
+                    }
+                    int versionCodeComparison = Long.compare(e0.versionCode, e1.versionCode);
+                    if (versionCodeComparison != 0) {
+                        return versionCodeComparison;
+                    }
+                    int timestampComparison = Long.compare(e0.timestamp, e1.timestamp);
+                    if (timestampComparison != 0) {
+                        return timestampComparison;
+                    }
+                    return 0;
+                }
+            });
+            List<MatomoEvent> toReport = new ArrayList<>();
+            RawEvent previousEvent = new RawEvent(new String[]{"0", "", "0", ""});
+            for (RawEvent event : events) {
+                if (!previousEvent.equals(event)) {
+                    toReport.add(new MatomoEvent(event));
+                    previousEvent = event;
+                }
+            }
+            // TODO add time to INSTALL_COMPLETE evnts, eg INSTALL_COMPLETE - INSTALL_STARTED
+            return toReport;
+        } catch (IOException e) {
+            // ignored
+        }
+        return Collections.emptyList();
+    }
+
+    public static String generateReport(Context context) {
+        long weekStart = getReportingWeekStart();
+        CleanInsightsReport cleanInsightsReport = new CleanInsightsReport();
+        PackageManager pm = context.getPackageManager();
+        List<PackageInfo> packageInfoList = pm.getInstalledPackages(0);
+        Collections.sort(packageInfoList, new Comparator<PackageInfo>() {
+            @Override
+            public int compare(PackageInfo p1, PackageInfo p2) {
+                return p1.packageName.compareTo(p2.packageName);
+            }
+        });
+        App[] installedApps = InstalledAppProvider.Helper.all(context);
+        final ArrayList<MatomoEvent> events = new ArrayList<>();
+        events.add(getDeviceEvent(weekStart, "isPrivilegedInstallerEnabled",
+                Preferences.get().isPrivilegedInstallerEnabled()));
+        events.add(getDeviceEvent(weekStart, "Build.VERSION.SDK_INT", Build.VERSION.SDK_INT));
+        if (Build.VERSION.SDK_INT >= 21) {
+            events.add(getDeviceEvent(weekStart, "Build.SUPPORTED_ABIS", Arrays.toString(Build.SUPPORTED_ABIS)));
+        }
+
+        for (PackageInfo packageInfo : packageInfoList) {
+            boolean found = false;
+            for (App app : installedApps) {
+                if (packageInfo.packageName.equals(app.packageName)) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) continue;
+
+            if (isTimestampInReportingWeek(weekStart, packageInfo.firstInstallTime)) {
+                events.add(getFirstInstallEvent(packageInfo));
+            }
+            if (isTimestampInReportingWeek(weekStart, packageInfo.lastUpdateTime)) {
+                events.add(getInstallerEvent(pm, packageInfo));
+            }
+            if (Build.VERSION.SDK_INT >= 21) {
+                try {
+                    long atime = Os.lstat(packageInfo.applicationInfo.sourceDir).st_atime;
+                    if (isTimestampInReportingWeek(atime)) {
+                        events.add(getApkOpenedEvent(atime, packageInfo));
+                    }
+                } catch (Exception e) {
+                    // TODO replace with ErrnoException when using minSdkVersion 19 or higher
+                    e.printStackTrace();
+                }
+            }
+        }
+        events.addAll(parseInstallHistoryCsv(context, weekStart));
+        cleanInsightsReport.events = events.toArray(new MatomoEvent[0]);
+
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);
+        mapper.enable(SerializationFeature.INDENT_OUTPUT);
+        try {
+            return mapper.writeValueAsString(cleanInsightsReport);
+        } catch (JsonProcessingException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    /**
+     * Bare minimum report data in CleanInsights/Matomo format.
+     *
+     * @see MatomoEvent
+     * @see <a href="https://gitlab.com/cleaninsights/clean-insights-matomo-proxy#api">CleanInsights CIMP API</a>
+     * @see <a href="https://matomo.org/docs/event-tracking/">Matomo Event Tracking</a>
+     */
+    private static class CleanInsightsReport {
+        @JsonProperty
+        MatomoEvent[] events = new MatomoEvent[0];
+        @JsonProperty
+        final long idsite = 3; // NOPMD
+        @JsonProperty
+        final String lang = Locale.getDefault().getLanguage();
+        @JsonProperty
+        final String ua = Utils.getUserAgent();
+    }
+
+    private static MatomoEvent getApkOpenedEvent(long timestamp, PackageInfo packageInfo) {
+        return getApkEvent(timestamp, packageInfo, "opened");
+    }
+
+    private static MatomoEvent getFirstInstallEvent(PackageInfo packageInfo) {
+        return getApkEvent(packageInfo.firstInstallTime, packageInfo, "PackageInfo.firstInstall");
+    }
+
+    private static MatomoEvent getApkEvent(long timestamp, PackageInfo packageInfo, String action) {
+        MatomoEvent matomoEvent = new MatomoEvent(timestamp);
+        matomoEvent.category = "APK";
+        matomoEvent.action = action;
+        matomoEvent.name = packageInfo.packageName;
+        return matomoEvent;
+    }
+
+    /**
+     * Which app store installed APKs.
+     */
+    private static MatomoEvent getInstallerEvent(PackageManager pm, PackageInfo packageInfo) {
+        MatomoEvent matomoEvent = new MatomoEvent(packageInfo.lastUpdateTime);
+        matomoEvent.category = "getInstallerPackageName";
+        matomoEvent.action = pm.getInstallerPackageName(packageInfo.packageName);
+        matomoEvent.name = packageInfo.packageName;
+        return matomoEvent;
+    }
+
+    /**
+     * Events which describe the device that is doing the reporting.
+     */
+    private static MatomoEvent getDeviceEvent(long startTime, String action, Object name) {
+        MatomoEvent matomoEvent = new MatomoEvent(startTime);
+        matomoEvent.category = "device";
+        matomoEvent.action = action;
+        matomoEvent.name = String.valueOf(name);
+        return matomoEvent;
+    }
+
+    /**
+     * An event to send to CleanInsights/Matomo with a period of a full,
+     * normalized week.
+     *
+     * @see <a href="https://gitlab.com/cleaninsights/clean-insights-design/-/blob/d4f96ae3/schemas/cimp.schema.json">CleanInsights JSON Schema</a>
+     * @see <a href="https://matomo.org/docs/event-tracking/">Matomo Event Tracking</a>
+     */
+    @SuppressWarnings("checkstyle:MemberName")
+    @JsonInclude(JsonInclude.Include.NON_EMPTY)
+    static class MatomoEvent {
+        @JsonProperty
+        String category;
+        @JsonProperty
+        String action;
+        @JsonProperty
+        String name;
+        @JsonProperty
+        final long period_start;
+        @JsonProperty
+        final long period_end;
+        @JsonProperty
+        final long times = 1; // NOPMD
+        @JsonProperty
+        String value;
+
+        MatomoEvent(long timestamp) {
+            period_end = toCleanInsightsTimestamp(timestamp);
+            period_start = period_end - (DateUtils.WEEK_IN_MILLIS / 1000);
+        }
+
+        MatomoEvent(RawEvent rawEvent) {
+            this(rawEvent.timestamp);
+            category = "package";
+            action = rawEvent.action;
+            name = rawEvent.applicationId;
+        }
+    }
+
+    /**
+     * A raw event as read from {@link InstallHistoryService}'s CSV log file.
+     * This should never leave the device as is, it must have data stripped
+     * from it first.
+     */
+    static class RawEvent {
+        final long timestamp;
+        final String applicationId;
+        final long versionCode;
+        final String action;
+
+        RawEvent(String[] o) {
+            timestamp = Long.parseLong(o[0]);
+            applicationId = o[1];
+            versionCode = Long.parseLong(o[2]);
+            action = o[3];
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            RawEvent event = (RawEvent) o;
+            return versionCode == event.versionCode &&
+                    applicationId.equals(event.applicationId) &&
+                    action.equals(event.action);
+        }
+
+        @Override
+        public int hashCode() {
+            if (Build.VERSION.SDK_INT >= 19) {
+                return Objects.hash(applicationId, versionCode, action);
+            } else {
+                return new Random().nextInt(); // quick kludge
+            }
+        }
+
+        @Override
+        public String toString() {
+            return "RawEvent{" +
+                    "timestamp=" + timestamp +
+                    ", applicationId='" + applicationId + '\'' +
+                    ", versionCode=" + versionCode +
+                    ", action='" + action + '\'' +
+                    '}';
+        }
+    }
+}

app/src/test/java/org/fdroid/fdroid/work/FDroidMetricsWorkerTest.java

@@ -0,0 +1,96 @@
+package org.fdroid.fdroid.work;
+
+import android.app.Application;
+import android.content.ContextWrapper;
+import android.text.format.DateUtils;
+import androidx.test.core.app.ApplicationProvider;
+import org.apache.commons.io.FileUtils;
+import org.fdroid.fdroid.Preferences;
+import org.fdroid.fdroid.TestUtils;
+import org.fdroid.fdroid.installer.InstallHistoryService;
+import org.fdroid.fdroid.work.FDroidMetricsWorker.MatomoEvent;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+@Config(application = Application.class)
+@RunWith(RobolectricTestRunner.class)
+public class FDroidMetricsWorkerTest {
+    protected ContextWrapper context;
+
+    @Before
+    public final void setUp() {
+        context = ApplicationProvider.getApplicationContext();
+        Preferences.setupForTests(context);
+    }
+
+    @Test
+    public void testNormalizeTimestampToWeek() {
+        long startTime = 1610038865743L;
+        long endTime = 1610037631519L;
+
+        long normalizedStart = FDroidMetricsWorker.toCleanInsightsTimestamp(startTime);
+        long normalizedEnd = FDroidMetricsWorker.toCleanInsightsTimestamp(endTime);
+        assertEquals(normalizedStart, normalizedEnd);
+
+        long normalizedRelativeEnd = FDroidMetricsWorker.toCleanInsightsTimestamp(startTime, endTime);
+        assertEquals(1609976365L, normalizedRelativeEnd);
+    }
+
+    @Test
+    public void testGenerateReport() throws IOException {
+        String json = FDroidMetricsWorker.generateReport(context);
+        System.out.println(json);
+        File downloads = new File(System.getenv("HOME"), "Downloads");
+        if (downloads.exists()) {
+            File output = new File(downloads, getClass().getName() + ".testGenerateReport.json");
+            FileUtils.writeStringToFile(output, json);
+        }
+        // TODO validate against the schema
+    }
+
+    @Test
+    public void testParseInstallHistory() throws IOException {
+        FileUtils.copyFile(TestUtils.copyResourceToTempFile("install_history_all"),
+                InstallHistoryService.getInstallHistoryFile(context));
+        long weekStart = FDroidMetricsWorker.getReportingWeekStart(1611268892206L + DateUtils.WEEK_IN_MILLIS);
+        Collection<? extends MatomoEvent> events = FDroidMetricsWorker.parseInstallHistoryCsv(context,
+                weekStart);
+        assertEquals(3, events.size());
+        for (MatomoEvent event : events) {
+            assertEquals(event.name, "com.termux");
+        }
+
+        Collection<? extends MatomoEvent> oneWeekAgo = FDroidMetricsWorker.parseInstallHistoryCsv(context,
+                weekStart - DateUtils.WEEK_IN_MILLIS);
+        assertEquals(11, oneWeekAgo.size());
+
+        Collection<? extends MatomoEvent> twoWeeksAgo = FDroidMetricsWorker.parseInstallHistoryCsv(context,
+                weekStart - (2 * DateUtils.WEEK_IN_MILLIS));
+        assertEquals(0, twoWeeksAgo.size());
+
+        Collection<? extends MatomoEvent> threeWeeksAgo = FDroidMetricsWorker.parseInstallHistoryCsv(context,
+                weekStart - (3 * DateUtils.WEEK_IN_MILLIS));
+        assertEquals(9, threeWeeksAgo.size());
+        assertNotEquals(oneWeekAgo, threeWeeksAgo);
+    }
+
+    @Test
+    public void testGetReportingWeekStart() throws ParseException {
+        long now = System.currentTimeMillis();
+        long start = FDroidMetricsWorker.getReportingWeekStart(now);
+        assertTrue((now - DateUtils.WEEK_IN_MILLIS) > start);
+        assertTrue((now - DateUtils.WEEK_IN_MILLIS) < (start + DateUtils.WEEK_IN_MILLIS));
+    }
+}

app/src/test/resources/cimp.schema.json

@@ -0,0 +1,129 @@
+{
+  "$schema": "http://json-schema.org/draft/2019-09/schema#",
+  "$id": "https://cleaninsights.org/schemas/cimp.schema.json",
+  "title": "CleanInsights Matomo Proxy API",
+  "description": "The scheme defining the JSON API of the CleanInsights Matomo Proxy.",
+  "type": "object",
+  "properties": {
+    "idsite": {
+      "title": "Matomo Site ID",
+      "description": "The site ID used in the Matomo server which will collect and analyze the gathered data.",
+      "examples": [1, 2, 3, 345345],
+      "type": "integer",
+      "minimum": 1
+    },
+    "lang": {
+      "title": "HTTP Accept-Language Header",
+      "description": "A HTTP Accept-Language header. Matomo uses this value to detect the visitor's country.",
+      "examples": ["fr-CH, fr;q=0.9, en;q=0.8, de;q=0.7, *;q=0.5", "en", "de_AT"],
+      "type": "string"
+    },
+    "ua": {
+      "title": "HTTP User-Agent Header",
+      "description": "A  HTTP User-Agent. The user agent is used to detect the operating system and browser used.",
+      "examples": ["Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0"],
+      "type": "string"
+    },
+    "visits": {
+      "title": "Visit Measurements",
+      "description": "List of aggregated measurements to specific pages/scenes/activities.",
+      "type": "array",
+      "items": {
+        "title": "Visit Measurement",
+        "description": "A single aggregated measurement of repeated visits to a page/scene/activity.",
+        "type": "object",
+        "properties": {
+          "action_name": {
+            "title": "Visited Page/Scene/Activity Identifier",
+            "description": "Main identifier to track page/scene/activity visits in Matomo.",
+            "examples": ["For example, Help / Feedback will create the Action Feedback in the category Help."],
+            "type": "string",
+            "minLength": 1
+          },
+          "period_start": {
+            "title": "Start UNIX Epoch Timestamp",
+            "description": "Beginning of the aggregation period in seconds since 1970-01-01 00:00:00 UTC",
+            "examples": [1602499451],
+            "type": "integer"
+          },
+          "period_end": {
+            "title": "End UNIX Epoch Timestamp",
+            "description": "End of the aggregation period in seconds since 1970-01-01 00:00:00 UTC",
+            "examples": [1602499451],
+            "type": "integer"
+          },
+          "times": {
+            "title": "Number of Times Occurred",
+            "description": "The number of times the visit to this page/scene/activity happened during the specified period.",
+            "examples": [1, 2, 3, 26745],
+            "type": "integer",
+            "minimum": 1
+          }
+        },
+        "additionalProperties": false,
+        "required": ["action_name", "period_start", "period_end", "times"]
+      }
+    },
+    "events": {
+      "title": "Event Measurement",
+      "description": "List of aggregated measurements of a specific event. (e.g. like a press of a button, picture taken etc.)",
+      "type": "array",
+      "items": {
+        "title": "Event Measurement",
+        "description": "A single aggregated measurement of a specific event.",
+        "type": "object",
+        "properties": {
+          "category": {
+            "title": "Event Category Identifier",
+            "description": "A category identifier for the Matomo event tracking: https://matomo.org/docs/event-tracking/",
+            "examples": ["Videos", "Music", "Games"],
+            "type": "string",
+            "minLength": 1
+          },
+          "action": {
+            "title": "Event Action Identifier",
+            "description": "An action identifier for the Matomo event tracking: https://matomo.org/docs/event-tracking/",
+            "examples": ["Play", "Pause", "Duration", "Add Playlist", "Downloaded", "Clicked"],
+            "type": "string",
+            "minLength": 1
+          },
+          "name": {
+            "title": "Event Name",
+            "description": "An action name for the Matomo event tracking: https://matomo.org/docs/event-tracking/",
+            "examples": ["Office Space", "Jonathan Coulton - Code Monkey", "kraftwerk-autobahn.mp3"],
+            "type": "string"
+          },
+          "value": {
+            "title": "Event Value",
+            "description": "A value for the Matomo event tracking: https://matomo.org/docs/event-tracking/",
+            "examples": [0, 1, 1.5, 100, 56.44332],
+            "type": "number"
+          },
+          "period_start": {
+            "title": "Start UNIX Epoch Timestamp",
+            "description": "Beginning of the aggregation period in seconds since 1970-01-01 00:00:00 UTC",
+            "examples": [1602499451],
+            "type": "integer"
+          },
+          "period_end": {
+            "title": "End UNIX Epoch Timestamp",
+            "description": "End of the aggregation period in seconds since 1970-01-01 00:00:00 UTC",
+            "examples": [1602499451],
+            "type": "integer"
+          },
+          "times": {
+            "title": "Number of Times Occurred",
+            "description": "The number of times the visit to this page/scene/activity happened during the specified period.",
+            "examples": [1, 2, 3, 26745],
+            "type": "integer",
+            "minimum": 1
+          }
+        },
+        "additionalProperties": false,
+        "required": ["category", "action","period_start", "period_end", "times"]
+      }
+    }
+  },
+  "additionalProperties": false,
+  "required": ["idsite"]
+}

tools/validate-fdroid-metrics.json.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import glob
+import json
+import os
+import sys
+import jsonschema
+
+os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+with open('app/src/test/resources/cimp.schema.json') as fp:
+    schema = json.load(fp)
+
+errors = 0
+files = sys.argv[1:]
+if not files:
+    files = glob.glob(os.path.join(os.getenv('HOME'), 'Downloads', '*.json'))
+if not files:
+    print('Usage: %s file.json ...' % __file__)
+    exit(1)
+for f in files:
+    print('checking', f)
+    with open(f) as fp:
+        report = json.load(fp)
+    if jsonschema.validate(report, schema) is not None:
+        print('ERROR: %s did not validate' % f)
+        errors += 1
+exit(errors)