Merge branch 'fix-778--migration-tests' into 'master'

Test all database migrations from db-version/42 onwards

While a bit arbitrary choosing 42, it is from more than two years ago. We can use this as our new baseline, in that this test will always check upgrades from 42 onwards, with each database bump.

Once the test was up and running, it immediately identified a bug in the database migration for v50 and v57. These have been fixed in this branch too.

Fixes #778.

See merge request !403

app/src/main/java/org/fdroid/fdroid/data/AppProvider.java

@@ -5,7 +5,6 @@ import android.content.ContentValues;
 import android.content.Context;
 import android.content.UriMatcher;
 import android.database.Cursor;
-import android.database.sqlite.SQLiteDatabase;
 import android.net.Uri;
 import android.text.TextUtils;
 import android.util.Log;
@@ -191,22 +190,6 @@ public class AppProvider extends FDroidProvider {
         }
     }
 
-    /**
-     * Class that only exists to call private methods in the {@link AppProvider} without having
-     * to go via a Context/ContentResolver. The reason is that if the {@link DBHelper} class
-     * was to try and use its getContext().getContentResolver() in order to access the app
-     * provider, then the AppProvider will end up creating a new instance of a writeable
-     * SQLiteDatabase. This causes problems because the {@link DBHelper} still has its reference
-     * open and locks certain tables.
-     */
-    static final class UpgradeHelper {
-
-        public static void updateIconUrls(Context context, SQLiteDatabase db) {
-            AppProvider.updateIconUrls(context, db, AppMetadataTable.NAME, ApkTable.NAME);
-        }
-
-    }
-
     /**
      * A QuerySelection which is aware of the option/need to join onto the
      * installed apps table. Not that the base classes
@@ -948,7 +931,7 @@ public class AppProvider extends FDroidProvider {
         updateCompatibleFlags();
         updateSuggestedFromUpstream();
         updateSuggestedFromLatest();
-        updateIconUrls(getContext(), db(), getTableName(), getApkTableName());
+        updateIconUrls();
     }
 
     private void updatePreferredMetadata() {
@@ -1047,14 +1030,11 @@ public class AppProvider extends FDroidProvider {
         db().execSQL(updateSql);
     }
 
-    /**
+    private void updateIconUrls() {
-     * Made static so that the {@link org.fdroid.fdroid.data.AppProvider.UpgradeHelper} can access
+        final String appTable = getTableName();
-     * it without instantiating an {@link AppProvider}. This is also the reason it needs to accept
+        final String apkTable = getApkTableName();
-     * the context and database as arguments.
+        final String iconsDir = Utils.getIconsDir(getContext(), 1.0);
-     */
+        final String iconsDirLarge = Utils.getIconsDir(getContext(), 1.5);
-    private static void updateIconUrls(Context context, SQLiteDatabase db, String appTable, String apkTable) {
-        final String iconsDir = Utils.getIconsDir(context, 1.0);
-        final String iconsDirLarge = Utils.getIconsDir(context, 1.5);
         String repoVersion = Integer.toString(Repo.VERSION_DENSITY_SPECIFIC_ICONS);
         Utils.debugLog(TAG, "Updating icon paths for apps belonging to repos with version >= " + repoVersion);
         Utils.debugLog(TAG, "Using icons dir '" + iconsDir + "'");
@@ -1064,7 +1044,7 @@ public class AppProvider extends FDroidProvider {
             repoVersion, iconsDir, Utils.FALLBACK_ICONS_DIR,
             repoVersion, iconsDirLarge, Utils.FALLBACK_ICONS_DIR,
         };
-        db.execSQL(query, params);
+        db().execSQL(query, params);
     }
 
     /**

app/src/main/java/org/fdroid/fdroid/data/DBHelper.java

@@ -156,9 +156,8 @@ class DBHelper extends SQLiteOpenHelper {
             + InstalledAppTable.Cols.HASH_TYPE + " TEXT NOT NULL, "
             + InstalledAppTable.Cols.HASH + " TEXT NOT NULL"
             + " );";
-    private static final String DROP_TABLE_INSTALLED_APP = "DROP TABLE " + InstalledAppTable.NAME + ";";
 
-    private static final int DB_VERSION = 64;
+    protected static final int DB_VERSION = 64;
 
     private final Context context;
 
@@ -765,7 +764,32 @@ class DBHelper extends SQLiteOpenHelper {
             return;
         }
         Utils.debugLog(TAG, "Recalculating app icon URLs so that the newly added large icons will get updated.");
-        AppProvider.UpgradeHelper.updateIconUrls(context, db);
+
+        String query = "UPDATE fdroid_app "
+                + "SET iconUrl = ("
+                + "  SELECT (fdroid_repo.address || CASE WHEN fdroid_repo.version >= ? THEN ? ELSE ? END || fdroid_app.icon) "
+                + "  FROM fdroid_apk "
+                + "  JOIN fdroid_repo ON (fdroid_repo._id = fdroid_apk.repo) "
+                + "  WHERE fdroid_app.id = fdroid_apk.id AND fdroid_apk.vercode = fdroid_app.suggestedVercode "
+                + "), iconUrlLarge = ("
+                + "  SELECT (fdroid_repo.address || CASE WHEN fdroid_repo.version >= ? THEN ? ELSE ? END || fdroid_app.icon) "
+                + "  FROM fdroid_apk "
+                + "  JOIN fdroid_repo ON (fdroid_repo._id = fdroid_apk.repo) "
+                + "  WHERE fdroid_app.id = fdroid_apk.id AND fdroid_apk.vercode = fdroid_app.suggestedVercode"
+                + ")";
+
+        String iconsDir = Utils.getIconsDir(context, 1.0);
+        String iconsDirLarge = Utils.getIconsDir(context, 1.5);
+        String repoVersion = Integer.toString(Repo.VERSION_DENSITY_SPECIFIC_ICONS);
+        Utils.debugLog(TAG, "Using icons dir '" + iconsDir + "'");
+        Utils.debugLog(TAG, "Using large icons dir '" + iconsDirLarge + "'");
+        String[] args = {
+                repoVersion, iconsDir, Utils.FALLBACK_ICONS_DIR,
+                repoVersion, iconsDirLarge, Utils.FALLBACK_ICONS_DIR,
+        };
+
+        db.rawQuery(query, args);
+
         clearRepoEtags(db);
     }
 
@@ -916,7 +940,10 @@ class DBHelper extends SQLiteOpenHelper {
             return;
         }
         Utils.debugLog(TAG, "(re)creating 'installed app' database table.");
-        db.execSQL(DROP_TABLE_INSTALLED_APP);
+        if (tableExists(db, "fdroid_installedApp")) {
+            db.execSQL("DROP TABLE fdroid_installedApp;");
+        }
+
         db.execSQL(CREATE_TABLE_INSTALLED_APP);
     }
 

app/src/test/java/org/fdroid/fdroid/data/DatabaseMigration.java

@@ -0,0 +1,168 @@
+package org.fdroid.fdroid.data;
+
+import android.app.Application;
+import android.content.ContentValues;
+import android.content.Context;
+import android.content.ContextWrapper;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+
+import org.fdroid.fdroid.BuildConfig;
+import org.fdroid.fdroid.TestUtils;
+import org.fdroid.fdroid.Utils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricGradleTestRunner;
+import org.robolectric.RuntimeEnvironment;
+import org.robolectric.Shadows;
+import org.robolectric.annotation.Config;
+import org.robolectric.shadows.ShadowContentResolver;
+
+// TODO: Use sdk=24 when Robolectric supports this
+@Config(constants = BuildConfig.class, application = Application.class, sdk = 23)
+@RunWith(RobolectricGradleTestRunner.class)
+public class DatabaseMigration {
+
+    protected ShadowContentResolver contentResolver;
+    protected ContextWrapper context;
+
+    @Before
+    public final void setupBase() {
+        contentResolver = Shadows.shadowOf(RuntimeEnvironment.application.getContentResolver());
+        context = TestUtils.createContextWithContentResolver(contentResolver);
+        ShadowContentResolver.registerProvider(AppProvider.getAuthority(), new AppProvider());
+    }
+
+    @Test
+    public void migrationsFromDbVersion42Onward() {
+        SQLiteOpenHelper opener = new MigrationRunningOpenHelper(context);
+        opener.getReadableDatabase();
+    }
+
+    /**
+     * The database created by this in {@link MigrationRunningOpenHelper#onCreate(SQLiteDatabase)}
+     * should be identical to the one which was created by F-Droid circa git tag "db-version/42".
+     * After creating the database, this will then ask the base
+     * {@link DBHelper#onUpgrade(SQLiteDatabase, int, int)} method to run up until the current
+     * {@link DBHelper#DB_VERSION}.
+     */
+    class MigrationRunningOpenHelper extends DBHelper {
+
+        public static final String TABLE_REPO = "fdroid_repo";
+
+        MigrationRunningOpenHelper(Context context) {
+            super(context);
+        }
+
+        @Override
+        public void onCreate(SQLiteDatabase db) {
+            createAppTable(db);
+            createApkTable(db);
+            createRepoTable(db);
+            insertRepos(db);
+            onUpgrade(db, 42, DBHelper.DB_VERSION);
+        }
+
+        private void createAppTable(SQLiteDatabase db) {
+            db.execSQL("CREATE TABLE fdroid_app ("
+                    + "id text not null, "
+                    + "name text not null, "
+                    + "summary text not null, "
+                    + "icon text, "
+                    + "description text not null, "
+                    + "license text not null, "
+                    + "webURL text, "
+                    + "trackerURL text, "
+                    + "sourceURL text, "
+                    + "suggestedVercode text,"
+                    + "upstreamVersion text,"
+                    + "upstreamVercode integer,"
+                    + "antiFeatures string,"
+                    + "donateURL string,"
+                    + "bitcoinAddr string,"
+                    + "litecoinAddr string,"
+                    + "dogecoinAddr string,"
+                    + "flattrID string,"
+                    + "requirements string,"
+                    + "categories string,"
+                    + "added string,"
+                    + "lastUpdated string,"
+                    + "compatible int not null,"
+                    + "ignoreAllUpdates int not null,"
+                    + "ignoreThisUpdate int not null,"
+                    + "iconUrl text, "
+                    + "primary key(id));");
+
+            db.execSQL("create index app_id on fdroid_app (id);");
+        }
+
+        private void createApkTable(SQLiteDatabase db) {
+            db.execSQL("CREATE TABLE fdroid_apk ( "
+                    + "id text not null, "
+                    + "version text not null, "
+                    + "repo integer not null, "
+                    + "hash text not null, "
+                    + "vercode int not null,"
+                    + "apkName text not null, "
+                    + "size int not null, "
+                    + "sig string, "
+                    + "srcname string, "
+                    + "minSdkVersion integer, "
+                    + "maxSdkVersion integer, "
+                    + "permissions string, "
+                    + "features string, "
+                    + "nativecode string, "
+                    + "hashType string, "
+                    + "added string, "
+                    + "compatible int not null, "
+                    + "incompatibleReasons text, "
+                    + "primary key(id, vercode)"
+                    + ");");
+            db.execSQL("create index apk_vercode on fdroid_apk (vercode);");
+            db.execSQL("create index apk_id on fdroid_apk (id);");
+        }
+
+        private void createRepoTable(SQLiteDatabase db) {
+            db.execSQL("create table " + TABLE_REPO + " ("
+                    + "_id integer primary key, "
+                    + "address text not null, "
+                    + "name text, description text, inuse integer not null, "
+                    + "priority integer not null, pubkey text, fingerprint text, "
+                    + "maxage integer not null default 0, "
+                    + "version integer not null default 0, "
+                    + "lastetag text, lastUpdated string);");
+        }
+
+        private void insertRepos(SQLiteDatabase db) {
+            String pubKey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef";
+            String fingerprint = Utils.calcFingerprint(pubKey);
+
+            ContentValues fdroidValues = new ContentValues();
+            fdroidValues.put("address", "https://f-droid.org/repo");
+            fdroidValues.put("name", "F-Droid");
+            fdroidValues.put("description", "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time.");
+            fdroidValues.put("pubkey", pubKey);
+            fdroidValues.put("fingerprint", fingerprint);
+            fdroidValues.put("maxage", 0);
+            fdroidValues.put("inuse", 1);
+            fdroidValues.put("priority", 10);
+            fdroidValues.put("lastetag", (String) null);
+            db.insert(TABLE_REPO, null, fdroidValues);
+
+            ContentValues archiveValues = new ContentValues();
+            archiveValues.put("address", "https://f-droid.org/archive");
+            archiveValues.put("name", "F-Droid Archive");
+            archiveValues.put("description", "The archive repository of the F-Droid client. This contains older versions of applications from the main repository.");
+            archiveValues.put("pubkey", pubKey);
+            archiveValues.put("fingerprint", fingerprint);
+            archiveValues.put("maxage", 0);
+            archiveValues.put("inuse", 0);
+            archiveValues.put("priority", 20);
+            archiveValues.put("lastetag", (String) null);
+            db.insert(TABLE_REPO, null, archiveValues);
+        }
+
+    }
+
+}