From 050d9974b7645fabed9e0e721e68884b2459df9c Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 10 Oct 2016 23:40:45 +1100 Subject: [PATCH 1/3] Added a test which runs all DB migrations since DB version 42. It was a little arbitrary to choose this date. However it was when the database looked quite close to what it looks like now and it is from well over two years ago. Going into the future, this test may as well always start out at 42 forever more to ensure that database migrations from that point continue to work for all future database migrations. --- .../java/org/fdroid/fdroid/data/DBHelper.java | 2 +- .../fdroid/fdroid/data/DatabaseMigration.java | 168 ++++++++++++++++++ 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 app/src/test/java/org/fdroid/fdroid/data/DatabaseMigration.java diff --git a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java index e39d54406..5cfe6cd4a 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java +++ b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java @@ -158,7 +158,7 @@ class DBHelper extends SQLiteOpenHelper { + " );"; private static final String DROP_TABLE_INSTALLED_APP = "DROP TABLE " + InstalledAppTable.NAME + ";"; - private static final int DB_VERSION = 64; + protected static final int DB_VERSION = 64; private final Context context; diff --git a/app/src/test/java/org/fdroid/fdroid/data/DatabaseMigration.java b/app/src/test/java/org/fdroid/fdroid/data/DatabaseMigration.java new file mode 100644 index 000000000..c53cbeefe --- /dev/null +++ b/app/src/test/java/org/fdroid/fdroid/data/DatabaseMigration.java @@ -0,0 +1,168 @@ +package org.fdroid.fdroid.data; + +import android.app.Application; +import android.content.ContentValues; +import android.content.Context; +import android.content.ContextWrapper; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; + +import org.fdroid.fdroid.BuildConfig; +import org.fdroid.fdroid.TestUtils; +import org.fdroid.fdroid.Utils; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricGradleTestRunner; +import org.robolectric.RuntimeEnvironment; +import org.robolectric.Shadows; +import org.robolectric.annotation.Config; +import org.robolectric.shadows.ShadowContentResolver; + +// TODO: Use sdk=24 when Robolectric supports this +@Config(constants = BuildConfig.class, application = Application.class, sdk = 23) +@RunWith(RobolectricGradleTestRunner.class) +public class DatabaseMigration { + + protected ShadowContentResolver contentResolver; + protected ContextWrapper context; + + @Before + public final void setupBase() { + contentResolver = Shadows.shadowOf(RuntimeEnvironment.application.getContentResolver()); + context = TestUtils.createContextWithContentResolver(contentResolver); + ShadowContentResolver.registerProvider(AppProvider.getAuthority(), new AppProvider()); + } + + @Test + public void migrationsFromDbVersion42Onward() { + SQLiteOpenHelper opener = new MigrationRunningOpenHelper(context); + opener.getReadableDatabase(); + } + + /** + * The database created by this in {@link MigrationRunningOpenHelper#onCreate(SQLiteDatabase)} + * should be identical to the one which was created by F-Droid circa git tag "db-version/42". + * After creating the database, this will then ask the base + * {@link DBHelper#onUpgrade(SQLiteDatabase, int, int)} method to run up until the current + * {@link DBHelper#DB_VERSION}. + */ + class MigrationRunningOpenHelper extends DBHelper { + + public static final String TABLE_REPO = "fdroid_repo"; + + MigrationRunningOpenHelper(Context context) { + super(context); + } + + @Override + public void onCreate(SQLiteDatabase db) { + createAppTable(db); + createApkTable(db); + createRepoTable(db); + insertRepos(db); + onUpgrade(db, 42, DBHelper.DB_VERSION); + } + + private void createAppTable(SQLiteDatabase db) { + db.execSQL("CREATE TABLE fdroid_app (" + + "id text not null, " + + "name text not null, " + + "summary text not null, " + + "icon text, " + + "description text not null, " + + "license text not null, " + + "webURL text, " + + "trackerURL text, " + + "sourceURL text, " + + "suggestedVercode text," + + "upstreamVersion text," + + "upstreamVercode integer," + + "antiFeatures string," + + "donateURL string," + + "bitcoinAddr string," + + "litecoinAddr string," + + "dogecoinAddr string," + + "flattrID string," + + "requirements string," + + "categories string," + + "added string," + + "lastUpdated string," + + "compatible int not null," + + "ignoreAllUpdates int not null," + + "ignoreThisUpdate int not null," + + "iconUrl text, " + + "primary key(id));"); + + db.execSQL("create index app_id on fdroid_app (id);"); + } + + private void createApkTable(SQLiteDatabase db) { + db.execSQL("CREATE TABLE fdroid_apk ( " + + "id text not null, " + + "version text not null, " + + "repo integer not null, " + + "hash text not null, " + + "vercode int not null," + + "apkName text not null, " + + "size int not null, " + + "sig string, " + + "srcname string, " + + "minSdkVersion integer, " + + "maxSdkVersion integer, " + + "permissions string, " + + "features string, " + + "nativecode string, " + + "hashType string, " + + "added string, " + + "compatible int not null, " + + "incompatibleReasons text, " + + "primary key(id, vercode)" + + ");"); + db.execSQL("create index apk_vercode on fdroid_apk (vercode);"); + db.execSQL("create index apk_id on fdroid_apk (id);"); + } + + private void createRepoTable(SQLiteDatabase db) { + db.execSQL("create table " + TABLE_REPO + " (" + + "_id integer primary key, " + + "address text not null, " + + "name text, description text, inuse integer not null, " + + "priority integer not null, pubkey text, fingerprint text, " + + "maxage integer not null default 0, " + + "version integer not null default 0, " + + "lastetag text, lastUpdated string);"); + } + + private void insertRepos(SQLiteDatabase db) { + String pubKey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef"; + String fingerprint = Utils.calcFingerprint(pubKey); + + ContentValues fdroidValues = new ContentValues(); + fdroidValues.put("address", "https://f-droid.org/repo"); + fdroidValues.put("name", "F-Droid"); + fdroidValues.put("description", "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time."); + fdroidValues.put("pubkey", pubKey); + fdroidValues.put("fingerprint", fingerprint); + fdroidValues.put("maxage", 0); + fdroidValues.put("inuse", 1); + fdroidValues.put("priority", 10); + fdroidValues.put("lastetag", (String) null); + db.insert(TABLE_REPO, null, fdroidValues); + + ContentValues archiveValues = new ContentValues(); + archiveValues.put("address", "https://f-droid.org/archive"); + archiveValues.put("name", "F-Droid Archive"); + archiveValues.put("description", "The archive repository of the F-Droid client. This contains older versions of applications from the main repository."); + archiveValues.put("pubkey", pubKey); + archiveValues.put("fingerprint", fingerprint); + archiveValues.put("maxage", 0); + archiveValues.put("inuse", 0); + archiveValues.put("priority", 20); + archiveValues.put("lastetag", (String) null); + db.insert(TABLE_REPO, null, archiveValues); + } + + } + +} From 0d4d160407f623acfa39ededea8bf68a842ab23a Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Tue, 11 Oct 2016 00:04:34 +1100 Subject: [PATCH 2/3] Fix migration for DB version 50. The migration resulted in a query being run which was broken. The query was broken because it was dynamically generated by Java code. This Java code resulted in a valid migration when until very recently when the query was refactored to deal with a new DB structure. Now the query is no longer suitable to be run against a DB_VERSION 49 database. To resolve this, the migration now hard codes the query to a string which is executable when the DB_VERSION is 49. --- .../org/fdroid/fdroid/data/AppProvider.java | 34 ++++--------------- .../java/org/fdroid/fdroid/data/DBHelper.java | 27 ++++++++++++++- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/app/src/main/java/org/fdroid/fdroid/data/AppProvider.java b/app/src/main/java/org/fdroid/fdroid/data/AppProvider.java index ff9cafa66..8866d1af0 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/AppProvider.java +++ b/app/src/main/java/org/fdroid/fdroid/data/AppProvider.java @@ -5,7 +5,6 @@ import android.content.ContentValues; import android.content.Context; import android.content.UriMatcher; import android.database.Cursor; -import android.database.sqlite.SQLiteDatabase; import android.net.Uri; import android.text.TextUtils; import android.util.Log; @@ -191,22 +190,6 @@ public class AppProvider extends FDroidProvider { } } - /** - * Class that only exists to call private methods in the {@link AppProvider} without having - * to go via a Context/ContentResolver. The reason is that if the {@link DBHelper} class - * was to try and use its getContext().getContentResolver() in order to access the app - * provider, then the AppProvider will end up creating a new instance of a writeable - * SQLiteDatabase. This causes problems because the {@link DBHelper} still has its reference - * open and locks certain tables. - */ - static final class UpgradeHelper { - - public static void updateIconUrls(Context context, SQLiteDatabase db) { - AppProvider.updateIconUrls(context, db, AppMetadataTable.NAME, ApkTable.NAME); - } - - } - /** * A QuerySelection which is aware of the option/need to join onto the * installed apps table. Not that the base classes @@ -948,7 +931,7 @@ public class AppProvider extends FDroidProvider { updateCompatibleFlags(); updateSuggestedFromUpstream(); updateSuggestedFromLatest(); - updateIconUrls(getContext(), db(), getTableName(), getApkTableName()); + updateIconUrls(); } private void updatePreferredMetadata() { @@ -1047,14 +1030,11 @@ public class AppProvider extends FDroidProvider { db().execSQL(updateSql); } - /** - * Made static so that the {@link org.fdroid.fdroid.data.AppProvider.UpgradeHelper} can access - * it without instantiating an {@link AppProvider}. This is also the reason it needs to accept - * the context and database as arguments. - */ - private static void updateIconUrls(Context context, SQLiteDatabase db, String appTable, String apkTable) { - final String iconsDir = Utils.getIconsDir(context, 1.0); - final String iconsDirLarge = Utils.getIconsDir(context, 1.5); + private void updateIconUrls() { + final String appTable = getTableName(); + final String apkTable = getApkTableName(); + final String iconsDir = Utils.getIconsDir(getContext(), 1.0); + final String iconsDirLarge = Utils.getIconsDir(getContext(), 1.5); String repoVersion = Integer.toString(Repo.VERSION_DENSITY_SPECIFIC_ICONS); Utils.debugLog(TAG, "Updating icon paths for apps belonging to repos with version >= " + repoVersion); Utils.debugLog(TAG, "Using icons dir '" + iconsDir + "'"); @@ -1064,7 +1044,7 @@ public class AppProvider extends FDroidProvider { repoVersion, iconsDir, Utils.FALLBACK_ICONS_DIR, repoVersion, iconsDirLarge, Utils.FALLBACK_ICONS_DIR, }; - db.execSQL(query, params); + db().execSQL(query, params); } /** diff --git a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java index 5cfe6cd4a..831d12d54 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java +++ b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java @@ -765,7 +765,32 @@ class DBHelper extends SQLiteOpenHelper { return; } Utils.debugLog(TAG, "Recalculating app icon URLs so that the newly added large icons will get updated."); - AppProvider.UpgradeHelper.updateIconUrls(context, db); + + String query = "UPDATE fdroid_app " + + "SET iconUrl = (" + + " SELECT (fdroid_repo.address || CASE WHEN fdroid_repo.version >= ? THEN ? ELSE ? END || fdroid_app.icon) " + + " FROM fdroid_apk " + + " JOIN fdroid_repo ON (fdroid_repo._id = fdroid_apk.repo) " + + " WHERE fdroid_app.id = fdroid_apk.id AND fdroid_apk.vercode = fdroid_app.suggestedVercode " + + "), iconUrlLarge = (" + + " SELECT (fdroid_repo.address || CASE WHEN fdroid_repo.version >= ? THEN ? ELSE ? END || fdroid_app.icon) " + + " FROM fdroid_apk " + + " JOIN fdroid_repo ON (fdroid_repo._id = fdroid_apk.repo) " + + " WHERE fdroid_app.id = fdroid_apk.id AND fdroid_apk.vercode = fdroid_app.suggestedVercode" + + ")"; + + String iconsDir = Utils.getIconsDir(context, 1.0); + String iconsDirLarge = Utils.getIconsDir(context, 1.5); + String repoVersion = Integer.toString(Repo.VERSION_DENSITY_SPECIFIC_ICONS); + Utils.debugLog(TAG, "Using icons dir '" + iconsDir + "'"); + Utils.debugLog(TAG, "Using large icons dir '" + iconsDirLarge + "'"); + String[] args = { + repoVersion, iconsDir, Utils.FALLBACK_ICONS_DIR, + repoVersion, iconsDirLarge, Utils.FALLBACK_ICONS_DIR, + }; + + db.rawQuery(query, args); + clearRepoEtags(db); } From 72a88583d62b55ba0147e821541fa84697fc2c4f Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Tue, 11 Oct 2016 00:13:28 +1100 Subject: [PATCH 3/3] Only drop fdroid_installedApp if it exists --- app/src/main/java/org/fdroid/fdroid/data/DBHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java index 831d12d54..aaee174b4 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java +++ b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java @@ -156,7 +156,6 @@ class DBHelper extends SQLiteOpenHelper { + InstalledAppTable.Cols.HASH_TYPE + " TEXT NOT NULL, " + InstalledAppTable.Cols.HASH + " TEXT NOT NULL" + " );"; - private static final String DROP_TABLE_INSTALLED_APP = "DROP TABLE " + InstalledAppTable.NAME + ";"; protected static final int DB_VERSION = 64; @@ -941,7 +940,10 @@ class DBHelper extends SQLiteOpenHelper { return; } Utils.debugLog(TAG, "(re)creating 'installed app' database table."); - db.execSQL(DROP_TABLE_INSTALLED_APP); + if (tableExists(db, "fdroid_installedApp")) { + db.execSQL("DROP TABLE fdroid_installedApp;"); + } + db.execSQL(CREATE_TABLE_INSTALLED_APP); }