Merge branch 'updater-speed-improvements' into 'master'

Updater speed improvements during "Saving Application Details"

I've been able to reproduce #324 OutOfMemory errors on an emulator with 12MiB of heap space. This branch did _not_ have an OOM error when updating with F-Droid and F-Droid Archive repos enabled. They successfully update without problem.

Fixes #45.

The "Saving Application Details" stage of repository updating is where each apk has its suggested version calculated, icon URLs calculated, etc. These all require [correlated subqueries](https://en.wikipedia.org/wiki/Correlated_subquery) resulting in a full scan of the apk table for each row in the app table. This takes in the order of 25 seconds on my Moto X 2nd Gen.

This branch improves this process by doing the queries in an [sqlite in-memory database](https://sqlite.org/inmemorydb.html), with the results transferred to the database on disk when done. The time required drops from 25 seconds to ~0.5 seconds on my device.

*Note:* I was hoping this would also improve the "Processing ..." Part of the udpater, given it was inserting into an in memory table instead of on disk. If it did have any effect, it was negligible though, so that part is still likely slower than it could be. Each 50 apps (and their associated apks) takes between 150ms and 500ms on my Moto X.

*Secondary Note:* When creating the in memory database, I create indexes for some columns as per before. This technically should slow down the inserts we do, however in practice they had almost no effect. As such, I've left the index creation there because it is required for the correlated subqueries to not suck. 

See merge request !269
This commit is contained in:
Daniel Martí 2016-05-02 21:31:35 +00:00
commit e3ebab9bc3
5 changed files with 45 additions and 39 deletions

View File

@ -1,5 +1,7 @@
### Upcoming release
* Significant performance improvements when updating repositories
* Show what repository each apk comes from
* Better support for Android 6.0

View File

@ -1057,20 +1057,7 @@ public class AppProvider extends FDroidProvider {
" JOIN " + repo + " ON (" + repo + "._id = " + apk + ".repo) " +
" WHERE " +
app + ".id = " + apk + ".id AND " +
apk + ".vercode = ( " +
// We only want the latest apk here. Ideally, we should
// instead join onto apk.suggestedVercode, but as per
// https://gitlab.com/fdroid/fdroidclient/issues/1 there
// may be some situations where suggestedVercode isn't
// set.
// TODO: If we can guarantee that suggestedVercode is set,
// then join onto that instead. This will save from doing
// a futher sub query for each app.
" SELECT MAX(inner_apk.vercode) " +
" FROM " + apk + " as inner_apk " +
" WHERE inner_apk.id = " + apk + ".id ) " +
" AND " + apk + ".repo = fdroid_repo._id ";
apk + ".vercode = " + app + ".suggestedVercode ";
return
" UPDATE " + app + " SET " +

View File

@ -91,9 +91,8 @@ public class RepoPersister {
// end of the process. This is due to the fact that we can't verify the cert
// the index was signed with until we've finished reading it - and we don't
// want to put stuff in the real database until we are sure it is from a
// trusted source.
// trusted source. It also helps performance as it is done via an in-memory database.
TempAppProvider.Helper.init(context);
TempApkProvider.Helper.init(context);
hasBeenInitialized = true;
}

View File

@ -68,12 +68,16 @@ public class TempApkProvider extends ApkProvider {
/**
* Deletes the old temporary table (if it exists). Then creates a new temporary apk provider
* table and populates it with all the data from the real apk provider table.
*
* This is package local because it must be invoked after
* {@link org.fdroid.fdroid.data.TempAppProvider.Helper#init(Context)}. Due to this
* dependence, that method invokes this one itself, rather than leaving it to the
* {@link RepoPersister}.
*/
public static void init(Context context) {
static void init(Context context) {
Uri uri = Uri.withAppendedPath(getContentUri(), PATH_INIT);
context.getContentResolver().insert(uri, new ContentValues());
}
}
@Override
@ -123,11 +127,11 @@ public class TempApkProvider extends ApkProvider {
private void initTable() {
final SQLiteDatabase db = db();
db.execSQL("DROP TABLE IF EXISTS " + getTableName());
db.execSQL("CREATE TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APK);
db.execSQL("CREATE INDEX IF NOT EXISTS apk_vercode on " + getTableName() + " (vercode);");
db.execSQL("CREATE INDEX IF NOT EXISTS apk_id on " + getTableName() + " (id);");
db.execSQL("CREATE INDEX IF NOT EXISTS apk_compatible ON " + getTableName() + " (compatible);");
final String memoryDbName = TempAppProvider.DB;
db.execSQL("CREATE TABLE " + memoryDbName + "." + getTableName() + " AS SELECT * FROM main." + DBHelper.TABLE_APK);
db.execSQL("CREATE INDEX IF NOT EXISTS " + memoryDbName + ".apk_vercode on " + getTableName() + " (vercode);");
db.execSQL("CREATE INDEX IF NOT EXISTS " + memoryDbName + ".apk_id on " + getTableName() + " (id);");
db.execSQL("CREATE INDEX IF NOT EXISTS " + memoryDbName + ".apk_compatible ON " + getTableName() + " (compatible);");
}
}

View File

@ -4,17 +4,18 @@ import android.content.ContentValues;
import android.content.Context;
import android.content.UriMatcher;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteException;
import android.net.Uri;
import android.util.Log;
import org.fdroid.fdroid.Utils;
/**
* This class does all of its operations in a temporary sqlite table.
*/
public class TempAppProvider extends AppProvider {
private static final String TAG = "TempAppProvider";
/**
* The name of the in memory database used for updating.
*/
static final String DB = "temp_update_db";
private static final String PROVIDER_NAME = "TempAppProvider";
@ -60,6 +61,7 @@ public class TempAppProvider extends AppProvider {
public static void init(Context context) {
Uri uri = Uri.withAppendedPath(getContentUri(), PATH_INIT);
context.getContentResolver().insert(uri, new ContentValues());
TempApkProvider.Helper.init(context);
}
/**
@ -111,13 +113,24 @@ public class TempAppProvider extends AppProvider {
return count;
}
private void ensureTempTableDetached(SQLiteDatabase db) {
try {
db.execSQL("DETACH DATABASE " + DB);
} catch (SQLiteException e) {
// We expect that most of the time the database will not exist unless an error occurred
// midway through the last update, The resulting exception is:
// android.database.sqlite.SQLiteException: no such database: temp_update_db (code 1)
}
}
private void initTable() {
final SQLiteDatabase db = db();
db.execSQL("DROP TABLE IF EXISTS " + getTableName());
db.execSQL("CREATE TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APP);
db.execSQL("CREATE INDEX IF NOT EXISTS app_id ON " + getTableName() + " (id);");
db.execSQL("CREATE INDEX IF NOT EXISTS app_upstreamVercode ON " + getTableName() + " (upstreamVercode);");
db.execSQL("CREATE INDEX IF NOT EXISTS app_compatible ON " + getTableName() + " (compatible);");
ensureTempTableDetached(db);
db.execSQL("ATTACH DATABASE ':memory:' AS " + DB);
db.execSQL("CREATE TABLE " + DB + "." + getTableName() + " AS SELECT * FROM main." + DBHelper.TABLE_APP);
db.execSQL("CREATE INDEX IF NOT EXISTS " + DB + ".app_id ON " + getTableName() + " (id);");
db.execSQL("CREATE INDEX IF NOT EXISTS " + DB + ".app_upstreamVercode ON " + getTableName() + " (upstreamVercode);");
db.execSQL("CREATE INDEX IF NOT EXISTS " + DB + ".app_compatible ON " + getTableName() + " (compatible);");
}
private void commitTable() {
@ -125,21 +138,22 @@ public class TempAppProvider extends AppProvider {
try {
db.beginTransaction();
Log.i(TAG, "Renaming " + TABLE_TEMP_APP + " to " + DBHelper.TABLE_APP);
db.execSQL("DROP TABLE " + DBHelper.TABLE_APP);
db.execSQL("ALTER TABLE " + TABLE_TEMP_APP + " RENAME TO " + DBHelper.TABLE_APP);
final String tempApp = DB + "." + TempAppProvider.TABLE_TEMP_APP;
final String tempApk = DB + "." + TempApkProvider.TABLE_TEMP_APK;
Log.i(TAG, "Renaming " + TempApkProvider.TABLE_TEMP_APK + " to " + DBHelper.TABLE_APK);
db.execSQL("DROP TABLE " + DBHelper.TABLE_APK);
db.execSQL("ALTER TABLE " + TempApkProvider.TABLE_TEMP_APK + " RENAME TO " + DBHelper.TABLE_APK);
db.execSQL("DELETE FROM " + DBHelper.TABLE_APP + " WHERE 1");
db.execSQL("INSERT INTO " + DBHelper.TABLE_APP + " SELECT * FROM " + tempApp);
db.execSQL("DELETE FROM " + DBHelper.TABLE_APK + " WHERE 1");
db.execSQL("INSERT INTO " + DBHelper.TABLE_APK + " SELECT * FROM " + tempApk);
Utils.debugLog(TAG, "Successfully renamed both tables, will commit transaction");
db.setTransactionSuccessful();
getContext().getContentResolver().notifyChange(AppProvider.getContentUri(), null);
getContext().getContentResolver().notifyChange(ApkProvider.getContentUri(), null);
} finally {
db.endTransaction();
db.execSQL("DETACH DATABASE " + DB); // Can't be done in a transaction.
}
}
}