From b989ef3eccc2eec2412615bf7bfb9fe8269cfbd9 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 31 Aug 2015 08:37:25 +1000 Subject: [PATCH 01/17] WIP: Stream index details to database rather than waiting until end. Refactored repo update to stream apks from network -> jar file reader -> xml parser -> database. No longer build up large lists of app metadata to save. Saves memory, but is MUCH slower. Does sig verification properly, but does it at the END of the process and DOESN'T ROLL BACK on failure. Quick and dirty benchmarks show an increase in time from ~25 seconds to ~30 seconds on my Nexus 4 with Android 5.0. This doesn't seem so bad to me, for the tradeoff that people on low end devices can actually update now. Also, as @eighthave pointed out, if we are able to stream the download directly from the internet, then that time will drop to essentially the time it takes to download the index. --- .../src/org/fdroid/fdroid/RepoPersister.java | 329 ------------- .../src/org/fdroid/fdroid/RepoUpdater.java | 438 ++++++++++++++---- .../src/org/fdroid/fdroid/RepoXMLHandler.java | 219 +++++---- .../src/org/fdroid/fdroid/UpdateService.java | 33 +- .../org/fdroid/fdroid/data/ApkProvider.java | 90 +++- .../org/fdroid/fdroid/data/AppProvider.java | 8 +- .../fdroid/fdroid/data/FDroidProvider.java | 2 +- .../fdroid/fdroid/MultiRepoUpdaterTest.java | 25 +- .../org/fdroid/fdroid/RepoUpdaterTest.java | 14 +- .../org/fdroid/fdroid/RepoXMLHandlerTest.java | 150 +++--- 10 files changed, 653 insertions(+), 655 deletions(-) delete mode 100644 F-Droid/src/org/fdroid/fdroid/RepoPersister.java diff --git a/F-Droid/src/org/fdroid/fdroid/RepoPersister.java b/F-Droid/src/org/fdroid/fdroid/RepoPersister.java deleted file mode 100644 index 2c98929f9..000000000 --- a/F-Droid/src/org/fdroid/fdroid/RepoPersister.java +++ /dev/null @@ -1,329 +0,0 @@ -package org.fdroid.fdroid; - -import android.content.ContentProviderOperation; -import android.content.ContentValues; -import android.content.Context; -import android.content.OperationApplicationException; -import android.database.Cursor; -import android.net.Uri; -import android.os.RemoteException; -import android.support.annotation.NonNull; -import android.util.Log; - -import org.fdroid.fdroid.data.Apk; -import org.fdroid.fdroid.data.ApkProvider; -import org.fdroid.fdroid.data.App; -import org.fdroid.fdroid.data.AppProvider; -import org.fdroid.fdroid.data.Repo; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Saves app and apk information to the database after a {@link RepoUpdater} has processed the - * relevant index file. - */ -public class RepoPersister { - - private static final String TAG = "RepoPersister"; - - /** - * When an app already exists in the db, and we are updating it on the off chance that some - * values changed in the index, some fields should not be updated. Rather, they should be - * ignored, because they were explicitly set by the user, and hence can't be automatically - * overridden by the index. - * - * NOTE: In the future, these attributes will be moved to a join table, so that the app table - * is essentially completely transient, and can be nuked at any time. - */ - private static final String[] APP_FIELDS_TO_IGNORE = { - AppProvider.DataColumns.IGNORE_ALLUPDATES, - AppProvider.DataColumns.IGNORE_THISUPDATE, - }; - - @NonNull - private final Context context; - - private final Map appsToUpdate = new HashMap<>(); - private final List apksToUpdate = new ArrayList<>(); - private final List repos = new ArrayList<>(); - - public RepoPersister(@NonNull Context context) { - this.context = context; - } - - public RepoPersister queueUpdater(RepoUpdater updater) { - queueApps(updater.getApps()); - queueApks(updater.getApks()); - repos.add(updater.repo); - return this; - } - - private void queueApps(List apps) { - for (final App app : apps) { - appsToUpdate.put(app.id, app); - } - } - - private void queueApks(List apks) { - apksToUpdate.addAll(apks); - } - - public void save(List disabledRepos) { - - List listOfAppsToUpdate = new ArrayList<>(); - listOfAppsToUpdate.addAll(appsToUpdate.values()); - - calcApkCompatibilityFlags(apksToUpdate); - - // Need to do this BEFORE updating the apks, otherwise when it continually - // calls "get existing apks for repo X" then it will be getting the newly - // created apks, rather than those from the fresh, juicy index we just processed. - removeApksNoLongerInRepo(apksToUpdate, repos); - - int totalInsertsUpdates = listOfAppsToUpdate.size() + apksToUpdate.size(); - updateOrInsertApps(listOfAppsToUpdate, totalInsertsUpdates, 0); - updateOrInsertApks(apksToUpdate, totalInsertsUpdates, listOfAppsToUpdate.size()); - removeApksFromRepos(disabledRepos); - removeAppsWithoutApks(); - - // This will sort out the icon urls, compatibility flags. and suggested version - // for each app. It used to happen here in Java code, but was moved to SQL when - // it became apparant we don't always have enough info (depending on which repos - // were updated). - AppProvider.Helper.calcDetailsFromIndex(context); - - } - - /** - * This cannot be offloaded to the database (as we did with the query which - * updates apps, depending on whether their apks are compatible or not). - * The reason is that we need to interact with the CompatibilityChecker - * in order to see if, and why an apk is not compatible. - */ - private void calcApkCompatibilityFlags(List apks) { - final CompatibilityChecker checker = new CompatibilityChecker(context); - for (final Apk apk : apks) { - final List reasons = checker.getIncompatibleReasons(apk); - if (reasons.size() > 0) { - apk.compatible = false; - apk.incompatibleReasons = Utils.CommaSeparatedList.make(reasons); - } else { - apk.compatible = true; - apk.incompatibleReasons = null; - } - } - } - - /** - * If a repo was updated (i.e. it is in use, and the index has changed - * since last time we did an update), then we want to remove any apks that - * belong to the repo which are not in the current list of apks that were - * retrieved. - */ - private void removeApksNoLongerInRepo(List apksToUpdate, List updatedRepos) { - - long startTime = System.currentTimeMillis(); - List toRemove = new ArrayList<>(); - - final String[] fields = { - ApkProvider.DataColumns.APK_ID, - ApkProvider.DataColumns.VERSION_CODE, - ApkProvider.DataColumns.VERSION, - }; - - for (final Repo repo : updatedRepos) { - final List existingApks = ApkProvider.Helper.findByRepo(context, repo, fields); - for (final Apk existingApk : existingApks) { - if (!isApkToBeUpdated(existingApk, apksToUpdate)) { - toRemove.add(existingApk); - } - } - } - - long duration = System.currentTimeMillis() - startTime; - Utils.debugLog(TAG, "Found " + toRemove.size() + " apks no longer in the updated repos (took " + duration + "ms)"); - - if (toRemove.size() > 0) { - ApkProvider.Helper.deleteApks(context, toRemove); - } - } - - private void updateOrInsertApps(List appsToUpdate, int totalUpdateCount, int currentCount) { - - List operations = new ArrayList<>(); - List knownAppIds = getKnownAppIds(appsToUpdate); - for (final App app : appsToUpdate) { - if (knownAppIds.contains(app.id)) { - operations.add(updateExistingApp(app)); - } else { - operations.add(insertNewApp(app)); - } - } - - Utils.debugLog(TAG, "Updating/inserting " + operations.size() + " apps."); - try { - executeBatchWithStatus(AppProvider.getAuthority(), operations, currentCount, totalUpdateCount); - } catch (RemoteException | OperationApplicationException e) { - Log.e(TAG, "Could not update or insert apps", e); - } - } - - private void executeBatchWithStatus(String providerAuthority, - List operations, - int currentCount, - int totalUpdateCount) - throws RemoteException, OperationApplicationException { - int i = 0; - while (i < operations.size()) { - int count = Math.min(operations.size() - i, 100); - int progress = (int) ((double) (currentCount + i) / totalUpdateCount * 100); - ArrayList o = new ArrayList<>(operations.subList(i, i + count)); - UpdateService.sendStatus(context, UpdateService.STATUS_INFO, context.getString( - R.string.status_inserting, progress), progress); - context.getContentResolver().applyBatch(providerAuthority, o); - i += 100; - } - } - - /** - * Return list of apps from the "apks" argument which are already in the database. - */ - private List getKnownApks(List apks) { - final String[] fields = { - ApkProvider.DataColumns.APK_ID, - ApkProvider.DataColumns.VERSION, - ApkProvider.DataColumns.VERSION_CODE, - }; - return ApkProvider.Helper.knownApks(context, apks, fields); - } - - private void updateOrInsertApks(List apksToUpdate, int totalApksAppsCount, int currentCount) { - - List operations = new ArrayList<>(); - - List knownApks = getKnownApks(apksToUpdate); - for (final Apk apk : apksToUpdate) { - boolean known = false; - for (final Apk knownApk : knownApks) { - if (knownApk.id.equals(apk.id) && knownApk.vercode == apk.vercode) { - known = true; - break; - } - } - - if (known) { - operations.add(updateExistingApk(apk)); - } else { - operations.add(insertNewApk(apk)); - knownApks.add(apk); // In case another repo has the same version/id combo for this apk. - } - } - - Utils.debugLog(TAG, "Updating/inserting " + operations.size() + " apks."); - try { - executeBatchWithStatus(ApkProvider.getAuthority(), operations, currentCount, totalApksAppsCount); - } catch (RemoteException | OperationApplicationException e) { - Log.e(TAG, "Could not update/insert apps", e); - } - } - - private ContentProviderOperation updateExistingApk(final Apk apk) { - Uri uri = ApkProvider.getContentUri(apk); - ContentValues values = apk.toContentValues(); - return ContentProviderOperation.newUpdate(uri).withValues(values).build(); - } - - private ContentProviderOperation insertNewApk(final Apk apk) { - ContentValues values = apk.toContentValues(); - Uri uri = ApkProvider.getContentUri(); - return ContentProviderOperation.newInsert(uri).withValues(values).build(); - } - - private ContentProviderOperation updateExistingApp(App app) { - Uri uri = AppProvider.getContentUri(app); - ContentValues values = app.toContentValues(); - for (final String toIgnore : APP_FIELDS_TO_IGNORE) { - if (values.containsKey(toIgnore)) { - values.remove(toIgnore); - } - } - return ContentProviderOperation.newUpdate(uri).withValues(values).build(); - } - - private ContentProviderOperation insertNewApp(App app) { - ContentValues values = app.toContentValues(); - Uri uri = AppProvider.getContentUri(); - return ContentProviderOperation.newInsert(uri).withValues(values).build(); - } - - private static boolean isApkToBeUpdated(Apk existingApk, List apksToUpdate) { - for (final Apk apkToUpdate : apksToUpdate) { - if (apkToUpdate.vercode == existingApk.vercode && apkToUpdate.id.equals(existingApk.id)) { - return true; - } - } - return false; - } - - private void removeApksFromRepos(List repos) { - for (final Repo repo : repos) { - Uri uri = ApkProvider.getRepoUri(repo.getId()); - int numDeleted = context.getContentResolver().delete(uri, null, null); - Utils.debugLog(TAG, "Removing " + numDeleted + " apks from repo " + repo.address); - } - } - - private void removeAppsWithoutApks() { - int numDeleted = context.getContentResolver().delete(AppProvider.getNoApksUri(), null, null); - Utils.debugLog(TAG, "Removing " + numDeleted + " apks that don't have any apks"); - } - - private List getKnownAppIds(List apps) { - List knownAppIds = new ArrayList<>(); - if (apps.isEmpty()) { - return knownAppIds; - } - if (apps.size() > AppProvider.MAX_APPS_TO_QUERY) { - int middle = apps.size() / 2; - List apps1 = apps.subList(0, middle); - List apps2 = apps.subList(middle, apps.size()); - knownAppIds.addAll(getKnownAppIds(apps1)); - knownAppIds.addAll(getKnownAppIds(apps2)); - } else { - knownAppIds.addAll(getKnownAppIdsFromProvider(apps)); - } - return knownAppIds; - } - - /** - * Looks in the database to see which apps we already know about. Only - * returns ids of apps that are in the database if they are in the "apps" - * array. - */ - private List getKnownAppIdsFromProvider(List apps) { - - final Uri uri = AppProvider.getContentUri(apps); - final String[] fields = {AppProvider.DataColumns.APP_ID}; - Cursor cursor = context.getContentResolver().query(uri, fields, null, null, null); - - int knownIdCount = cursor != null ? cursor.getCount() : 0; - List knownIds = new ArrayList<>(knownIdCount); - if (cursor != null) { - if (knownIdCount > 0) { - cursor.moveToFirst(); - while (!cursor.isAfterLast()) { - knownIds.add(cursor.getString(0)); - cursor.moveToNext(); - } - } - cursor.close(); - } - - return knownIds; - } - -} - diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 45d90a9d0..97c935e7a 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -1,13 +1,20 @@ package org.fdroid.fdroid; +import android.content.ContentProviderOperation; import android.content.ContentValues; import android.content.Context; +import android.content.OperationApplicationException; +import android.net.Uri; +import android.os.RemoteException; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.text.TextUtils; +import android.util.Log; import org.fdroid.fdroid.data.Apk; +import org.fdroid.fdroid.data.ApkProvider; import org.fdroid.fdroid.data.App; +import org.fdroid.fdroid.data.AppProvider; import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.net.Downloader; @@ -26,7 +33,9 @@ import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -35,8 +44,12 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; /** - * Handles getting the index metadata for an app repo, then verifying the - * signature on the index metdata, implementing as a JAR signature. + * + * Responsible for updating an individual repository. This will: + * * Download the index.jar + * * Verify that it is signed correctly and by the correct certificate + * * Parse the index.xml from the .jar file + * * Save the resulting repo, apps, and apks to the database. * * WARNING: this class is the central piece of the entire security model of * FDroid! Avoid modifying it when possible, if you absolutely must, be very, @@ -49,13 +62,26 @@ public class RepoUpdater { public static final String PROGRESS_TYPE_PROCESS_XML = "processingXml"; public static final String PROGRESS_DATA_REPO_ADDRESS = "repoAddress"; + /** + * When an app already exists in the db, and we are updating it on the off chance that some + * values changed in the index, some fields should not be updated. Rather, they should be + * ignored, because they were explicitly set by the user, and hence can't be automatically + * overridden by the index. + * + * NOTE: In the future, these attributes will be moved to a join table, so that the app table + * is essentially completely transient, and can be nuked at any time. + */ + private static final String[] APP_FIELDS_TO_IGNORE = { + AppProvider.DataColumns.IGNORE_ALLUPDATES, + AppProvider.DataColumns.IGNORE_THISUPDATE + }; + @NonNull protected final Context context; @NonNull protected final Repo repo; - private List apps = new ArrayList<>(); - private List apks = new ArrayList<>(); - private RepoUpdateRememberer rememberer; - protected boolean hasChanged; + protected boolean hasChanged = false; @Nullable protected ProgressListener progressListener; + private String cacheTag; + private X509Certificate signingCertFromJar; /** * Updates an app repo as read out of the database into a {@link Repo} instance. @@ -74,15 +100,7 @@ public class RepoUpdater { return hasChanged; } - public List getApps() { - return apps; - } - - public List getApks() { - return apks; - } - - private URL getIndexAddress() throws MalformedURLException { + protected URL getIndexAddress() throws MalformedURLException { String urlString = repo.address + "/index.jar"; String versionName = Utils.getVersionName(context); if (versionName != null) { @@ -110,7 +128,9 @@ public class RepoUpdater { } catch (IOException e) { if (downloader != null && downloader.getFile() != null) { - downloader.getFile().delete(); + if (!downloader.getFile().delete()) { + Log.i(TAG, "Couldn't delete file: " + downloader.getFile().getAbsolutePath()); + } } throw new UpdateException(repo, "Error getting index file", e); @@ -133,11 +153,257 @@ public class RepoUpdater { if (hasChanged) { // Don't worry about checking the status code for 200. If it was a // successful download, then we will have a file ready to use: - processDownloadedFile(downloader.getFile(), downloader.getCacheTag()); + cacheTag = downloader.getCacheTag(); + processDownloadedFile(downloader.getFile()); } } - protected void processDownloadedFile(File downloadedFile, String cacheTag) throws UpdateException { + private ContentValues repoDetailsToSave = null; + private String signingCertFromIndexXml = null; + + private RepoXMLHandler.IndexReceiver createIndexReceiver() { + return new RepoXMLHandler.IndexReceiver() { + @Override + public void receiveRepo(String name, String description, String signingCert, int maxAge, int version) { + signingCertFromIndexXml = signingCert; + repoDetailsToSave = prepareRepoDetailsForSaving(name, description, maxAge, version); + } + + @Override + public void receiveApp(App app, List packages) { + try { + saveToDb(app, packages); + } catch (UpdateException e) { + throw new RuntimeException("Error while saving repo details to database.", e); + } + } + }; + } + + /** + * My crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get: + * * 25 = { 39, 35 } seconds + * * 50 = { 36, 30 } seconds + * * 100 = { 33, 27 } seconds + * * 200 = { 30, 33 } seconds + */ + private static final int MAX_APP_BUFFER = 50; + + private List appsToSave = new ArrayList<>(); + private Map> apksToSave = new HashMap<>(); + + private void saveToDb(App app, List packages) throws UpdateException { + appsToSave.add(app); + apksToSave.put(app.id, packages); + + if (appsToSave.size() >= MAX_APP_BUFFER) { + flushBufferToDb(); + } + } + + private void flushBufferToDb() throws UpdateException { + Log.d(TAG, "Flushing details of " + MAX_APP_BUFFER + " and their packages to the database."); + flushAppsToDbInBatch(); + flushApksToDbInBatch(); + apksToSave.clear(); + appsToSave.clear(); + } + + private void flushApksToDbInBatch() throws UpdateException { + List apksToSaveList = new ArrayList<>(); + for (Map.Entry> entries : apksToSave.entrySet()) { + apksToSaveList.addAll(entries.getValue()); + } + + calcApkCompatibilityFlags(apksToSaveList); + + ArrayList apkOperations = new ArrayList<>(); + ContentProviderOperation clearOrphans = deleteOrphanedApks(appsToSave, apksToSave); + if (clearOrphans != null) { + apkOperations.add(clearOrphans); + } + apkOperations.addAll(insertOrUpdateApks(apksToSaveList)); + + try { + context.getContentResolver().applyBatch(ApkProvider.getAuthority(), apkOperations); + } catch (RemoteException | OperationApplicationException e) { + throw new UpdateException(repo, "An internal error occured while updating the database", e); + } + } + + private void flushAppsToDbInBatch() throws UpdateException { + ArrayList appOperations = insertOrUpdateApps(appsToSave); + + try { + context.getContentResolver().applyBatch(AppProvider.getAuthority(), appOperations); + } catch (RemoteException|OperationApplicationException e) { + Log.e(TAG, "Error updating apps", e); + throw new UpdateException(repo, "Error updating apps: " + e.getMessage(), e); + } + } + + /** + * Depending on whether the {@link App}s have been added to the database previously, this + * will queue up an update or an insert {@link ContentProviderOperation} for each app. + */ + private ArrayList insertOrUpdateApps(List apps) { + ArrayList operations = new ArrayList<>(apps.size()); + for (App app : apps) { + if (isAppInDatabase(app)) { + operations.add(updateExistingApp(app)); + } else { + operations.add(insertNewApp(app)); + } + } + return operations; + } + + /** + * Depending on whether the .apks have been added to the database previously, this + * will queue up an update or an insert {@link ContentProviderOperation} for each package. + */ + private ArrayList insertOrUpdateApks(List packages) { + List existingApks = ApkProvider.Helper.knownApks(context, packages, new String[]{ApkProvider.DataColumns.VERSION_CODE}); + ArrayList operations = new ArrayList<>(packages.size()); + for (Apk apk : packages) { + boolean exists = false; + for (Apk existing : existingApks) { + if (existing.vercode == apk.vercode) { + exists = true; + break; + } + } + + if (exists) { + operations.add(updateExistingApk(apk)); + } else { + operations.add(insertNewApk(apk)); + } + } + + return operations; + } + + /** + * Creates an update {@link ContentProviderOperation} for the {@link App} in question. + * Does not do any checks to see if the app already exists or not. + */ + private ContentProviderOperation updateExistingApp(App app) { + Uri uri = AppProvider.getContentUri(app); + ContentValues values = app.toContentValues(); + for (final String toIgnore : APP_FIELDS_TO_IGNORE) { + if (values.containsKey(toIgnore)) { + values.remove(toIgnore); + } + } + return ContentProviderOperation.newUpdate(uri).withValues(values).build(); + } + + /** + * Creates an insert {@link ContentProviderOperation} for the {@link App} in question. + * Does not do any checks to see if the app already exists or not. + */ + private ContentProviderOperation insertNewApp(App app) { + ContentValues values = app.toContentValues(); + Uri uri = AppProvider.getContentUri(); + return ContentProviderOperation.newInsert(uri).withValues(values).build(); + } + + /** + * Looks in the database to see which apps we already know about. Only + * returns ids of apps that are in the database if they are in the "apps" + * array. + */ + private boolean isAppInDatabase(App app) { + String[] fields = { AppProvider.DataColumns.APP_ID }; + App found = AppProvider.Helper.findById(context.getContentResolver(), app.id, fields); + return found != null; + } + + /** + * Creates an update {@link ContentProviderOperation} for the {@link Apk} in question. + * Does not do any checks to see if the apk already exists or not. + */ + private ContentProviderOperation updateExistingApk(final Apk apk) { + Uri uri = ApkProvider.getContentUri(apk); + ContentValues values = apk.toContentValues(); + return ContentProviderOperation.newUpdate(uri).withValues(values).build(); + } + + /** + * Creates an insert {@link ContentProviderOperation} for the {@link Apk} in question. + * Does not do any checks to see if the apk already exists or not. + */ + private ContentProviderOperation insertNewApk(final Apk apk) { + ContentValues values = apk.toContentValues(); + Uri uri = ApkProvider.getContentUri(); + return ContentProviderOperation.newInsert(uri).withValues(values).build(); + } + + /** + * Finds all apks from the repo we are currently updating, that belong to the specified app, + * and delete them as they are no longer provided by that repo. + */ + @Nullable + private ContentProviderOperation deleteOrphanedApks(List apps, Map> packages) { + + String[] projection = new String[]{ApkProvider.DataColumns.APK_ID, ApkProvider.DataColumns.VERSION_CODE}; + List existing = ApkProvider.Helper.find(context, repo, apps, projection); + + List toDelete = new ArrayList<>(); + + for (Apk existingApk : existing) { + + boolean shouldStay = false; + + for (Map.Entry> entry : packages.entrySet()) { + for (Apk newApk : entry.getValue()) { + if (newApk.vercode == existingApk.vercode) { + shouldStay = true; + break; + } + } + + if (shouldStay) { + break; + } + } + + if (!shouldStay) { + toDelete.add(existingApk); + } + } + + // TODO: Deal with more than MAX_QUERY_PARAMS... + if (toDelete.size() > 0) { + Uri uri = ApkProvider.getContentUriForApks(repo, toDelete); + return ContentProviderOperation.newDelete(uri).build(); + } else { + return null; + } + } + + /** + * This cannot be offloaded to the database (as we did with the query which + * updates apps, depending on whether their apks are compatible or not). + * The reason is that we need to interact with the CompatibilityChecker + * in order to see if, and why an apk is not compatible. + */ + public void calcApkCompatibilityFlags(List apks) { + final CompatibilityChecker checker = new CompatibilityChecker(context); + for (final Apk apk : apks) { + final List reasons = checker.getIncompatibleReasons(apk); + if (reasons.size() > 0) { + apk.compatible = false; + apk.incompatibleReasons = Utils.CommaSeparatedList.make(reasons); + } else { + apk.compatible = true; + apk.incompatibleReasons = null; + } + } + } + + public void processDownloadedFile(File downloadedFile) throws UpdateException { InputStream indexInputStream = null; try { if (downloadedFile == null || !downloadedFile.exists()) @@ -153,94 +419,77 @@ public class RepoUpdater { indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry), progressListener, repo, (int) indexEntry.getSize()); + /* JarEntry can only read certificates after the file represented by that JarEntry + * has been read completely, so verification cannot run until now... */ + // Process the index... final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); final XMLReader reader = parser.getXMLReader(); - final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo); + final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo, createIndexReceiver()); reader.setContentHandler(repoXMLHandler); reader.parse(new InputSource(indexInputStream)); + signingCertFromJar = getSigningCertFromJar(indexEntry); - /* JarEntry can only read certificates after the file represented by that JarEntry - * has been read completely, so verification cannot run until now... */ - X509Certificate certFromJar = getSigningCertFromJar(indexEntry); + assertSigningCertFromXmlCorrect(); + RepoProvider.Helper.update(context, repo, repoDetailsToSave); - String certFromIndexXml = repoXMLHandler.getSigningCertFromIndexXml(); - - // no signing cert read from database, this is the first use - if (repo.pubkey == null) { - verifyAndStoreTOFUCerts(certFromIndexXml, certFromJar); - } - verifyCerts(certFromIndexXml, certFromJar); - - apps = repoXMLHandler.getApps(); - apks = repoXMLHandler.getApks(); - - rememberer = new RepoUpdateRememberer(); - rememberer.context = context; - rememberer.repo = repo; - rememberer.values = prepareRepoDetailsForSaving(repoXMLHandler, cacheTag); } catch (SAXException | ParserConfigurationException | IOException e) { throw new UpdateException(repo, "Error parsing index", e); } finally { FDroidApp.enableSpongyCastleOnLollipop(); Utils.closeQuietly(indexInputStream); if (downloadedFile != null) { - downloadedFile.delete(); + if (!downloadedFile.delete()) { + Log.i(TAG, "Couldn't delete file: " + downloadedFile.getAbsolutePath()); + } } } } + private void assertSigningCertFromXmlCorrect() throws SigningException { + + // no signing cert read from database, this is the first use + if (repo.pubkey == null) { + verifyAndStoreTOFUCerts(signingCertFromIndexXml, signingCertFromJar); + } + verifyCerts(signingCertFromIndexXml, signingCertFromJar); + + } + /** * Update tracking data for the repo represented by this instance (index version, etag, * description, human-readable name, etc. */ - private ContentValues prepareRepoDetailsForSaving(RepoXMLHandler handler, String etag) { + private ContentValues prepareRepoDetailsForSaving(String name, String description, int maxAge, int version) { ContentValues values = new ContentValues(); values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatTime(new Date(), "")); - if (repo.lastetag == null || !repo.lastetag.equals(etag)) { - values.put(RepoProvider.DataColumns.LAST_ETAG, etag); + if (repo.lastetag == null || !repo.lastetag.equals(cacheTag)) { + values.put(RepoProvider.DataColumns.LAST_ETAG, cacheTag); } - if (handler.getVersion() != -1 && handler.getVersion() != repo.version) { - Utils.debugLog(TAG, "Repo specified a new version: from " - + repo.version + " to " + handler.getVersion()); - values.put(RepoProvider.DataColumns.VERSION, handler.getVersion()); + if (version != -1 && version != repo.version) { + Utils.debugLog(TAG, "Repo specified a new version: from " + repo.version + " to " + version); + values.put(RepoProvider.DataColumns.VERSION, version); } - if (handler.getMaxAge() != -1 && handler.getMaxAge() != repo.maxage) { + if (maxAge != -1 && maxAge != repo.maxage) { Utils.debugLog(TAG, "Repo specified a new maximum age - updated"); - values.put(RepoProvider.DataColumns.MAX_AGE, handler.getMaxAge()); + values.put(RepoProvider.DataColumns.MAX_AGE, maxAge); } - if (handler.getDescription() != null && !handler.getDescription().equals(repo.description)) { - values.put(RepoProvider.DataColumns.DESCRIPTION, handler.getDescription()); + if (description != null && !description.equals(repo.description)) { + values.put(RepoProvider.DataColumns.DESCRIPTION, description); } - if (handler.getName() != null && !handler.getName().equals(repo.name)) { - values.put(RepoProvider.DataColumns.NAME, handler.getName()); + if (name != null && !name.equals(repo.name)) { + values.put(RepoProvider.DataColumns.NAME, name); } return values; } - public RepoUpdateRememberer getRememberer() { - return rememberer; - } - - public static class RepoUpdateRememberer { - - private Context context; - private Repo repo; - private ContentValues values; - - public void rememberUpdate() { - RepoProvider.Helper.update(context, repo, values); - } - - } - public static class UpdateException extends Exception { private static final long serialVersionUID = -4492452418826132803L; @@ -257,23 +506,29 @@ public class RepoUpdater { } } + public static class SigningException extends UpdateException { + public SigningException(Repo repo, String message) { + super(repo, "Repository was not signed correctly: " + message); + } + } + /** * FDroid's index.jar is signed using a particular format and does not allow lots of * signing setups that would be valid for a regular jar. This validates those * restrictions. */ - private X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws UpdateException { + private X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws SigningException { final CodeSigner[] codeSigners = jarEntry.getCodeSigners(); if (codeSigners == null || codeSigners.length == 0) { - throw new UpdateException(repo, "No signature found in index"); + throw new SigningException(repo, "No signature found in index"); } /* we could in theory support more than 1, but as of now we do not */ if (codeSigners.length > 1) { - throw new UpdateException(repo, "index.jar must be signed by a single code signer!"); + throw new SigningException(repo, "index.jar must be signed by a single code signer!"); } List certs = codeSigners[0].getSignerCertPath().getCertificates(); if (certs.size() != 1) { - throw new UpdateException(repo, "index.jar code signers must only have a single certificate!"); + throw new SigningException(repo, "index.jar code signers must only have a single certificate!"); } return (X509Certificate) certs.get(0); } @@ -285,7 +540,7 @@ public class RepoUpdater { * check that the signing certificate in the jar matches that fingerprint. */ private void verifyAndStoreTOFUCerts(String certFromIndexXml, X509Certificate rawCertFromJar) - throws UpdateException { + throws SigningException { if (repo.pubkey != null) return; // there is a repo.pubkey already, nothing to TOFU @@ -293,30 +548,20 @@ public class RepoUpdater { * fingerprint. In that case, check that fingerprint against what is * actually in the index.jar itself. If no fingerprint, just store the * signing certificate */ - boolean trustNewSigningCertificate; - // If the fingerprint has never been set, it will be null (never "" or something else) - if (repo.fingerprint == null) { - // no info to check things are valid, so just Trust On First Use - trustNewSigningCertificate = true; - } else { + if (repo.fingerprint != null) { String fingerprintFromIndexXml = Utils.calcFingerprint(certFromIndexXml); String fingerprintFromJar = Utils.calcFingerprint(rawCertFromJar); - if (repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml) - && repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) { - trustNewSigningCertificate = true; - } else { - throw new UpdateException(repo, "Supplied certificate fingerprint does not match: '" - + repo.fingerprint + "' '" + fingerprintFromIndexXml + "' '" + fingerprintFromJar + "'"); + if (!repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml) + || !repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) { + throw new SigningException(repo, "Supplied certificate fingerprint does not match!"); } - } + } // else - no info to check things are valid, so just Trust On First Use - if (trustNewSigningCertificate) { - Utils.debugLog(TAG, "Saving new signing certificate in the database for " + repo.address); - ContentValues values = new ContentValues(2); - values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatTime(new Date(), "")); - values.put(RepoProvider.DataColumns.PUBLIC_KEY, Hasher.hex(rawCertFromJar)); - RepoProvider.Helper.update(context, repo, values); - } + Utils.debugLog(TAG, "Saving new signing certificate in the database for " + repo.address); + ContentValues values = new ContentValues(2); + values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatDate(new Date(), "")); + values.put(RepoProvider.DataColumns.PUBLIC_KEY, Hasher.hex(rawCertFromJar)); + RepoProvider.Helper.update(context, repo, values); } /** @@ -331,16 +576,15 @@ public class RepoUpdater { * @param certFromIndexXml the cert written into the header of the index XML * @param rawCertFromJar the {@link X509Certificate} embedded in the downloaded jar */ - private void verifyCerts(String certFromIndexXml, X509Certificate rawCertFromJar) throws UpdateException { + private void verifyCerts(String certFromIndexXml, X509Certificate rawCertFromJar) throws SigningException { // convert binary data to string version that is used in FDroid's database String certFromJar = Hasher.hex(rawCertFromJar); // repo and repo.pubkey must be pre-loaded from the database - if (repo == null - || TextUtils.isEmpty(repo.pubkey) + if (TextUtils.isEmpty(repo.pubkey) || TextUtils.isEmpty(certFromJar) || TextUtils.isEmpty(certFromIndexXml)) - throw new UpdateException(repo, "A empty repo or signing certificate is invalid!"); + throw new SigningException(repo, "A empty repo or signing certificate is invalid!"); // though its called repo.pubkey, its actually a X509 certificate if (repo.pubkey.equals(certFromJar) @@ -348,7 +592,7 @@ public class RepoUpdater { && certFromIndexXml.equals(certFromJar)) { return; // we have a match! } - throw new UpdateException(repo, "Signing certificate does not match!"); + throw new SigningException(repo, "Signing certificate does not match!"); } } diff --git a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java index 210bc714b..c0dd841f0 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java @@ -19,6 +19,9 @@ package org.fdroid.fdroid; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import org.fdroid.fdroid.data.Apk; import org.fdroid.fdroid.data.App; import org.fdroid.fdroid.data.Repo; @@ -31,61 +34,52 @@ import java.util.List; /** * Parses the index.xml into Java data structures. + * + * For streaming apks from an index file, it is helpful if the index has the tag before + * any tags. This means that apps and apks can be saved instantly by the RepoUpdater, + * without having to buffer them at all, saving memory. The XML spec doesn't mandate order like + * this, though it is almost always a fair assumption: + * + * http://www.ibm.com/developerworks/library/x-eleord/index.html + * + * This is doubly so, as repo indices are likely from fdroidserver, which will output everybodys + * repo the same way. Having said that, this also should not be _forced_ upon people, but we can + * at least consider rejecting malformed indexes. */ public class RepoXMLHandler extends DefaultHandler { // The repo we're processing. private final Repo repo; - private final List apps = new ArrayList<>(); - private final List apksList = new ArrayList<>(); + private List apksList = new ArrayList<>(); private App curapp; private Apk curapk; - private final StringBuilder curchars = new StringBuilder(); + + private String currentApkHashType = null; // After processing the XML, these will be -1 if the index didn't specify // them - otherwise it will be the value specified. - private int version = -1; - private int maxage = -1; + private int repoMaxAge = -1; + private int repoVersion = 0; + private String repoDescription; + private String repoName; - /** the X.509 signing certificate stored in the header of index.xml */ - private String signingCertFromIndexXml; + // the X.509 signing certificate stored in the header of index.xml + private String repoSigningCert; - private String name; - private String description; - private String hashType; + private final StringBuilder curchars = new StringBuilder(); - public RepoXMLHandler(Repo repo) { + interface IndexReceiver { + void receiveRepo(String name, String description, String signingCert, int maxage, int version); + void receiveApp(App app, List packages); + } + + private IndexReceiver receiver; + + public RepoXMLHandler(Repo repo, @NonNull IndexReceiver receiver) { this.repo = repo; - } - - public List getApps() { - return apps; - } - - public List getApks() { - return apksList; - } - - public int getMaxAge() { - return maxage; - } - - public int getVersion() { - return version; - } - - public String getDescription() { - return description; - } - - public String getName() { - return name; - } - - public String getSigningCertFromIndexXml() { - return signingCertFromIndexXml; + this.receiver = receiver; } @Override @@ -97,22 +91,13 @@ public class RepoXMLHandler extends DefaultHandler { public void endElement(String uri, String localName, String qName) throws SAXException { - super.endElement(uri, localName, qName); - if ("application".equals(localName) && curapp != null) { - apps.add(curapp); - curapp = null; - // If the app id is already present in this apps list, then it - // means the same index file has a duplicate app, which should - // not be allowed. - // However, I'm thinking that it should be unefined behaviour, - // because it is probably a bug in the fdroid server that made it - // happen, and I don't *think* it will crash the client, because - // the first app will insert, the second one will update the newly - // inserted one. + onApplicationParsed(); } else if ("package".equals(localName) && curapk != null && curapp != null) { apksList.add(curapk); curapk = null; + } else if ("repo".equals(localName)) { + onRepoParsed(); } else if (curchars.length() == 0) { // All options below require non-empty content return; @@ -120,53 +105,53 @@ public class RepoXMLHandler extends DefaultHandler { final String str = curchars.toString().trim(); if (curapk != null) { switch (localName) { - case "version": - curapk.version = str; - break; - case "versioncode": - curapk.vercode = Utils.parseInt(str, -1); - break; - case "size": - curapk.size = Utils.parseInt(str, 0); - break; - case "hash": - if (hashType == null || "md5".equals(hashType)) { - if (curapk.hash == null) { - curapk.hash = str; - curapk.hashType = "MD5"; - } - } else if ("sha256".equals(hashType)) { + case "version": + curapk.version = str; + break; + case "versioncode": + curapk.vercode = Utils.parseInt(str, -1); + break; + case "size": + curapk.size = Utils.parseInt(str, 0); + break; + case "hash": + if (currentApkHashType == null || currentApkHashType.equals("md5")) { + if (curapk.hash == null) { curapk.hash = str; curapk.hashType = "SHA-256"; } - break; - case "sig": - curapk.sig = str; - break; - case "srcname": - curapk.srcname = str; - break; - case "apkname": - curapk.apkName = str; - break; - case "sdkver": - curapk.minSdkVersion = Utils.parseInt(str, 0); - break; - case "maxsdkver": - curapk.maxSdkVersion = Utils.parseInt(str, 0); - break; - case "added": - curapk.added = Utils.parseDate(str, null); - break; - case "permissions": - curapk.permissions = Utils.CommaSeparatedList.make(str); - break; - case "features": - curapk.features = Utils.CommaSeparatedList.make(str); - break; - case "nativecode": - curapk.nativecode = Utils.CommaSeparatedList.make(str); - break; + } else if (currentApkHashType.equals("sha256")) { + curapk.hash = str; + curapk.hashType = "SHA-256"; + } + break; + case "sig": + curapk.sig = str; + break; + case "srcname": + curapk.srcname = str; + break; + case "apkname": + curapk.apkName = str; + break; + case "sdkver": + curapk.minSdkVersion = Utils.parseInt(str, 0); + break; + case "maxsdkver": + curapk.maxSdkVersion = Utils.parseInt(str, 0); + break; + case "added": + curapk.added = Utils.parseDate(str, null); + break; + case "permissions": + curapk.permissions = Utils.CommaSeparatedList.make(str); + break; + case "features": + curapk.features = Utils.CommaSeparatedList.make(str); + break; + case "nativecode": + curapk.nativecode = Utils.CommaSeparatedList.make(str); + break; } } else if (curapp != null) { switch (localName) { @@ -239,27 +224,39 @@ public class RepoXMLHandler extends DefaultHandler { break; } } else if ("description".equals(localName)) { - description = cleanWhiteSpace(str); + repoDescription = cleanWhiteSpace(str); } } + private void onApplicationParsed() { + receiver.receiveApp(curapp, apksList); + curapp = null; + apksList = new ArrayList<>(); + // If the app id is already present in this apps list, then it + // means the same index file has a duplicate app, which should + // not be allowed. + // However, I'm thinking that it should be undefined behaviour, + // because it is probably a bug in the fdroid server that made it + // happen, and I don't *think* it will crash the client, because + // the first app will insert, the second one will update the newly + // inserted one. + } + + private void onRepoParsed() { + receiver.receiveRepo(repoName, repoDescription, repoSigningCert, repoMaxAge, repoVersion); + } + @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { super.startElement(uri, localName, qName, attributes); if ("repo".equals(localName)) { - signingCertFromIndexXml = attributes.getValue("", "pubkey"); - maxage = Utils.parseInt(attributes.getValue("", "maxage"), -1); - version = Utils.parseInt(attributes.getValue("", "version"), -1); - - final String nm = attributes.getValue("", "name"); - if (nm != null) - name = cleanWhiteSpace(nm); - final String dc = attributes.getValue("", "description"); - if (dc != null) - description = cleanWhiteSpace(dc); - + repoSigningCert = attributes.getValue("", "pubkey"); + repoMaxAge = Utils.parseInt(attributes.getValue("", "maxage"), -1); + repoVersion = Utils.parseInt(attributes.getValue("", "version"), -1); + repoName = cleanWhiteSpace(attributes.getValue("", "name")); + repoDescription = cleanWhiteSpace(attributes.getValue("", "description")); } else if ("application".equals(localName) && curapp == null) { curapp = new App(); curapp.id = attributes.getValue("", "id"); @@ -267,15 +264,15 @@ public class RepoXMLHandler extends DefaultHandler { curapk = new Apk(); curapk.id = curapp.id; curapk.repo = repo.getId(); - hashType = null; + currentApkHashType = null; } else if ("hash".equals(localName) && curapk != null) { - hashType = attributes.getValue("", "type"); + currentApkHashType = attributes.getValue("", "type"); } curchars.setLength(0); } - private String cleanWhiteSpace(String str) { - return str.replaceAll("\n", " ").replaceAll(" ", " "); + private String cleanWhiteSpace(@Nullable String str) { + return str == null ? null : str.replaceAll("\\s", " "); } } diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index 0e19d7e7d..3105b5851 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -139,11 +139,16 @@ public class UpdateService extends IntentService implements ProgressListener { .setOngoing(true) .setCategory(NotificationCompat.CATEGORY_SERVICE) .setContentTitle(getString(R.string.update_notification_title)); - - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH) { - Intent intent = new Intent(this, FDroid.class); - // TODO: Is this the correct FLAG? - notificationBuilder.setContentIntent(PendingIntent.getActivity(this, 0, intent, PendingIntent.FLAG_UPDATE_CURRENT)); + + // Android docs are a little sketchy, however it seems that Gingerbread is the last + // sdk that made a content intent mandatory: + // + // http://stackoverflow.com/a/20032920 + // + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.GINGERBREAD_MR1) { + Intent pendingIntent = new Intent(this, FDroid.class); + pendingIntent.addFlags (Intent.FLAG_ACTIVITY_NEW_TASK); + notificationBuilder.setContentIntent(PendingIntent.getActivity(this, 0, pendingIntent, PendingIntent.FLAG_UPDATE_CURRENT)); } notificationManager.notify(NOTIFY_ID_UPDATING, notificationBuilder.build()); @@ -324,6 +329,7 @@ public class UpdateService extends IntentService implements ProgressListener { @Override protected void onHandleIntent(Intent intent) { + final long startTime = System.currentTimeMillis(); String address = intent.getStringExtra(EXTRA_ADDRESS); boolean manualUpdate = intent.getBooleanExtra(EXTRA_MANUAL_UPDATE, false); @@ -341,16 +347,12 @@ public class UpdateService extends IntentService implements ProgressListener { // database while we do all the downloading, etc... List repos = RepoProvider.Helper.all(this); - // Process each repo... - RepoPersister appSaver = new RepoPersister(this); - //List swapRepos = new ArrayList<>(); List unchangedRepos = new ArrayList<>(); List updatedRepos = new ArrayList<>(); List disabledRepos = new ArrayList<>(); List errorRepos = new ArrayList<>(); ArrayList repoErrors = new ArrayList<>(); - List repoUpdateRememberers = new ArrayList<>(); boolean changes = false; boolean singleRepoUpdate = !TextUtils.isEmpty(address); for (final Repo repo : repos) { @@ -374,10 +376,8 @@ public class UpdateService extends IntentService implements ProgressListener { try { updater.update(); if (updater.hasChanged()) { - appSaver.queueUpdater(updater); updatedRepos.add(repo); changes = true; - repoUpdateRememberers.add(updater.getRememberer()); } else { unchangedRepos.add(repo); } @@ -392,16 +392,8 @@ public class UpdateService extends IntentService implements ProgressListener { Utils.debugLog(TAG, "Not checking app details or compatibility, because all repos were up to date."); } else { sendStatus(this, STATUS_INFO, getString(R.string.status_checking_compatibility)); - - appSaver.save(disabledRepos); - notifyContentProviders(); - //we only remember the update if everything has gone well - for (RepoUpdater.RepoUpdateRememberer rememberer : repoUpdateRememberers) { - rememberer.rememberUpdate(); - } - if (prefs.getBoolean(Preferences.PREF_UPD_NOTIFY, true)) { performUpdateNotification(); } @@ -428,6 +420,9 @@ public class UpdateService extends IntentService implements ProgressListener { Log.e(TAG, "Exception during update processing", e); sendStatus(this, STATUS_ERROR_GLOBAL, e.getMessage()); } + + long time = System.currentTimeMillis() - startTime; + Log.i(TAG, "Updating repo(s) complete, took " + time / 1000 + " seconds to complete."); } private void notifyContentProviders() { diff --git a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java index 26d5bb83d..495689c17 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java @@ -88,6 +88,24 @@ public class ApkProvider extends FDroidProvider { return find(context, id, versionCode, DataColumns.ALL); } + /** + * Find all apks for a particular app, but limit it to those originating from the + * specified repo. + */ + public static List find(Context context, Repo repo, List apps, String[] projection) { + ContentResolver resolver = context.getContentResolver(); + final Uri uri = getContentUriForApps(repo, apps); + Cursor cursor = resolver.query(uri, projection, null, null, null); + return cursorToList(cursor); + } + + /** + * @see org.fdroid.fdroid.data.ApkProvider.Helper#find(Context, Repo, List, String[]) + */ + public static List find(Context context, Repo repo, List apps) { + return find(context, repo, apps, DataColumns.ALL); + } + public static Apk find(Context context, String id, int versionCode, String[] projection) { ContentResolver resolver = context.getContentResolver(); final Uri uri = getContentUri(id, versionCode); @@ -208,12 +226,16 @@ public class ApkProvider extends FDroidProvider { private static final int CODE_APP = CODE_SINGLE + 1; private static final int CODE_REPO = CODE_APP + 1; private static final int CODE_APKS = CODE_REPO + 1; + private static final int CODE_REPO_APPS = CODE_APKS + 1; + private static final int CODE_REPO_APK = CODE_REPO_APPS + 1; - private static final String PROVIDER_NAME = "ApkProvider"; - private static final String PATH_APK = "apk"; - private static final String PATH_APKS = "apks"; - private static final String PATH_APP = "app"; - private static final String PATH_REPO = "repo"; + private static final String PROVIDER_NAME = "ApkProvider"; + private static final String PATH_APK = "apk"; + private static final String PATH_APKS = "apks"; + private static final String PATH_APP = "app"; + private static final String PATH_REPO = "repo"; + private static final String PATH_REPO_APPS = "repo-apps"; + private static final String PATH_REPO_APK = "repo-apk"; private static final UriMatcher matcher = new UriMatcher(-1); @@ -227,6 +249,8 @@ public class ApkProvider extends FDroidProvider { matcher.addURI(getAuthority(), PATH_APK + "/#/*", CODE_SINGLE); matcher.addURI(getAuthority(), PATH_APKS + "/*", CODE_APKS); matcher.addURI(getAuthority(), PATH_APP + "/*", CODE_APP); + matcher.addURI(getAuthority(), PATH_REPO_APPS + "/#/*", CODE_REPO_APPS); + matcher.addURI(getAuthority(), PATH_REPO_APK + "/#/*", CODE_REPO_APK); matcher.addURI(getAuthority(), null, CODE_LIST); } @@ -267,6 +291,24 @@ public class ApkProvider extends FDroidProvider { .build(); } + public static Uri getContentUriForApps(Repo repo, List apps) { + return getContentUri() + .buildUpon() + .appendPath(PATH_REPO_APPS) + .appendPath(Long.toString(repo.id)) + .appendPath(buildAppString(apps)) + .build(); + } + + public static Uri getContentUriForApks(Repo repo, List apks) { + return getContentUri() + .buildUpon() + .appendPath(PATH_REPO_APK) + .appendPath(Long.toString(repo.id)) + .appendPath(buildApkString(apks)) + .build(); + } + /** * Intentionally left protected because it will break if apks is larger than * {@link org.fdroid.fdroid.data.ApkProvider#MAX_APKS_TO_QUERY}. Instead of using @@ -274,6 +316,13 @@ public class ApkProvider extends FDroidProvider { * {@link org.fdroid.fdroid.data.ApkProvider.Helper#knownApks(android.content.Context, java.util.List, String[])} */ protected static Uri getContentUri(List apks) { + return getContentUri().buildUpon() + .appendPath(PATH_APKS) + .appendPath(buildApkString(apks)) + .build(); + } + + private static String buildApkString(List apks) { StringBuilder builder = new StringBuilder(); for (int i = 0; i < apks.size(); i++) { if (i != 0) { @@ -282,10 +331,18 @@ public class ApkProvider extends FDroidProvider { final Apk a = apks.get(i); builder.append(a.id).append(':').append(a.vercode); } - return getContentUri().buildUpon() - .appendPath(PATH_APKS) - .appendPath(builder.toString()) - .build(); + return builder.toString(); + } + + private static String buildAppString(List apks) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < apks.size(); i++) { + if (i != 0) { + builder.append(','); + } + builder.append(apks.get(0).id); + } + return builder.toString(); } @Override @@ -360,6 +417,10 @@ public class ApkProvider extends FDroidProvider { return new QuerySelection(selection, args); } + private QuerySelection queryRepoApps(long repoId, String appIds) { + return queryRepo(repoId).add(AppProvider.queryApps(appIds, DataColumns.APK_ID)); + } + private QuerySelection queryApks(String apkKeys) { final String[] apkDetails = apkKeys.split(","); if (apkDetails.length > MAX_APKS_TO_QUERY) { @@ -408,6 +469,11 @@ public class ApkProvider extends FDroidProvider { query = query.add(queryRepo(Long.parseLong(uri.getLastPathSegment()))); break; + case CODE_REPO_APPS: + List pathSegments = uri.getPathSegments(); + query = query.add(queryRepoApps(Long.parseLong(pathSegments.get(1)), pathSegments.get(2))); + break; + default: Log.e(TAG, "Invalid URI for apk content provider: " + uri); throw new UnsupportedOperationException("Invalid URI for apk content provider: " + uri); @@ -469,6 +535,12 @@ public class ApkProvider extends FDroidProvider { case CODE_APKS: query = query.add(queryApks(uri.getLastPathSegment())); break; + + // TODO: Add tests for this. + case CODE_REPO_APK: + List pathSegments = uri.getPathSegments(); + query = query.add(queryRepo(Long.parseLong(pathSegments.get(1)))).add(queryApks(pathSegments.get(2))); + break; case CODE_LIST: throw new UnsupportedOperationException("Can't delete all apks."); diff --git a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java index 1033e799f..8ddde3c53 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java @@ -686,12 +686,16 @@ public class AppProvider extends FDroidProvider { return new AppQuerySelection(selection); } - private AppQuerySelection queryApps(String appIds) { + static AppQuerySelection queryApps(String appIds, String idField) { String[] args = appIds.split(","); - String selection = "fdroid_app.id IN (" + generateQuestionMarksForInClause(args.length) + ")"; + String selection = idField + " IN (" + generateQuestionMarksForInClause(args.length) + ")"; return new AppQuerySelection(selection, args); } + private static AppQuerySelection queryApps(String appIds) { + return queryApps(appIds, "fdroid_app.id"); + } + @Override public Cursor query(Uri uri, String[] projection, String customSelection, String[] selectionArgs, String sortOrder) { AppQuerySelection selection = new AppQuerySelection(customSelection, selectionArgs); diff --git a/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java b/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java index ab5210adc..61267cde3 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java @@ -99,7 +99,7 @@ public abstract class FDroidProvider extends ContentProvider { protected abstract UriMatcher getMatcher(); - protected String generateQuestionMarksForInClause(int num) { + protected static String generateQuestionMarksForInClause(int num) { StringBuilder sb = new StringBuilder(num * 2); for (int i = 0; i < num; i++) { if (i != 0) { diff --git a/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java b/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java index 71c6e43ff..b5471f92c 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java @@ -22,7 +22,6 @@ import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; import java.io.File; -import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -38,7 +37,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { private RepoUpdater mainRepoUpdater; private RepoUpdater archiveRepoUpdater; private File testFilesDir; - private RepoPersister persister; private static final String PUB_KEY = "3082050b308202f3a003020102020420d8f212300d06092a864886f70d01010b050030363110300e0603" + @@ -137,8 +135,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { RepoProvider.Helper.remove(context, 3); RepoProvider.Helper.remove(context, 4); - persister = new RepoPersister(context); - conflictingRepoUpdater = createUpdater(REPO_CONFLICTING, context); mainRepoUpdater = createUpdater(REPO_MAIN, context); archiveRepoUpdater = createUpdater(REPO_ARCHIVE, context); @@ -285,7 +281,7 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { } } - assertTrue("Found app " + appId + ", v" + versionCode, found); + assertTrue("Couldn't find app " + appId + ", v" + versionCode, found); } } @@ -303,10 +299,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { } } - private void persistData() { - persister.save(new ArrayList(0)); - } - /* At time fo writing, the following tests did not pass. This is because the multi-repo support in F-Droid was not sufficient. When working on proper multi repo support than this should be ucommented and all these tests should pass: @@ -314,7 +306,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectConflictingThenMainThenArchive() throws UpdateException { assertEmpty(); if (updateConflicting() && updateMain() && updateArchive()) { - persistData(); assertExpected(); } } @@ -322,7 +313,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectConflictingThenArchiveThenMain() throws UpdateException { assertEmpty(); if (updateConflicting() && updateArchive() && updateMain()) { - persistData(); assertExpected(); } } @@ -330,7 +320,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectArchiveThenMainThenConflicting() throws UpdateException { assertEmpty(); if (updateArchive() && updateMain() && updateConflicting()) { - persistData(); assertExpected(); } } @@ -338,7 +327,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectArchiveThenConflictingThenMain() throws UpdateException { assertEmpty(); if (updateArchive() && updateConflicting() && updateMain()) { - persistData(); assertExpected(); } } @@ -346,7 +334,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectMainThenArchiveThenConflicting() throws UpdateException { assertEmpty(); if (updateMain() && updateArchive() && updateConflicting()) { - persistData(); assertExpected(); } } @@ -354,7 +341,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectMainThenConflictingThenArchive() throws UpdateException { assertEmpty(); if (updateMain() && updateConflicting() && updateArchive()) { - persistData(); assertExpected(); } } @@ -364,7 +350,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableConflictingThenMainThenArchive() throws UpdateException { assertEmpty(); if (updateConflicting() && updateMain() && updateArchive()) { - persistData(); assertSomewhatAcceptable(); } } @@ -372,7 +357,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableConflictingThenArchiveThenMain() throws UpdateException { assertEmpty(); if (updateConflicting() && updateArchive() && updateMain()) { - persistData(); assertSomewhatAcceptable(); } } @@ -380,7 +364,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableArchiveThenMainThenConflicting() throws UpdateException { assertEmpty(); if (updateArchive() && updateMain() && updateConflicting()) { - persistData(); assertSomewhatAcceptable(); } } @@ -388,7 +371,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableArchiveThenConflictingThenMain() throws UpdateException { assertEmpty(); if (updateArchive() && updateConflicting() && updateMain()) { - persistData(); assertSomewhatAcceptable(); } } @@ -396,7 +378,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableMainThenArchiveThenConflicting() throws UpdateException { assertEmpty(); if (updateMain() && updateArchive() && updateConflicting()) { - persistData(); assertSomewhatAcceptable(); } } @@ -404,7 +385,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableMainThenConflictingThenArchive() throws UpdateException { assertEmpty(); if (updateMain() && updateConflicting() && updateArchive()) { - persistData(); assertSomewhatAcceptable(); } } @@ -444,8 +424,7 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { return false; File indexJar = TestUtils.copyAssetToDir(context, indexJarPath, testFilesDir); - updater.processDownloadedFile(indexJar, UUID.randomUUID().toString()); - persister.queueUpdater(updater); + updater.processDownloadedFile(indexJar); return true; } diff --git a/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java b/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java index a588df536..c8692105b 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java @@ -35,7 +35,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // these are supposed to succeed try { - repoUpdater.processDownloadedFile(simpleIndexJar, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(simpleIndexJar); } catch (UpdateException e) { e.printStackTrace(); fail(); @@ -48,7 +48,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithoutSignature.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { // success! @@ -61,7 +61,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedManifest.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -77,7 +77,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedSignature.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -93,7 +93,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedCertificate.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -109,7 +109,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedEverything.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -125,7 +125,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "masterKeyIndex.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException | SecurityException e) { // success! diff --git a/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java b/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java index ee37d748a..c9be7f43c 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java @@ -15,6 +15,7 @@ import org.xml.sax.XMLReader; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.List; import javax.xml.parsers.ParserConfigurationException; @@ -24,9 +25,34 @@ import javax.xml.parsers.SAXParserFactory; public class RepoXMLHandlerTest extends AndroidTestCase { private static final String TAG = "RepoXMLHandlerTest"; - private Repo repo; + private final Repo actualRepo = new Repo(); - private String fakePubkey = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345"; + public final List actualApps = new ArrayList<>(); + public final List actualApks = new ArrayList<>(); + + private final static String FAKE_PUBKEY = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345"; + + private RepoXMLHandler.IndexReceiver indexReceiver = new RepoXMLHandler.IndexReceiver() { + + private boolean hasReceivedRepo; + + @Override + public void receiveRepo(String name, String description, String signingCert, int maxage, int version) { + assertFalse("Repo XML contains more than one .", hasReceivedRepo); + actualRepo.name = name; + actualRepo.description = description; + actualRepo.pubkey = signingCert; + actualRepo.maxage = maxage; + actualRepo.version = version; + hasReceivedRepo = true; + } + + @Override + public void receiveApp(App app, List packages) { + actualApps.add(app); + actualApks.addAll(packages); + } + }; public RepoXMLHandlerTest() { } @@ -34,28 +60,29 @@ public class RepoXMLHandlerTest extends AndroidTestCase { @Override protected void setUp() throws Exception { super.setUp(); - repo = new Repo(); } public void testSimpleIndex() { - repo.name = "F-Droid"; - repo.pubkey = "308201ee30820157a0030201020204300d845b300d06092a864886f70d01010b0500302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e301e170d3134303432373030303633315a170d3431303931323030303633315a302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e30819f300d06092a864886f70d010101050003818d0030818902818100a439472e4b6d01141bfc94ecfe131c7c728fdda670bb14c57ca60bd1c38a8b8bc0879d22a0a2d0bc0d6fdd4cb98d1d607c2caefbe250a0bd0322aedeb365caf9b236992fac13e6675d3184a6c7c6f07f73410209e399a9da8d5d7512bbd870508eebacff8b57c3852457419434d34701ccbf692267cbc3f42f1c5d1e23762d790203010001a321301f301d0603551d0e041604140b1840691dab909746fde4bfe28207d1cae15786300d06092a864886f70d01010b05000381810062424c928ffd1b6fd419b44daafef01ca982e09341f7077fb865905087aeac882534b3bd679b51fdfb98892cef38b63131c567ed26c9d5d9163afc775ac98ad88c405d211d6187bde0b0d236381cc574ba06ef9080721a92ae5a103a7301b2c397eecc141cc850dd3e123813ebc41c59d31ddbcb6e984168280c53272f6a442b"; - repo.description = "The official repository of the F-Droid client. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitorious.org/f-droid."; - RepoXMLHandler handler = getFromFile(repo, "simpleIndex.xml"); - handlerTestSuite(repo, handler, 0, 0); - assertEquals(handler.getMaxAge(), -1); - assertEquals(handler.getVersion(), 12); + Repo expectedRepo = new Repo(); + expectedRepo.name = "F-Droid"; + expectedRepo.pubkey = "308201ee30820157a0030201020204300d845b300d06092a864886f70d01010b0500302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e301e170d3134303432373030303633315a170d3431303931323030303633315a302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e30819f300d06092a864886f70d010101050003818d0030818902818100a439472e4b6d01141bfc94ecfe131c7c728fdda670bb14c57ca60bd1c38a8b8bc0879d22a0a2d0bc0d6fdd4cb98d1d607c2caefbe250a0bd0322aedeb365caf9b236992fac13e6675d3184a6c7c6f07f73410209e399a9da8d5d7512bbd870508eebacff8b57c3852457419434d34701ccbf692267cbc3f42f1c5d1e23762d790203010001a321301f301d0603551d0e041604140b1840691dab909746fde4bfe28207d1cae15786300d06092a864886f70d01010b05000381810062424c928ffd1b6fd419b44daafef01ca982e09341f7077fb865905087aeac882534b3bd679b51fdfb98892cef38b63131c567ed26c9d5d9163afc775ac98ad88c405d211d6187bde0b0d236381cc574ba06ef9080721a92ae5a103a7301b2c397eecc141cc850dd3e123813ebc41c59d31ddbcb6e984168280c53272f6a442b"; + expectedRepo.description = "The official repository of the F-Droid client. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitorious.org/f-droid."; + processFile("simpleIndex.xml"); + handlerTestSuite(expectedRepo, 0, 0); + assertEquals(actualRepo.maxage, -1); + assertEquals(actualRepo.version, 12); } public void testSmallRepo() { - repo.name = "Android-Nexus-7-20139453 on UNSET"; - repo.pubkey = "308202da308201c2a00302010202080eb08c796fec91aa300d06092a864886f70d0101050500302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a656374301e170d3134313030333135303631325a170d3135313030333135303631325a302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a65637430820122300d06092a864886f70d01010105000382010f003082010a0282010100c7ab44b130be5c00eedcc3625462f6f6ac26e502641cd641f3e30cbb0ff1ba325158611e7fc2448a35b6a6df30dc6e23602cf6909448befcf11e2fe486b580f1e76fe5887d159050d00afd2c4079f6538896bb200627f4b3e874f011ce5df0fef5d150fcb0b377b531254e436eaf4083ea72fe3b8c3ef450789fa858f2be8f6c5335bb326aff3dda689fbc7b5ba98dea53651dbea7452c38d294985ac5dd8a9e491a695de92c706d682d6911411fcaef3b0a08a030fe8a84e47acaab0b7edcda9d190ce39e810b79b1d8732eca22b15f0d048c8d6f00503a7ee81ab6e08919ff465883432304d95238b95e95c5f74e0a421809e2a6a85825aed680e0d6939e8f0203010001300d06092a864886f70d010105050003820101006d17aad3271b8b2c299dbdb7b1182849b0d5ddb9f1016dcb3487ae0db02b6be503344c7d066e2050bcd01d411b5ee78c7ed450f0ff9da5ce228f774cbf41240361df53d9c6078159d16f4d34379ab7dedf6186489397c83b44b964251a2ebb42b7c4689a521271b1056d3b5a5fa8f28ba64fb8ce5e2226c33c45d27ba3f632dc266c12abf582b8438c2abcf3eae9de9f31152b4158ace0ef33435c20eb809f1b3988131db6e5a1442f2617c3491d9565fedb3e320e8df4236200d3bd265e47934aa578f84d0d1a5efeb49b39907e876452c46996d0feff9404b41aa5631b4482175d843d5512ded45e12a514690646492191e7add434afce63dbff8f0b03ec0c"; - repo.description = "A local FDroid repo generated from apps installed on Android-Nexus-7-20139453"; - RepoXMLHandler handler = getFromFile(repo, "smallRepo.xml"); - handlerTestSuite(repo, handler, 12, 12); - assertEquals(handler.getMaxAge(), 14); - assertEquals(handler.getVersion(), -1); - checkIncludedApps(handler.getApps(), new String[] { + Repo expectedRepo = new Repo(); + expectedRepo.name = "Android-Nexus-7-20139453 on UNSET"; + expectedRepo.pubkey = "308202da308201c2a00302010202080eb08c796fec91aa300d06092a864886f70d0101050500302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a656374301e170d3134313030333135303631325a170d3135313030333135303631325a302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a65637430820122300d06092a864886f70d01010105000382010f003082010a0282010100c7ab44b130be5c00eedcc3625462f6f6ac26e502641cd641f3e30cbb0ff1ba325158611e7fc2448a35b6a6df30dc6e23602cf6909448befcf11e2fe486b580f1e76fe5887d159050d00afd2c4079f6538896bb200627f4b3e874f011ce5df0fef5d150fcb0b377b531254e436eaf4083ea72fe3b8c3ef450789fa858f2be8f6c5335bb326aff3dda689fbc7b5ba98dea53651dbea7452c38d294985ac5dd8a9e491a695de92c706d682d6911411fcaef3b0a08a030fe8a84e47acaab0b7edcda9d190ce39e810b79b1d8732eca22b15f0d048c8d6f00503a7ee81ab6e08919ff465883432304d95238b95e95c5f74e0a421809e2a6a85825aed680e0d6939e8f0203010001300d06092a864886f70d010105050003820101006d17aad3271b8b2c299dbdb7b1182849b0d5ddb9f1016dcb3487ae0db02b6be503344c7d066e2050bcd01d411b5ee78c7ed450f0ff9da5ce228f774cbf41240361df53d9c6078159d16f4d34379ab7dedf6186489397c83b44b964251a2ebb42b7c4689a521271b1056d3b5a5fa8f28ba64fb8ce5e2226c33c45d27ba3f632dc266c12abf582b8438c2abcf3eae9de9f31152b4158ace0ef33435c20eb809f1b3988131db6e5a1442f2617c3491d9565fedb3e320e8df4236200d3bd265e47934aa578f84d0d1a5efeb49b39907e876452c46996d0feff9404b41aa5631b4482175d843d5512ded45e12a514690646492191e7add434afce63dbff8f0b03ec0c"; + expectedRepo.description = "A local FDroid repo generated from apps installed on Android-Nexus-7-20139453"; + processFile("smallRepo.xml"); + handlerTestSuite(expectedRepo, 12, 12); + assertEquals(actualRepo.maxage, 14); + assertEquals(actualRepo.version, -1); + checkIncludedApps(new String[] { "org.mozilla.firefox", "com.koushikdutta.superuser", "info.guardianproject.courier", @@ -72,14 +99,15 @@ public class RepoXMLHandlerTest extends AndroidTestCase { } public void testMediumRepo() { - repo.name = "Guardian Project Official Releases"; - repo.pubkey = "308205d8308203c0020900a397b4da7ecda034300d06092a864886f70d01010505003081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f301e170d3134303632363139333931385a170d3431313131303139333931385a3081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f30820222300d06092a864886f70d01010105000382020f003082020a0282020100b3cd79121b9b883843be3c4482e320809106b0a23755f1dd3c7f46f7d315d7bb2e943486d61fc7c811b9294dcc6b5baac4340f8db2b0d5e14749e7f35e1fc211fdbc1071b38b4753db201c314811bef885bd8921ad86facd6cc3b8f74d30a0b6e2e6e576f906e9581ef23d9c03e926e06d1f033f28bd1e21cfa6a0e3ff5c9d8246cf108d82b488b9fdd55d7de7ebb6a7f64b19e0d6b2ab1380a6f9d42361770d1956701a7f80e2de568acd0bb4527324b1e0973e89595d91c8cc102d9248525ae092e2c9b69f7414f724195b81427f28b1d3d09a51acfe354387915fd9521e8c890c125fc41a12bf34d2a1b304067ab7251e0e9ef41833ce109e76963b0b256395b16b886bca21b831f1408f836146019e7908829e716e72b81006610a2af08301de5d067c9e114a1e5759db8a6be6a3cc2806bcfe6fafd41b5bc9ddddb3dc33d6f605b1ca7d8a9e0ecdd6390d38906649e68a90a717bea80fa220170eea0c86fc78a7e10dac7b74b8e62045a3ecca54e035281fdc9fe5920a855fde3c0be522e3aef0c087524f13d973dff3768158b01a5800a060c06b451ec98d627dd052eda804d0556f60dbc490d94e6e9dea62ffcafb5beffbd9fc38fb2f0d7050004fe56b4dda0a27bc47554e1e0a7d764e17622e71f83a475db286bc7862deee1327e2028955d978272ea76bf0b88e70a18621aba59ff0c5993ef5f0e5d6b6b98e68b70203010001300d06092a864886f70d0101050500038202010079c79c8ef408a20d243d8bd8249fb9a48350dc19663b5e0fce67a8dbcb7de296c5ae7bbf72e98a2020fb78f2db29b54b0e24b181aa1c1d333cc0303685d6120b03216a913f96b96eb838f9bff125306ae3120af838c9fc07ebb5100125436bd24ec6d994d0bff5d065221871f8410daf536766757239bf594e61c5432c9817281b985263bada8381292e543a49814061ae11c92a316e7dc100327b59e3da90302c5ada68c6a50201bda1fcce800b53f381059665dbabeeb0b50eb22b2d7d2d9b0aa7488ca70e67ac6c518adb8e78454a466501e89d81a45bf1ebc350896f2c3ae4b6679ecfbf9d32960d4f5b493125c7876ef36158562371193f600bc511000a67bdb7c664d018f99d9e589868d103d7e0994f166b2ba18ff7e67d8c4da749e44dfae1d930ae5397083a51675c409049dfb626a96246c0015ca696e94ebb767a20147834bf78b07fece3f0872b057c1c519ff882501995237d8206b0b3832f78753ebd8dcbd1d3d9f5ba733538113af6b407d960ec4353c50eb38ab29888238da843cd404ed8f4952f59e4bbc0035fc77a54846a9d419179c46af1b4a3b7fc98e4d312aaa29b9b7d79e739703dc0fa41c7280d5587709277ffa11c3620f5fba985b82c238ba19b17ebd027af9424be0941719919f620dd3bb3c3f11638363708aa11f858e153cf3a69bce69978b90e4a273836100aa1e617ba455cd00426847f"; - repo.description = "The official app repository of The Guardian Project. Applications in this repository are official binaries build by the original application developers and signed by the same key as the APKs that are released in the Google Play store."; - RepoXMLHandler handler = getFromFile(repo, "mediumRepo.xml"); - handlerTestSuite(repo, handler, 15, 36); - assertEquals(handler.getMaxAge(), 60); - assertEquals(handler.getVersion(), 12); - checkIncludedApps(handler.getApps(), new String[] { + Repo expectedRepo = new Repo(); + expectedRepo.name = "Guardian Project Official Releases"; + expectedRepo.pubkey = "308205d8308203c0020900a397b4da7ecda034300d06092a864886f70d01010505003081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f301e170d3134303632363139333931385a170d3431313131303139333931385a3081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f30820222300d06092a864886f70d01010105000382020f003082020a0282020100b3cd79121b9b883843be3c4482e320809106b0a23755f1dd3c7f46f7d315d7bb2e943486d61fc7c811b9294dcc6b5baac4340f8db2b0d5e14749e7f35e1fc211fdbc1071b38b4753db201c314811bef885bd8921ad86facd6cc3b8f74d30a0b6e2e6e576f906e9581ef23d9c03e926e06d1f033f28bd1e21cfa6a0e3ff5c9d8246cf108d82b488b9fdd55d7de7ebb6a7f64b19e0d6b2ab1380a6f9d42361770d1956701a7f80e2de568acd0bb4527324b1e0973e89595d91c8cc102d9248525ae092e2c9b69f7414f724195b81427f28b1d3d09a51acfe354387915fd9521e8c890c125fc41a12bf34d2a1b304067ab7251e0e9ef41833ce109e76963b0b256395b16b886bca21b831f1408f836146019e7908829e716e72b81006610a2af08301de5d067c9e114a1e5759db8a6be6a3cc2806bcfe6fafd41b5bc9ddddb3dc33d6f605b1ca7d8a9e0ecdd6390d38906649e68a90a717bea80fa220170eea0c86fc78a7e10dac7b74b8e62045a3ecca54e035281fdc9fe5920a855fde3c0be522e3aef0c087524f13d973dff3768158b01a5800a060c06b451ec98d627dd052eda804d0556f60dbc490d94e6e9dea62ffcafb5beffbd9fc38fb2f0d7050004fe56b4dda0a27bc47554e1e0a7d764e17622e71f83a475db286bc7862deee1327e2028955d978272ea76bf0b88e70a18621aba59ff0c5993ef5f0e5d6b6b98e68b70203010001300d06092a864886f70d0101050500038202010079c79c8ef408a20d243d8bd8249fb9a48350dc19663b5e0fce67a8dbcb7de296c5ae7bbf72e98a2020fb78f2db29b54b0e24b181aa1c1d333cc0303685d6120b03216a913f96b96eb838f9bff125306ae3120af838c9fc07ebb5100125436bd24ec6d994d0bff5d065221871f8410daf536766757239bf594e61c5432c9817281b985263bada8381292e543a49814061ae11c92a316e7dc100327b59e3da90302c5ada68c6a50201bda1fcce800b53f381059665dbabeeb0b50eb22b2d7d2d9b0aa7488ca70e67ac6c518adb8e78454a466501e89d81a45bf1ebc350896f2c3ae4b6679ecfbf9d32960d4f5b493125c7876ef36158562371193f600bc511000a67bdb7c664d018f99d9e589868d103d7e0994f166b2ba18ff7e67d8c4da749e44dfae1d930ae5397083a51675c409049dfb626a96246c0015ca696e94ebb767a20147834bf78b07fece3f0872b057c1c519ff882501995237d8206b0b3832f78753ebd8dcbd1d3d9f5ba733538113af6b407d960ec4353c50eb38ab29888238da843cd404ed8f4952f59e4bbc0035fc77a54846a9d419179c46af1b4a3b7fc98e4d312aaa29b9b7d79e739703dc0fa41c7280d5587709277ffa11c3620f5fba985b82c238ba19b17ebd027af9424be0941719919f620dd3bb3c3f11638363708aa11f858e153cf3a69bce69978b90e4a273836100aa1e617ba455cd00426847f"; + expectedRepo.description = "The official app repository of The Guardian Project. Applications in this repository are official binaries build by the original application developers and signed by the same key as the APKs that are released in the Google Play store."; + processFile("mediumRepo.xml"); + handlerTestSuite(expectedRepo, 15, 36); + assertEquals(expectedRepo.maxage, 60); + assertEquals(expectedRepo.version, 12); + checkIncludedApps(new String[] { "info.guardianproject.cacert", "info.guardianproject.otr.app.im", "info.guardianproject.soundrecorder", @@ -99,18 +127,19 @@ public class RepoXMLHandlerTest extends AndroidTestCase { } public void testLargeRepo() { - repo.name = "F-Droid"; - repo.pubkey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef"; - repo.description = "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time."; - RepoXMLHandler handler = getFromFile(repo, "largeRepo.xml"); - handlerTestSuite(repo, handler, 1211, 2381); - assertEquals(handler.getMaxAge(), 14); - assertEquals(handler.getVersion(), 12); + Repo expectedRepo = new Repo(); + expectedRepo.name = "F-Droid"; + expectedRepo.pubkey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef"; + expectedRepo.description = "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time."; + processFile("largeRepo.xml"); + handlerTestSuite(expectedRepo, 1211, 2381); + assertEquals("Repo max age", 14, expectedRepo.maxage); + assertEquals("Repo version", 12, expectedRepo.version); /* * generated using: sed 's, apps, String[] packageNames) { - assertNotNull(apps); + private void checkIncludedApps(String[] packageNames) { + assertNotNull(actualApps); assertNotNull(packageNames); - assertEquals(apps.size(), packageNames.length); + assertEquals(actualApps.size(), packageNames.length); for (String id : packageNames) { boolean thisAppMissing = true; - for (App app : apps) { - if (TextUtils.equals(app.id, id)) + for (App app : actualApps) { + if (TextUtils.equals(app.id, id)) { thisAppMissing = false; + break; + } } assertFalse(thisAppMissing); } } - private void handlerTestSuite(Repo repo, RepoXMLHandler handler, int appCount, int apkCount) { - assertNotNull(handler); - assertFalse(TextUtils.isEmpty(handler.getSigningCertFromIndexXml())); - assertEquals(repo.pubkey.length(), handler.getSigningCertFromIndexXml().length()); - assertEquals(repo.pubkey, handler.getSigningCertFromIndexXml()); - assertFalse(fakePubkey.equals(handler.getSigningCertFromIndexXml())); + private void handlerTestSuite(Repo expectedRepo, int appCount, int apkCount) { + assertFalse(TextUtils.isEmpty(actualRepo.pubkey)); + assertEquals(expectedRepo.pubkey.length(), actualRepo.pubkey.length()); + assertEquals(expectedRepo.pubkey, actualRepo.pubkey); + assertFalse(FAKE_PUBKEY.equals(actualRepo.pubkey)); - assertFalse(TextUtils.isEmpty(handler.getName())); - assertEquals(repo.name.length(), handler.getName().length()); - assertEquals(repo.name, handler.getName()); + assertFalse(TextUtils.isEmpty(actualRepo.name)); + assertEquals(expectedRepo.name.length(), actualRepo.name.length()); + assertEquals(expectedRepo.name, actualRepo.name); - assertFalse(TextUtils.isEmpty(handler.getDescription())); - assertEquals(repo.description.length(), handler.getDescription().length()); - assertEquals(repo.description, handler.getDescription()); + assertFalse(TextUtils.isEmpty(actualRepo.description)); + assertEquals(expectedRepo.description.length(), actualRepo.description.length()); + assertEquals(expectedRepo.description, actualRepo.description); - List apps = handler.getApps(); - assertNotNull(apps); - assertEquals(apps.size(), appCount); + assertNotNull(actualApps); + assertEquals(actualApps.size(), appCount); - List apks = handler.getApks(); + List apks = actualApks; assertNotNull(apks); assertEquals(apks.size(), apkCount); } - private RepoXMLHandler getFromFile(Repo repo, String indexFilename) { + private static class MockRepo extends Repo { + @Override + public long getId() { + return 10000; + } + } + + private RepoXMLHandler processFile(String indexFilename) { SAXParser parser; try { parser = SAXParserFactory.newInstance().newSAXParser(); XMLReader reader = parser.getXMLReader(); - RepoXMLHandler handler = new RepoXMLHandler(repo); + RepoXMLHandler handler = new RepoXMLHandler(new MockRepo(), indexReceiver); reader.setContentHandler(handler); String resName = "assets/" + indexFilename; Log.i(TAG, "test file: " + getClass().getClassLoader().getResource(resName)); From 9a2d39027965bdd31285e8c484e0523eb41b58bf Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Sat, 5 Sep 2015 10:12:12 +1000 Subject: [PATCH 02/17] WIP: Do repo update database work in temp table for apks, then copy at end. At the start of a repo update, it will create a copy of the apk table. Throughout the update, it will query the original apk table for info. All inserts and updates happen to the temp table. After the repo has been verified as trusted, the original apk table is emptied, and all apks are copied from the temp table to the real one. I realise that the work done to query the apk table for info during the update could happen against the temp table, but it was not neccesary to move all of the queries that are required for this task to the temp apk provider. --- F-Droid/AndroidManifest.xml | 5 + .../src/org/fdroid/fdroid/RepoUpdater.java | 23 ++- .../org/fdroid/fdroid/data/ApkProvider.java | 8 +- .../fdroid/fdroid/data/TempApkProvider.java | 139 ++++++++++++++++++ 4 files changed, 164 insertions(+), 11 deletions(-) create mode 100644 F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java diff --git a/F-Droid/AndroidManifest.xml b/F-Droid/AndroidManifest.xml index 25a5f24ac..3fffd013c 100644 --- a/F-Droid/AndroidManifest.xml +++ b/F-Droid/AndroidManifest.xml @@ -83,6 +83,11 @@ android:name="org.fdroid.fdroid.data.ApkProvider" android:exported="false"/> + + Does not do any checks to see if the apk already exists or not. */ private ContentProviderOperation updateExistingApk(final Apk apk) { - Uri uri = ApkProvider.getContentUri(apk); + Uri uri = TempApkProvider.getApkUri(apk); ContentValues values = apk.toContentValues(); return ContentProviderOperation.newUpdate(uri).withValues(values).build(); } @@ -336,7 +337,7 @@ public class RepoUpdater { */ private ContentProviderOperation insertNewApk(final Apk apk) { ContentValues values = apk.toContentValues(); - Uri uri = ApkProvider.getContentUri(); + Uri uri = TempApkProvider.getContentUri(); return ContentProviderOperation.newInsert(uri).withValues(values).build(); } @@ -376,7 +377,7 @@ public class RepoUpdater { // TODO: Deal with more than MAX_QUERY_PARAMS... if (toDelete.size() > 0) { - Uri uri = ApkProvider.getContentUriForApks(repo, toDelete); + Uri uri = TempApkProvider.getApksUri(repo, toDelete); return ContentProviderOperation.newDelete(uri).build(); } else { return null; @@ -409,6 +410,13 @@ public class RepoUpdater { if (downloadedFile == null || !downloadedFile.exists()) throw new UpdateException(repo, downloadedFile + " does not exist!"); + // This is where we will store all of the metadata before commiting at the + // end of the process. This is due to the fact that we can't verify the cert + // the index was signed with until we've finished reading it - and we don't + // want to put stuff in the real database until we are sure it is from a + // trusted source. + TempApkProvider.Helper.init(context); + // Due to a bug in Android 5.0 Lollipop, the inclusion of spongycastle causes // breakage when verifying the signature of the downloaded .jar. For more // details, check out https://gitlab.com/fdroid/fdroidclient/issues/111. @@ -419,9 +427,6 @@ public class RepoUpdater { indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry), progressListener, repo, (int) indexEntry.getSize()); - /* JarEntry can only read certificates after the file represented by that JarEntry - * has been read completely, so verification cannot run until now... */ - // Process the index... final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); final XMLReader reader = parser.getXMLReader(); @@ -430,7 +435,11 @@ public class RepoUpdater { reader.parse(new InputSource(indexInputStream)); signingCertFromJar = getSigningCertFromJar(indexEntry); + // JarEntry can only read certificates after the file represented by that JarEntry + // has been read completely, so verification cannot run until now... assertSigningCertFromXmlCorrect(); + + TempApkProvider.Helper.commit(context); RepoProvider.Helper.update(context, repo, repoDetailsToSave); } catch (SAXException | ParserConfigurationException | IOException e) { diff --git a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java index 495689c17..2a2b4bed1 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java @@ -227,15 +227,15 @@ public class ApkProvider extends FDroidProvider { private static final int CODE_REPO = CODE_APP + 1; private static final int CODE_APKS = CODE_REPO + 1; private static final int CODE_REPO_APPS = CODE_APKS + 1; - private static final int CODE_REPO_APK = CODE_REPO_APPS + 1; + protected static final int CODE_REPO_APK = CODE_REPO_APPS + 1; private static final String PROVIDER_NAME = "ApkProvider"; - private static final String PATH_APK = "apk"; + protected static final String PATH_APK = "apk"; private static final String PATH_APKS = "apks"; private static final String PATH_APP = "app"; private static final String PATH_REPO = "repo"; private static final String PATH_REPO_APPS = "repo-apps"; - private static final String PATH_REPO_APK = "repo-apk"; + protected static final String PATH_REPO_APK = "repo-apk"; private static final UriMatcher matcher = new UriMatcher(-1); @@ -322,7 +322,7 @@ public class ApkProvider extends FDroidProvider { .build(); } - private static String buildApkString(List apks) { + protected static String buildApkString(List apks) { StringBuilder builder = new StringBuilder(); for (int i = 0; i < apks.size(); i++) { if (i != 0) { diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java new file mode 100644 index 000000000..a8bc1dbbc --- /dev/null +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -0,0 +1,139 @@ +package org.fdroid.fdroid.data; + +import android.content.ContentResolver; +import android.content.ContentValues; +import android.content.Context; +import android.content.UriMatcher; +import android.database.Cursor; +import android.net.Uri; +import android.provider.BaseColumns; +import android.util.Log; + +import org.fdroid.fdroid.Utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class does all of its operations in a temporary sqlite table. + */ +public class TempApkProvider extends ApkProvider { + + private static final String TAG = "TempApkProvider"; + + private static final String PROVIDER_NAME = "TempApkProvider"; + + private static final String PATH_INIT = "init"; + private static final String PATH_COMMIT = "commit"; + private static final String PATH_ROLLBACK = "rollback"; + + private static final int CODE_INIT = 10000; + private static final int CODE_COMMIT = CODE_INIT + 1; + private static final int CODE_ROLLBACK = CODE_INIT + 2; + + private static final UriMatcher matcher = new UriMatcher(-1); + + static { + matcher.addURI(getAuthority(), PATH_INIT, CODE_INIT); + matcher.addURI(getAuthority(), PATH_COMMIT, CODE_COMMIT); + matcher.addURI(getAuthority(), PATH_ROLLBACK, CODE_ROLLBACK); + matcher.addURI(getAuthority(), PATH_APK + "/#/*", CODE_SINGLE); + matcher.addURI(getAuthority(), PATH_REPO_APK + "/#/*", CODE_REPO_APK); + } + + @Override + protected String getTableName() { + return "temp_" + super.getTableName(); + } + + public static String getAuthority() { + return AUTHORITY + "." + PROVIDER_NAME; + } + + public static Uri getContentUri() { + return Uri.parse("content://" + getAuthority()); + } + + public static Uri getApkUri(Apk apk) { + return getContentUri() + .buildUpon() + .appendPath(PATH_APK) + .appendPath(Integer.toString(apk.vercode)) + .appendPath(apk.id) + .build(); + } + + public static Uri getApksUri(Repo repo, List apks) { + return getContentUri() + .buildUpon() + .appendPath(PATH_REPO_APK) + .appendPath(Long.toString(repo.id)) + .appendPath(buildApkString(apks)) + .build(); + } + + public static class Helper { + + /** + * Deletes the old temporary table (if it exists). Then creates a new temporary apk provider + * table and populates it with all the data from the real apk provider table. + */ + public static void init(Context context) { + Uri uri = Uri.withAppendedPath(getContentUri(), PATH_INIT); + context.getContentResolver().insert(uri, new ContentValues()); + } + + /** + * Saves data from the temp table to the apk table, by removing _EVERYTHING_ from the real + * apk table and inserting all of the records from here. The temporary table is then removed. + */ + public static void commit(Context context) { + Uri uri = Uri.withAppendedPath(getContentUri(), PATH_COMMIT); + context.getContentResolver().insert(uri, new ContentValues()); + } + + /** + * Not sure that this is strictly necessary, but this will remove the temp table. + */ + public static void rollback(Context context) { + Uri uri = Uri.withAppendedPath(getContentUri(), PATH_ROLLBACK); + context.getContentResolver().insert(uri, new ContentValues()); + } + + } + + @Override + public Uri insert(Uri uri, ContentValues values) { + int code = matcher.match(uri); + + if (code == CODE_INIT) { + initTable(); + return null; + } else if (code == CODE_COMMIT) { + commitTable(); + return null; + } else if (code == CODE_ROLLBACK) { + removeTable(); + return null; + } else { + return super.insert(uri, values); + } + } + + private void initTable() { + removeTable(); + write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APK); + } + + private void commitTable() { + Log.d(TAG, "Deleting all apks from " + DBHelper.TABLE_APK + " so they can be copied from " + getTableName()); + write().execSQL("DELETE FROM " + DBHelper.TABLE_APK); + write().execSQL("INSERT INTO " + DBHelper.TABLE_APK + " SELECT * FROM " + getTableName()); + } + + private void removeTable() { + write().execSQL("DROP TABLE IF EXISTS " + getTableName()); + } +} From cc0adcc5ad6e103acbcd57a85978416701140cfd Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Sat, 5 Sep 2015 10:39:31 +1000 Subject: [PATCH 03/17] Save app details to temp table, then flush after update verified. --- F-Droid/AndroidManifest.xml | 5 + .../src/org/fdroid/fdroid/RepoUpdater.java | 25 +++-- .../fdroid/fdroid/data/TempApkProvider.java | 15 +-- .../fdroid/fdroid/data/TempAppProvider.java | 103 ++++++++++++++++++ 4 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java diff --git a/F-Droid/AndroidManifest.xml b/F-Droid/AndroidManifest.xml index 3fffd013c..b9c86c37b 100644 --- a/F-Droid/AndroidManifest.xml +++ b/F-Droid/AndroidManifest.xml @@ -88,6 +88,11 @@ android:name="org.fdroid.fdroid.data.TempApkProvider" android:exported="false"/> + + 0 || appsToSave.size() > 0) { + Log.d(TAG, "Flushing details of up to " + MAX_APP_BUFFER + " apps and their packages to the database."); + flushAppsToDbInBatch(); + flushApksToDbInBatch(); + apksToSave.clear(); + appsToSave.clear(); + } } private void flushApksToDbInBatch() throws UpdateException { @@ -236,7 +239,7 @@ public class RepoUpdater { ArrayList appOperations = insertOrUpdateApps(appsToSave); try { - context.getContentResolver().applyBatch(AppProvider.getAuthority(), appOperations); + context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations); } catch (RemoteException|OperationApplicationException e) { Log.e(TAG, "Error updating apps", e); throw new UpdateException(repo, "Error updating apps: " + e.getMessage(), e); @@ -290,7 +293,7 @@ public class RepoUpdater { * Does not do any checks to see if the app already exists or not. */ private ContentProviderOperation updateExistingApp(App app) { - Uri uri = AppProvider.getContentUri(app); + Uri uri = TempAppProvider.getAppUri(app); ContentValues values = app.toContentValues(); for (final String toIgnore : APP_FIELDS_TO_IGNORE) { if (values.containsKey(toIgnore)) { @@ -306,7 +309,7 @@ public class RepoUpdater { */ private ContentProviderOperation insertNewApp(App app) { ContentValues values = app.toContentValues(); - Uri uri = AppProvider.getContentUri(); + Uri uri = TempAppProvider.getContentUri(); return ContentProviderOperation.newInsert(uri).withValues(values).build(); } @@ -415,6 +418,7 @@ public class RepoUpdater { // the index was signed with until we've finished reading it - and we don't // want to put stuff in the real database until we are sure it is from a // trusted source. + TempAppProvider.Helper.init(context); TempApkProvider.Helper.init(context); // Due to a bug in Android 5.0 Lollipop, the inclusion of spongycastle causes @@ -433,12 +437,17 @@ public class RepoUpdater { final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo, createIndexReceiver()); reader.setContentHandler(repoXMLHandler); reader.parse(new InputSource(indexInputStream)); + + flushBufferToDb(); + signingCertFromJar = getSigningCertFromJar(indexEntry); // JarEntry can only read certificates after the file represented by that JarEntry // has been read completely, so verification cannot run until now... assertSigningCertFromXmlCorrect(); + Log.i(TAG, "Repo signature verified, saving app metadata to database."); + TempAppProvider.Helper.commit(context); TempApkProvider.Helper.commit(context); RepoProvider.Helper.update(context, repo, repoDetailsToSave); diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java index a8bc1dbbc..e2806fd49 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -31,14 +31,12 @@ public class TempApkProvider extends ApkProvider { private static final int CODE_INIT = 10000; private static final int CODE_COMMIT = CODE_INIT + 1; - private static final int CODE_ROLLBACK = CODE_INIT + 2; private static final UriMatcher matcher = new UriMatcher(-1); static { matcher.addURI(getAuthority(), PATH_INIT, CODE_INIT); matcher.addURI(getAuthority(), PATH_COMMIT, CODE_COMMIT); - matcher.addURI(getAuthority(), PATH_ROLLBACK, CODE_ROLLBACK); matcher.addURI(getAuthority(), PATH_APK + "/#/*", CODE_SINGLE); matcher.addURI(getAuthority(), PATH_REPO_APK + "/#/*", CODE_REPO_APK); } @@ -94,14 +92,6 @@ public class TempApkProvider extends ApkProvider { context.getContentResolver().insert(uri, new ContentValues()); } - /** - * Not sure that this is strictly necessary, but this will remove the temp table. - */ - public static void rollback(Context context) { - Uri uri = Uri.withAppendedPath(getContentUri(), PATH_ROLLBACK); - context.getContentResolver().insert(uri, new ContentValues()); - } - } @Override @@ -114,16 +104,13 @@ public class TempApkProvider extends ApkProvider { } else if (code == CODE_COMMIT) { commitTable(); return null; - } else if (code == CODE_ROLLBACK) { - removeTable(); - return null; } else { return super.insert(uri, values); } } private void initTable() { - removeTable(); + write().execSQL("DROP TABLE IF EXISTS " + getTableName()); write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APK); } diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java new file mode 100644 index 000000000..b4c9fb7eb --- /dev/null +++ b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java @@ -0,0 +1,103 @@ +package org.fdroid.fdroid.data; + +import android.content.ContentValues; +import android.content.Context; +import android.content.UriMatcher; +import android.net.Uri; +import android.util.Log; + +import java.util.List; + +/** + * This class does all of its operations in a temporary sqlite table. + */ +public class TempAppProvider extends AppProvider { + + private static final String TAG = "TempAppProvider"; + + private static final String PROVIDER_NAME = "TempAppProvider"; + + private static final String PATH_INIT = "init"; + private static final String PATH_COMMIT = "commit"; + private static final String PATH_ROLLBACK = "rollback"; + + private static final int CODE_INIT = 10000; + private static final int CODE_COMMIT = CODE_INIT + 1; + + private static final UriMatcher matcher = new UriMatcher(-1); + + static { + matcher.addURI(getAuthority(), PATH_INIT, CODE_INIT); + matcher.addURI(getAuthority(), PATH_COMMIT, CODE_COMMIT); + matcher.addURI(getAuthority(), "*", CODE_SINGLE); + } + + @Override + protected String getTableName() { + return "temp_" + super.getTableName(); + } + + public static String getAuthority() { + return AUTHORITY + "." + PROVIDER_NAME; + } + + public static Uri getContentUri() { + return Uri.parse("content://" + getAuthority()); + } + + public static Uri getAppUri(App app) { + return Uri.withAppendedPath(getContentUri(), app.id); + } + + public static class Helper { + + /** + * Deletes the old temporary table (if it exists). Then creates a new temporary apk provider + * table and populates it with all the data from the real apk provider table. + */ + public static void init(Context context) { + Uri uri = Uri.withAppendedPath(getContentUri(), PATH_INIT); + context.getContentResolver().insert(uri, new ContentValues()); + } + + /** + * Saves data from the temp table to the apk table, by removing _EVERYTHING_ from the real + * apk table and inserting all of the records from here. The temporary table is then removed. + */ + public static void commit(Context context) { + Uri uri = Uri.withAppendedPath(getContentUri(), PATH_COMMIT); + context.getContentResolver().insert(uri, new ContentValues()); + } + + } + + @Override + public Uri insert(Uri uri, ContentValues values) { + int code = matcher.match(uri); + + if (code == CODE_INIT) { + initTable(); + return null; + } else if (code == CODE_COMMIT) { + commitTable(); + return null; + } else { + return super.insert(uri, values); + } + } + + private void initTable() { + write().execSQL("DROP TABLE IF EXISTS " + getTableName()); + write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APP); + } + + private void commitTable() { + Log.d(TAG, "Deleting all apks from " + DBHelper.TABLE_APP + " so they can be copied from " + getTableName()); + write().execSQL("DELETE FROM " + DBHelper.TABLE_APP); + write().execSQL("INSERT INTO " + DBHelper.TABLE_APP + " SELECT * FROM " + getTableName()); + } + + private void removeTable() { + + } +} From b34853a7764295692314528dc772d1c189762736 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Sat, 5 Sep 2015 11:12:24 +1000 Subject: [PATCH 04/17] Cleanup before CR. --- F-Droid/src/org/fdroid/fdroid/RepoUpdater.java | 14 +++++++------- F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java | 11 ----------- .../org/fdroid/fdroid/data/TempApkProvider.java | 13 ------------- .../org/fdroid/fdroid/data/TempAppProvider.java | 7 ------- 4 files changed, 7 insertions(+), 38 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 10f7ff526..78b27ddff 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -184,10 +184,12 @@ public class RepoUpdater { /** * My crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get: - * * 25 = { 39, 35 } seconds - * * 50 = { 36, 30 } seconds - * * 100 = { 33, 27 } seconds - * * 200 = { 30, 33 } seconds + * * 25 = 37 seconds + * * 50 = 33 seconds + * * 100 = 30 seconds + * * 200 = 32 seconds + * Raising this means more memory consumption, so we'd like it to be low, but not + * so low that it takes too long. */ private static final int MAX_APP_BUFFER = 50; @@ -241,8 +243,7 @@ public class RepoUpdater { try { context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations); } catch (RemoteException|OperationApplicationException e) { - Log.e(TAG, "Error updating apps", e); - throw new UpdateException(repo, "Error updating apps: " + e.getMessage(), e); + throw new UpdateException(repo, "An internal error occured while updating the database", e); } } @@ -378,7 +379,6 @@ public class RepoUpdater { } } - // TODO: Deal with more than MAX_QUERY_PARAMS... if (toDelete.size() > 0) { Uri uri = TempApkProvider.getApksUri(repo, toDelete); return ContentProviderOperation.newDelete(uri).build(); diff --git a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java index c0dd841f0..8f336863a 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java @@ -34,17 +34,6 @@ import java.util.List; /** * Parses the index.xml into Java data structures. - * - * For streaming apks from an index file, it is helpful if the index has the tag before - * any tags. This means that apps and apks can be saved instantly by the RepoUpdater, - * without having to buffer them at all, saving memory. The XML spec doesn't mandate order like - * this, though it is almost always a fair assumption: - * - * http://www.ibm.com/developerworks/library/x-eleord/index.html - * - * This is doubly so, as repo indices are likely from fdroidserver, which will output everybodys - * repo the same way. Having said that, this also should not be _forced_ upon people, but we can - * at least consider rejecting malformed indexes. */ public class RepoXMLHandler extends DefaultHandler { diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java index e2806fd49..8a6f8e20e 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -1,20 +1,12 @@ package org.fdroid.fdroid.data; -import android.content.ContentResolver; import android.content.ContentValues; import android.content.Context; import android.content.UriMatcher; -import android.database.Cursor; import android.net.Uri; -import android.provider.BaseColumns; import android.util.Log; -import org.fdroid.fdroid.Utils; - -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * This class does all of its operations in a temporary sqlite table. @@ -27,7 +19,6 @@ public class TempApkProvider extends ApkProvider { private static final String PATH_INIT = "init"; private static final String PATH_COMMIT = "commit"; - private static final String PATH_ROLLBACK = "rollback"; private static final int CODE_INIT = 10000; private static final int CODE_COMMIT = CODE_INIT + 1; @@ -119,8 +110,4 @@ public class TempApkProvider extends ApkProvider { write().execSQL("DELETE FROM " + DBHelper.TABLE_APK); write().execSQL("INSERT INTO " + DBHelper.TABLE_APK + " SELECT * FROM " + getTableName()); } - - private void removeTable() { - write().execSQL("DROP TABLE IF EXISTS " + getTableName()); - } } diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java index b4c9fb7eb..e469191f8 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java @@ -6,8 +6,6 @@ import android.content.UriMatcher; import android.net.Uri; import android.util.Log; -import java.util.List; - /** * This class does all of its operations in a temporary sqlite table. */ @@ -19,7 +17,6 @@ public class TempAppProvider extends AppProvider { private static final String PATH_INIT = "init"; private static final String PATH_COMMIT = "commit"; - private static final String PATH_ROLLBACK = "rollback"; private static final int CODE_INIT = 10000; private static final int CODE_COMMIT = CODE_INIT + 1; @@ -96,8 +93,4 @@ public class TempAppProvider extends AppProvider { write().execSQL("DELETE FROM " + DBHelper.TABLE_APP); write().execSQL("INSERT INTO " + DBHelper.TABLE_APP + " SELECT * FROM " + getTableName()); } - - private void removeTable() { - - } } From 1d951e7689dd8674ba7311824e426d26d02a3bdd Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Sun, 6 Sep 2015 08:15:19 +1000 Subject: [PATCH 05/17] Fixed repo updater tests. Fix to temp app/apk providers. The repo xml handler now has a different mechanism for returning data about the parsed xml file. This is done via a callback, rather than storing the data in member variables. The tests now deal with this correctly. The update/delete operations of the TempAp[pk]Provider's didn't work, so that has now been fixed. --- .../src/org/fdroid/fdroid/RepoUpdater.java | 51 ++++--- .../src/org/fdroid/fdroid/RepoXMLHandler.java | 97 ++++++------ .../src/org/fdroid/fdroid/UpdateService.java | 81 +++++----- .../org/fdroid/fdroid/data/ApkProvider.java | 21 +-- .../org/fdroid/fdroid/data/AppProvider.java | 57 +++---- .../fdroid/fdroid/data/TempApkProvider.java | 36 ++++- .../fdroid/fdroid/data/TempAppProvider.java | 20 +++ .../fdroid/fdroid/MultiRepoUpdaterTest.java | 4 + .../org/fdroid/fdroid/RepoUpdaterTest.java | 1 - .../org/fdroid/fdroid/RepoXMLHandlerTest.java | 142 +++++++++--------- 10 files changed, 286 insertions(+), 224 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 78b27ddff..493b1cdb1 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -46,7 +46,6 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; /** - * * Responsible for updating an individual repository. This will: * * Download the index.jar * * Verify that it is signed correctly and by the correct certificate @@ -74,19 +73,23 @@ public class RepoUpdater { * is essentially completely transient, and can be nuked at any time. */ private static final String[] APP_FIELDS_TO_IGNORE = { - AppProvider.DataColumns.IGNORE_ALLUPDATES, - AppProvider.DataColumns.IGNORE_THISUPDATE + AppProvider.DataColumns.IGNORE_ALLUPDATES, + AppProvider.DataColumns.IGNORE_THISUPDATE, }; - @NonNull protected final Context context; - @NonNull protected final Repo repo; - protected boolean hasChanged = false; - @Nullable protected ProgressListener progressListener; + @NonNull + protected final Context context; + @NonNull + protected final Repo repo; + protected boolean hasChanged; + @Nullable + protected ProgressListener progressListener; private String cacheTag; private X509Certificate signingCertFromJar; /** * Updates an app repo as read out of the database into a {@link Repo} instance. + * * @param repo A {@link Repo} read out of the local database */ public RepoUpdater(@NonNull Context context, @NonNull Repo repo) { @@ -160,8 +163,8 @@ public class RepoUpdater { } } - private ContentValues repoDetailsToSave = null; - private String signingCertFromIndexXml = null; + private ContentValues repoDetailsToSave; + private String signingCertFromIndexXml; private RepoXMLHandler.IndexReceiver createIndexReceiver() { return new RepoXMLHandler.IndexReceiver() { @@ -184,12 +187,12 @@ public class RepoUpdater { /** * My crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get: - * * 25 = 37 seconds - * * 50 = 33 seconds - * * 100 = 30 seconds - * * 200 = 32 seconds - * Raising this means more memory consumption, so we'd like it to be low, but not - * so low that it takes too long. + * * 25 = 37 seconds + * * 50 = 33 seconds + * * 100 = 30 seconds + * * 200 = 32 seconds + * Raising this means more memory consumption, so we'd like it to be low, but not + * so low that it takes too long. */ private static final int MAX_APP_BUFFER = 50; @@ -242,7 +245,7 @@ public class RepoUpdater { try { context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations); - } catch (RemoteException|OperationApplicationException e) { + } catch (RemoteException | OperationApplicationException e) { throw new UpdateException(repo, "An internal error occured while updating the database", e); } } @@ -320,7 +323,7 @@ public class RepoUpdater { * array. */ private boolean isAppInDatabase(App app) { - String[] fields = { AppProvider.DataColumns.APP_ID }; + String[] fields = {AppProvider.DataColumns.APP_ID}; App found = AppProvider.Helper.findById(context.getContentResolver(), app.id, fields); return found != null; } @@ -429,7 +432,7 @@ public class RepoUpdater { JarFile jarFile = new JarFile(downloadedFile, true); JarEntry indexEntry = (JarEntry) jarFile.getEntry("index.xml"); indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry), - progressListener, repo, (int) indexEntry.getSize()); + progressListener, repo, (int) indexEntry.getSize()); // Process the index... final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); @@ -558,7 +561,7 @@ public class RepoUpdater { * check that the signing certificate in the jar matches that fingerprint. */ private void verifyAndStoreTOFUCerts(String certFromIndexXml, X509Certificate rawCertFromJar) - throws SigningException { + throws SigningException { if (repo.pubkey != null) return; // there is a repo.pubkey already, nothing to TOFU @@ -570,7 +573,7 @@ public class RepoUpdater { String fingerprintFromIndexXml = Utils.calcFingerprint(certFromIndexXml); String fingerprintFromJar = Utils.calcFingerprint(rawCertFromJar); if (!repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml) - || !repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) { + || !repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) { throw new SigningException(repo, "Supplied certificate fingerprint does not match!"); } } // else - no info to check things are valid, so just Trust On First Use @@ -600,14 +603,14 @@ public class RepoUpdater { // repo and repo.pubkey must be pre-loaded from the database if (TextUtils.isEmpty(repo.pubkey) - || TextUtils.isEmpty(certFromJar) - || TextUtils.isEmpty(certFromIndexXml)) + || TextUtils.isEmpty(certFromJar) + || TextUtils.isEmpty(certFromIndexXml)) throw new SigningException(repo, "A empty repo or signing certificate is invalid!"); // though its called repo.pubkey, its actually a X509 certificate if (repo.pubkey.equals(certFromJar) - && repo.pubkey.equals(certFromIndexXml) - && certFromIndexXml.equals(certFromJar)) { + && repo.pubkey.equals(certFromIndexXml) + && certFromIndexXml.equals(certFromJar)) { return; // we have a match! } throw new SigningException(repo, "Signing certificate does not match!"); diff --git a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java index 8f336863a..72f330006 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java @@ -45,12 +45,12 @@ public class RepoXMLHandler extends DefaultHandler { private App curapp; private Apk curapk; - private String currentApkHashType = null; + private String currentApkHashType; // After processing the XML, these will be -1 if the index didn't specify // them - otherwise it will be the value specified. private int repoMaxAge = -1; - private int repoVersion = 0; + private int repoVersion; private String repoDescription; private String repoName; @@ -61,6 +61,7 @@ public class RepoXMLHandler extends DefaultHandler { interface IndexReceiver { void receiveRepo(String name, String description, String signingCert, int maxage, int version); + void receiveApp(App app, List packages); } @@ -78,7 +79,7 @@ public class RepoXMLHandler extends DefaultHandler { @Override public void endElement(String uri, String localName, String qName) - throws SAXException { + throws SAXException { if ("application".equals(localName) && curapp != null) { onApplicationParsed(); @@ -94,53 +95,53 @@ public class RepoXMLHandler extends DefaultHandler { final String str = curchars.toString().trim(); if (curapk != null) { switch (localName) { - case "version": - curapk.version = str; - break; - case "versioncode": - curapk.vercode = Utils.parseInt(str, -1); - break; - case "size": - curapk.size = Utils.parseInt(str, 0); - break; - case "hash": - if (currentApkHashType == null || currentApkHashType.equals("md5")) { - if (curapk.hash == null) { + case "version": + curapk.version = str; + break; + case "versioncode": + curapk.vercode = Utils.parseInt(str, -1); + break; + case "size": + curapk.size = Utils.parseInt(str, 0); + break; + case "hash": + if (currentApkHashType == null || "md5".equals(currentApkHashType)) { + if (curapk.hash == null) { + curapk.hash = str; + curapk.hashType = "SHA-256"; + } + } else if ("sha256".equals(currentApkHashType)) { curapk.hash = str; curapk.hashType = "SHA-256"; } - } else if (currentApkHashType.equals("sha256")) { - curapk.hash = str; - curapk.hashType = "SHA-256"; - } - break; - case "sig": - curapk.sig = str; - break; - case "srcname": - curapk.srcname = str; - break; - case "apkname": - curapk.apkName = str; - break; - case "sdkver": - curapk.minSdkVersion = Utils.parseInt(str, 0); - break; - case "maxsdkver": - curapk.maxSdkVersion = Utils.parseInt(str, 0); - break; - case "added": - curapk.added = Utils.parseDate(str, null); - break; - case "permissions": - curapk.permissions = Utils.CommaSeparatedList.make(str); - break; - case "features": - curapk.features = Utils.CommaSeparatedList.make(str); - break; - case "nativecode": - curapk.nativecode = Utils.CommaSeparatedList.make(str); - break; + break; + case "sig": + curapk.sig = str; + break; + case "srcname": + curapk.srcname = str; + break; + case "apkname": + curapk.apkName = str; + break; + case "sdkver": + curapk.minSdkVersion = Utils.parseInt(str, 0); + break; + case "maxsdkver": + curapk.maxSdkVersion = Utils.parseInt(str, 0); + break; + case "added": + curapk.added = Utils.parseDate(str, null); + break; + case "permissions": + curapk.permissions = Utils.CommaSeparatedList.make(str); + break; + case "features": + curapk.features = Utils.CommaSeparatedList.make(str); + break; + case "nativecode": + curapk.nativecode = Utils.CommaSeparatedList.make(str); + break; } } else if (curapp != null) { switch (localName) { @@ -237,7 +238,7 @@ public class RepoXMLHandler extends DefaultHandler { @Override public void startElement(String uri, String localName, String qName, - Attributes attributes) throws SAXException { + Attributes attributes) throws SAXException { super.startElement(uri, localName, qName, attributes); if ("repo".equals(localName)) { diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index 3105b5851..b618a2455 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -101,7 +101,7 @@ public class UpdateService extends IntentService implements ProgressListener { public static void schedule(Context ctx) { SharedPreferences prefs = PreferenceManager - .getDefaultSharedPreferences(ctx); + .getDefaultSharedPreferences(ctx); String sint = prefs.getString(Preferences.PREF_UPD_INTERVAL, "0"); int interval = Integer.parseInt(sint); @@ -109,12 +109,12 @@ public class UpdateService extends IntentService implements ProgressListener { PendingIntent pending = PendingIntent.getService(ctx, 0, intent, 0); AlarmManager alarm = (AlarmManager) ctx - .getSystemService(Context.ALARM_SERVICE); + .getSystemService(Context.ALARM_SERVICE); alarm.cancel(pending); if (interval > 0) { alarm.setInexactRepeating(AlarmManager.ELAPSED_REALTIME, - SystemClock.elapsedRealtime() + 5000, - AlarmManager.INTERVAL_HOUR, pending); + SystemClock.elapsedRealtime() + 5000, + AlarmManager.INTERVAL_HOUR, pending); Utils.debugLog(TAG, "Update scheduler alarm set"); } else { Utils.debugLog(TAG, "Update scheduler alarm not set"); @@ -128,18 +128,18 @@ public class UpdateService extends IntentService implements ProgressListener { localBroadcastManager = LocalBroadcastManager.getInstance(this); localBroadcastManager.registerReceiver(downloadProgressReceiver, - new IntentFilter(Downloader.LOCAL_ACTION_PROGRESS)); + new IntentFilter(Downloader.LOCAL_ACTION_PROGRESS)); localBroadcastManager.registerReceiver(updateStatusReceiver, - new IntentFilter(LOCAL_ACTION_STATUS)); + new IntentFilter(LOCAL_ACTION_STATUS)); notificationManager = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE); notificationBuilder = new NotificationCompat.Builder(this) - .setSmallIcon(R.drawable.ic_refresh_white) - .setOngoing(true) - .setCategory(NotificationCompat.CATEGORY_SERVICE) - .setContentTitle(getString(R.string.update_notification_title)); - + .setSmallIcon(R.drawable.ic_refresh_white) + .setOngoing(true) + .setCategory(NotificationCompat.CATEGORY_SERVICE) + .setContentTitle(getString(R.string.update_notification_title)); + // Android docs are a little sketchy, however it seems that Gingerbread is the last // sdk that made a content intent mandatory: // @@ -147,7 +147,7 @@ public class UpdateService extends IntentService implements ProgressListener { // if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.GINGERBREAD_MR1) { Intent pendingIntent = new Intent(this, FDroid.class); - pendingIntent.addFlags (Intent.FLAG_ACTIVITY_NEW_TASK); + pendingIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); notificationBuilder.setContentIntent(PendingIntent.getActivity(this, 0, pendingIntent, PendingIntent.FLAG_UPDATE_CURRENT)); } @@ -234,7 +234,7 @@ public class UpdateService extends IntentService implements ProgressListener { switch (resultCode) { case STATUS_INFO: notificationBuilder.setContentText(message) - .setCategory(NotificationCompat.CATEGORY_SERVICE); + .setCategory(NotificationCompat.CATEGORY_SERVICE); if (progress != -1) { notificationBuilder.setProgress(100, progress, false); } @@ -243,8 +243,8 @@ public class UpdateService extends IntentService implements ProgressListener { case STATUS_ERROR_GLOBAL: text = context.getString(R.string.global_error_updating_repos, message); notificationBuilder.setContentText(text) - .setCategory(NotificationCompat.CATEGORY_ERROR) - .setSmallIcon(android.R.drawable.ic_dialog_alert); + .setCategory(NotificationCompat.CATEGORY_ERROR) + .setSmallIcon(android.R.drawable.ic_dialog_alert); notificationManager.notify(NOTIFY_ID_UPDATING, notificationBuilder.build()); Toast.makeText(context, text, Toast.LENGTH_LONG).show(); break; @@ -261,8 +261,8 @@ public class UpdateService extends IntentService implements ProgressListener { } text = msgBuilder.toString(); notificationBuilder.setContentText(text) - .setCategory(NotificationCompat.CATEGORY_ERROR) - .setSmallIcon(android.R.drawable.ic_dialog_info); + .setCategory(NotificationCompat.CATEGORY_ERROR) + .setSmallIcon(android.R.drawable.ic_dialog_info); notificationManager.notify(NOTIFY_ID_UPDATING, notificationBuilder.build()); Toast.makeText(context, text, Toast.LENGTH_LONG).show(); break; @@ -271,7 +271,7 @@ public class UpdateService extends IntentService implements ProgressListener { case STATUS_COMPLETE_AND_SAME: text = context.getString(R.string.repos_unchanged); notificationBuilder.setContentText(text) - .setCategory(NotificationCompat.CATEGORY_SERVICE); + .setCategory(NotificationCompat.CATEGORY_SERVICE); notificationManager.notify(NOTIFY_ID_UPDATING, notificationBuilder.build()); break; } @@ -281,10 +281,11 @@ public class UpdateService extends IntentService implements ProgressListener { /** * Check whether it is time to run the scheduled update. * We don't want to run if: - * - The time between scheduled runs is set to zero (though don't know - * when that would occur) - * - Last update was too recent - * - Not on wifi, but the property for "Only auto update on wifi" is set. + * - The time between scheduled runs is set to zero (though don't know + * when that would occur) + * - Last update was too recent + * - Not on wifi, but the property for "Only auto update on wifi" is set. + * * @return True if we are due for a scheduled update. */ private boolean verifyIsTimeForScheduledRun() { @@ -299,7 +300,7 @@ public class UpdateService extends IntentService implements ProgressListener { long elapsed = System.currentTimeMillis() - lastUpdate; if (elapsed < interval * 60 * 60 * 1000) { Log.i(TAG, "Skipping update - done " + elapsed - + "ms ago, interval is " + interval + " hours"); + + "ms ago, interval is " + interval + " hours"); return false; } @@ -319,7 +320,7 @@ public class UpdateService extends IntentService implements ProgressListener { SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); if (activeNetwork.getType() != ConnectivityManager.TYPE_WIFI - && prefs.getBoolean(Preferences.PREF_UPD_WIFI_ONLY, false)) { + && prefs.getBoolean(Preferences.PREF_UPD_WIFI_ONLY, false)) { Log.i(TAG, "Skipping update - wifi not available"); return false; } @@ -432,9 +433,9 @@ public class UpdateService extends IntentService implements ProgressListener { private void performUpdateNotification() { Cursor cursor = getContentResolver().query( - AppProvider.getCanUpdateUri(), - AppProvider.DataColumns.ALL, - null, null, null); + AppProvider.getCanUpdateUri(), + AppProvider.DataColumns.ALL, + null, null, null); if (cursor != null) { if (cursor.getCount() > 0) { showAppUpdatesNotification(cursor); @@ -446,8 +447,8 @@ public class UpdateService extends IntentService implements ProgressListener { private PendingIntent createNotificationIntent() { Intent notifyIntent = new Intent(this, FDroid.class).putExtra(FDroid.EXTRA_TAB_UPDATE, true); TaskStackBuilder stackBuilder = TaskStackBuilder - .create(this).addParentStack(FDroid.class) - .addNextIntent(notifyIntent); + .create(this).addParentStack(FDroid.class) + .addNextIntent(notifyIntent); return stackBuilder.getPendingIntent(0, PendingIntent.FLAG_UPDATE_CURRENT); } @@ -456,8 +457,8 @@ public class UpdateService extends IntentService implements ProgressListener { private NotificationCompat.Style createNotificationBigStyle(Cursor hasUpdates) { final String contentText = hasUpdates.getCount() > 1 - ? getString(R.string.many_updates_available, hasUpdates.getCount()) - : getString(R.string.one_update_available); + ? getString(R.string.many_updates_available, hasUpdates.getCount()) + : getString(R.string.one_update_available); NotificationCompat.InboxStyle inboxStyle = new NotificationCompat.InboxStyle(); inboxStyle.setBigContentTitle(contentText); @@ -482,17 +483,17 @@ public class UpdateService extends IntentService implements ProgressListener { final int icon = Build.VERSION.SDK_INT >= 11 ? R.drawable.ic_stat_notify_updates : R.drawable.ic_launcher; final String contentText = hasUpdates.getCount() > 1 - ? getString(R.string.many_updates_available, hasUpdates.getCount()) - : getString(R.string.one_update_available); + ? getString(R.string.many_updates_available, hasUpdates.getCount()) + : getString(R.string.one_update_available); NotificationCompat.Builder builder = - new NotificationCompat.Builder(this) - .setAutoCancel(true) - .setContentTitle(getString(R.string.fdroid_updates_available)) - .setSmallIcon(icon) - .setContentIntent(createNotificationIntent()) - .setContentText(contentText) - .setStyle(createNotificationBigStyle(hasUpdates)); + new NotificationCompat.Builder(this) + .setAutoCancel(true) + .setContentTitle(getString(R.string.fdroid_updates_available)) + .setSmallIcon(icon) + .setContentIntent(createNotificationIntent()) + .setContentText(contentText) + .setStyle(createNotificationBigStyle(hasUpdates)); notificationManager.notify(NOTIFY_ID_UPDATES_AVAILABLE, builder.build()); } diff --git a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java index 2a2b4bed1..d8be8e343 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java @@ -229,10 +229,10 @@ public class ApkProvider extends FDroidProvider { private static final int CODE_REPO_APPS = CODE_APKS + 1; protected static final int CODE_REPO_APK = CODE_REPO_APPS + 1; - private static final String PROVIDER_NAME = "ApkProvider"; - protected static final String PATH_APK = "apk"; - private static final String PATH_APKS = "apks"; - private static final String PATH_APP = "app"; + private static final String PROVIDER_NAME = "ApkProvider"; + protected static final String PATH_APK = "apk"; + private static final String PATH_APKS = "apks"; + private static final String PATH_APP = "app"; private static final String PATH_REPO = "repo"; private static final String PATH_REPO_APPS = "repo-apps"; protected static final String PATH_REPO_APK = "repo-apk"; @@ -411,7 +411,7 @@ public class ApkProvider extends FDroidProvider { return new QuerySelection(selection, args); } - private QuerySelection queryRepo(long repoId) { + protected QuerySelection queryRepo(long repoId) { final String selection = DataColumns.REPO_ID + " = ? "; final String[] args = {Long.toString(repoId)}; return new QuerySelection(selection, args); @@ -421,7 +421,7 @@ public class ApkProvider extends FDroidProvider { return queryRepo(repoId).add(AppProvider.queryApps(appIds, DataColumns.APK_ID)); } - private QuerySelection queryApks(String apkKeys) { + protected QuerySelection queryApks(String apkKeys) { final String[] apkDetails = apkKeys.split(","); if (apkDetails.length > MAX_APKS_TO_QUERY) { throw new IllegalArgumentException( @@ -473,7 +473,7 @@ public class ApkProvider extends FDroidProvider { List pathSegments = uri.getPathSegments(); query = query.add(queryRepoApps(Long.parseLong(pathSegments.get(1)), pathSegments.get(2))); break; - + default: Log.e(TAG, "Invalid URI for apk content provider: " + uri); throw new UnsupportedOperationException("Invalid URI for apk content provider: " + uri); @@ -535,7 +535,7 @@ public class ApkProvider extends FDroidProvider { case CODE_APKS: query = query.add(queryApks(uri.getLastPathSegment())); break; - + // TODO: Add tests for this. case CODE_REPO_APK: List pathSegments = uri.getPathSegments(); @@ -561,11 +561,13 @@ public class ApkProvider extends FDroidProvider { @Override public int update(Uri uri, ContentValues values, String where, String[] whereArgs) { - if (matcher.match(uri) != CODE_SINGLE) { throw new UnsupportedOperationException("Cannot update anything other than a single apk."); } + return performUpdateUnchecked(uri, values, where, whereArgs); + } + protected int performUpdateUnchecked(Uri uri, ContentValues values, String where, String[] whereArgs) { validateFields(DataColumns.ALL, values); removeRepoFields(values); @@ -577,7 +579,6 @@ public class ApkProvider extends FDroidProvider { getContext().getContentResolver().notifyChange(uri, null); } return numRows; - } } diff --git a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java index 8ddde3c53..ee4812949 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java @@ -286,7 +286,7 @@ public class AppProvider extends FDroidProvider { } - private static class Query extends QueryBuilder { + private class Query extends QueryBuilder { private boolean isSuggestedApkTableAdded; private boolean requiresInstalledTable; @@ -366,14 +366,14 @@ public class AppProvider extends FDroidProvider { if (field.equals(DataColumns.CATEGORIES)) { categoryFieldAdded = true; } - appendField(field, "fdroid_app"); + appendField(field, getTableName()); break; } } private void appendCountField() { countFieldAppended = true; - appendField("COUNT( DISTINCT fdroid_app.id ) AS " + DataColumns._COUNT); + appendField("COUNT( DISTINCT " + getTableName() + ".id ) AS " + DataColumns._COUNT); } private void addSuggestedApkVersionField() { @@ -388,7 +388,7 @@ public class AppProvider extends FDroidProvider { leftJoin( DBHelper.TABLE_APK, "suggestedApk", - "fdroid_app.suggestedVercode = suggestedApk.vercode AND fdroid_app.id = suggestedApk.id"); + getTableName() + ".suggestedVercode = suggestedApk.vercode AND " + getTableName() + ".id = suggestedApk.id"); } appendField(fieldName, "suggestedApk", alias); } @@ -570,15 +570,15 @@ public class AppProvider extends FDroidProvider { } private AppQuerySelection queryCanUpdate() { - final String ignoreCurrent = " fdroid_app.ignoreThisUpdate != fdroid_app.suggestedVercode "; - final String ignoreAll = " fdroid_app.ignoreAllUpdates != 1 "; + final String ignoreCurrent = getTableName() + ".ignoreThisUpdate != " + getTableName() + ".suggestedVercode "; + final String ignoreAll = getTableName() + ".ignoreAllUpdates != 1 "; final String ignore = " ( " + ignoreCurrent + " AND " + ignoreAll + " ) "; - final String where = ignore + " AND fdroid_app." + DataColumns.SUGGESTED_VERSION_CODE + " > installed.versionCode"; + final String where = ignore + " AND " + getTableName() + "." + DataColumns.SUGGESTED_VERSION_CODE + " > installed.versionCode"; return new AppQuerySelection(where).requireNaturalInstalledTable(); } private AppQuerySelection queryRepo(long repoId) { - final String selection = " fdroid_apk.repo = ? "; + final String selection = DBHelper.TABLE_APK + ".repo = ? "; final String[] args = {String.valueOf(repoId)}; return new AppQuerySelection(selection, args); } @@ -589,10 +589,10 @@ public class AppProvider extends FDroidProvider { private AppQuerySelection querySearch(String query) { final String[] columns = { - "fdroid_app.id", - "fdroid_app.name", - "fdroid_app.summary", - "fdroid_app.description", + getTableName() + ".id", + getTableName() + ".name", + getTableName() + ".summary", + getTableName() + ".description", }; // Remove duplicates, surround in % for case insensitive searching @@ -632,15 +632,16 @@ public class AppProvider extends FDroidProvider { return new AppQuerySelection(selection.toString(), selectionKeywords); } - private AppQuerySelection querySingle(String id) { - final String selection = "fdroid_app.id = ?"; + protected AppQuerySelection querySingle(String id) { + final String selection = getTableName() + ".id = ?"; final String[] args = {id}; return new AppQuerySelection(selection, args); } private AppQuerySelection queryIgnored() { - final String selection = "fdroid_app.ignoreAllUpdates = 1 OR " + - "fdroid_app.ignoreThisUpdate >= fdroid_app.suggestedVercode"; + final String table = getTableName(); + final String selection = table + ".ignoreAllUpdates = 1 OR " + + table + ".ignoreThisUpdate >= " + table + ".suggestedVercode"; return new AppQuerySelection(selection); } @@ -653,13 +654,13 @@ public class AppProvider extends FDroidProvider { } private AppQuerySelection queryNewlyAdded() { - final String selection = "fdroid_app.added > ?"; + final String selection = getTableName() + ".added > ?"; final String[] args = {Utils.formatDate(Preferences.get().calcMaxHistory(), "")}; return new AppQuerySelection(selection, args); } private AppQuerySelection queryRecentlyUpdated() { - final String selection = "fdroid_app.added != fdroid_app.lastUpdated AND fdroid_app.lastUpdated > ?"; + final String selection = getTableName() + ".added != fdroid_app.lastUpdated AND fdroid_app.lastUpdated > ?"; final String[] args = {Utils.formatDate(Preferences.get().calcMaxHistory(), "")}; return new AppQuerySelection(selection, args); } @@ -668,10 +669,10 @@ public class AppProvider extends FDroidProvider { // TODO: In the future, add a new table for categories, // so we can join onto it. final String selection = - " fdroid_app.categories = ? OR " + // Only category e.g. "internet" - " fdroid_app.categories LIKE ? OR " + // First category e.g. "internet,%" - " fdroid_app.categories LIKE ? OR " + // Last category e.g. "%,internet" - " fdroid_app.categories LIKE ? "; // One of many categories e.g. "%,internet,%" + getTableName() + ".categories = ? OR " + // Only category e.g. "internet" + getTableName() + ".categories LIKE ? OR " + // First category e.g. "internet,%" + getTableName() + ".categories LIKE ? OR " + // Last category e.g. "%,internet" + getTableName() + ".categories LIKE ? "; // One of many categories e.g. "%,internet,%" final String[] args = { category, category + ",%", @@ -682,7 +683,7 @@ public class AppProvider extends FDroidProvider { } private AppQuerySelection queryNoApks() { - String selection = "(SELECT COUNT(*) FROM fdroid_apk WHERE fdroid_apk.id = fdroid_app.id) = 0"; + String selection = "(SELECT COUNT(*) FROM " + DBHelper.TABLE_APK + " WHERE " + DBHelper.TABLE_APK + ".id = " + getTableName() + ".id) = 0"; return new AppQuerySelection(selection); } @@ -692,8 +693,8 @@ public class AppProvider extends FDroidProvider { return new AppQuerySelection(selection, args); } - private static AppQuerySelection queryApps(String appIds) { - return queryApps(appIds, "fdroid_app.id"); + private AppQuerySelection queryApps(String appIds) { + return queryApps(appIds, getTableName() + ".id"); } @Override @@ -754,13 +755,13 @@ public class AppProvider extends FDroidProvider { break; case RECENTLY_UPDATED: - sortOrder = " fdroid_app.lastUpdated DESC"; + sortOrder = getTableName() + ".lastUpdated DESC"; selection = selection.add(queryRecentlyUpdated()); includeSwap = false; break; case NEWLY_ADDED: - sortOrder = " fdroid_app.added DESC"; + sortOrder = getTableName() + ".added DESC"; selection = selection.add(queryNewlyAdded()); includeSwap = false; break; @@ -775,7 +776,7 @@ public class AppProvider extends FDroidProvider { } if (AppProvider.DataColumns.NAME.equals(sortOrder)) { - sortOrder = " fdroid_app." + sortOrder + " COLLATE LOCALIZED "; + sortOrder = getTableName() + sortOrder + " COLLATE LOCALIZED "; } Query query = new Query(); diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java index 8a6f8e20e..0079487b7 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -88,7 +88,6 @@ public class TempApkProvider extends ApkProvider { @Override public Uri insert(Uri uri, ContentValues values) { int code = matcher.match(uri); - if (code == CODE_INIT) { initTable(); return null; @@ -100,6 +99,40 @@ public class TempApkProvider extends ApkProvider { } } + @Override + public int update(Uri uri, ContentValues values, String where, String[] whereArgs) { + + if (matcher.match(uri) != CODE_SINGLE) { + throw new UnsupportedOperationException("Cannot update anything other than a single apk."); + } + + return performUpdateUnchecked(uri, values, where, whereArgs); + } + + @Override + public int delete(Uri uri, String where, String[] whereArgs) { + + QuerySelection query = new QuerySelection(where, whereArgs); + + switch (matcher.match(uri)) { + case CODE_REPO_APK: + List pathSegments = uri.getPathSegments(); + query = query.add(queryRepo(Long.parseLong(pathSegments.get(1)))).add(queryApks(pathSegments.get(2))); + break; + + default: + Log.e(TAG, "Invalid URI for apk content provider: " + uri); + throw new UnsupportedOperationException("Invalid URI for apk content provider: " + uri); + } + + int rowsAffected = write().delete(getTableName(), query.getSelection(), query.getArgs()); + if (!isApplyingBatch()) { + getContext().getContentResolver().notifyChange(uri, null); + } + return rowsAffected; + + } + private void initTable() { write().execSQL("DROP TABLE IF EXISTS " + getTableName()); write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APK); @@ -109,5 +142,6 @@ public class TempApkProvider extends ApkProvider { Log.d(TAG, "Deleting all apks from " + DBHelper.TABLE_APK + " so they can be copied from " + getTableName()); write().execSQL("DELETE FROM " + DBHelper.TABLE_APK); write().execSQL("INSERT INTO " + DBHelper.TABLE_APK + " SELECT * FROM " + getTableName()); + getContext().getContentResolver().notifyChange(ApkProvider.getContentUri(), null); } } diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java index e469191f8..2e6737ac9 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java @@ -83,6 +83,25 @@ public class TempAppProvider extends AppProvider { } } + @Override + public int update(Uri uri, ContentValues values, String where, String[] whereArgs) { + QuerySelection query = new QuerySelection(where, whereArgs); + switch (matcher.match(uri)) { + case CODE_SINGLE: + query = query.add(querySingle(uri.getLastPathSegment())); + break; + + default: + throw new UnsupportedOperationException("Update not supported for " + uri + "."); + } + + int count = write().update(getTableName(), values, query.getSelection(), query.getArgs()); + if (!isApplyingBatch()) { + getContext().getContentResolver().notifyChange(uri, null); + } + return count; + } + private void initTable() { write().execSQL("DROP TABLE IF EXISTS " + getTableName()); write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APP); @@ -92,5 +111,6 @@ public class TempAppProvider extends AppProvider { Log.d(TAG, "Deleting all apks from " + DBHelper.TABLE_APP + " so they can be copied from " + getTableName()); write().execSQL("DELETE FROM " + DBHelper.TABLE_APP); write().execSQL("INSERT INTO " + DBHelper.TABLE_APP + " SELECT * FROM " + getTableName()); + getContext().getContentResolver().notifyChange(AppProvider.getContentUri(), null); } } diff --git a/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java b/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java index b5471f92c..7664d1d4a 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java @@ -20,6 +20,8 @@ import org.fdroid.fdroid.data.ApkProvider; import org.fdroid.fdroid.data.AppProvider; import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; +import org.fdroid.fdroid.data.TempApkProvider; +import org.fdroid.fdroid.data.TempAppProvider; import java.io.File; import java.util.List; @@ -82,6 +84,8 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { resolver.addProvider(AppProvider.getAuthority(), prepareProvider(new AppProvider())); resolver.addProvider(ApkProvider.getAuthority(), prepareProvider(new ApkProvider())); resolver.addProvider(RepoProvider.getAuthority(), prepareProvider(new RepoProvider())); + resolver.addProvider(TempAppProvider.getAuthority(), prepareProvider(new TempAppProvider())); + resolver.addProvider(TempApkProvider.getAuthority(), prepareProvider(new TempApkProvider())); } private ContentProvider prepareProvider(ContentProvider provider) { diff --git a/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java b/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java index c8692105b..a7fdfd6b4 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java @@ -8,7 +8,6 @@ import org.fdroid.fdroid.RepoUpdater.UpdateException; import org.fdroid.fdroid.data.Repo; import java.io.File; -import java.util.UUID; public class RepoUpdaterTest extends InstrumentationTestCase { private static final String TAG = "RepoUpdaterTest"; diff --git a/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java b/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java index c9be7f43c..21bab2c5f 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java @@ -1,6 +1,7 @@ package org.fdroid.fdroid; +import android.support.annotation.NonNull; import android.test.AndroidTestCase; import android.text.TextUtils; import android.util.Log; @@ -8,6 +9,7 @@ import android.util.Log; import org.fdroid.fdroid.data.Apk; import org.fdroid.fdroid.data.App; import org.fdroid.fdroid.data.Repo; +import org.fdroid.fdroid.mock.MockRepo; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.XMLReader; @@ -25,34 +27,7 @@ import javax.xml.parsers.SAXParserFactory; public class RepoXMLHandlerTest extends AndroidTestCase { private static final String TAG = "RepoXMLHandlerTest"; - private final Repo actualRepo = new Repo(); - - public final List actualApps = new ArrayList<>(); - public final List actualApks = new ArrayList<>(); - - private final static String FAKE_PUBKEY = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345"; - - private RepoXMLHandler.IndexReceiver indexReceiver = new RepoXMLHandler.IndexReceiver() { - - private boolean hasReceivedRepo; - - @Override - public void receiveRepo(String name, String description, String signingCert, int maxage, int version) { - assertFalse("Repo XML contains more than one .", hasReceivedRepo); - actualRepo.name = name; - actualRepo.description = description; - actualRepo.pubkey = signingCert; - actualRepo.maxage = maxage; - actualRepo.version = version; - hasReceivedRepo = true; - } - - @Override - public void receiveApp(App app, List packages) { - actualApps.add(app); - actualApks.addAll(packages); - } - }; + private static final String FAKE_PUBKEY = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345"; public RepoXMLHandlerTest() { } @@ -67,10 +42,8 @@ public class RepoXMLHandlerTest extends AndroidTestCase { expectedRepo.name = "F-Droid"; expectedRepo.pubkey = "308201ee30820157a0030201020204300d845b300d06092a864886f70d01010b0500302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e301e170d3134303432373030303633315a170d3431303931323030303633315a302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e30819f300d06092a864886f70d010101050003818d0030818902818100a439472e4b6d01141bfc94ecfe131c7c728fdda670bb14c57ca60bd1c38a8b8bc0879d22a0a2d0bc0d6fdd4cb98d1d607c2caefbe250a0bd0322aedeb365caf9b236992fac13e6675d3184a6c7c6f07f73410209e399a9da8d5d7512bbd870508eebacff8b57c3852457419434d34701ccbf692267cbc3f42f1c5d1e23762d790203010001a321301f301d0603551d0e041604140b1840691dab909746fde4bfe28207d1cae15786300d06092a864886f70d01010b05000381810062424c928ffd1b6fd419b44daafef01ca982e09341f7077fb865905087aeac882534b3bd679b51fdfb98892cef38b63131c567ed26c9d5d9163afc775ac98ad88c405d211d6187bde0b0d236381cc574ba06ef9080721a92ae5a103a7301b2c397eecc141cc850dd3e123813ebc41c59d31ddbcb6e984168280c53272f6a442b"; expectedRepo.description = "The official repository of the F-Droid client. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitorious.org/f-droid."; - processFile("simpleIndex.xml"); - handlerTestSuite(expectedRepo, 0, 0); - assertEquals(actualRepo.maxage, -1); - assertEquals(actualRepo.version, 12); + RepoDetails actualDetails = getFromFile("simpleIndex.xml"); + handlerTestSuite(expectedRepo, actualDetails, 0, 0, -1, 12); } public void testSmallRepo() { @@ -78,11 +51,9 @@ public class RepoXMLHandlerTest extends AndroidTestCase { expectedRepo.name = "Android-Nexus-7-20139453 on UNSET"; expectedRepo.pubkey = "308202da308201c2a00302010202080eb08c796fec91aa300d06092a864886f70d0101050500302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a656374301e170d3134313030333135303631325a170d3135313030333135303631325a302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a65637430820122300d06092a864886f70d01010105000382010f003082010a0282010100c7ab44b130be5c00eedcc3625462f6f6ac26e502641cd641f3e30cbb0ff1ba325158611e7fc2448a35b6a6df30dc6e23602cf6909448befcf11e2fe486b580f1e76fe5887d159050d00afd2c4079f6538896bb200627f4b3e874f011ce5df0fef5d150fcb0b377b531254e436eaf4083ea72fe3b8c3ef450789fa858f2be8f6c5335bb326aff3dda689fbc7b5ba98dea53651dbea7452c38d294985ac5dd8a9e491a695de92c706d682d6911411fcaef3b0a08a030fe8a84e47acaab0b7edcda9d190ce39e810b79b1d8732eca22b15f0d048c8d6f00503a7ee81ab6e08919ff465883432304d95238b95e95c5f74e0a421809e2a6a85825aed680e0d6939e8f0203010001300d06092a864886f70d010105050003820101006d17aad3271b8b2c299dbdb7b1182849b0d5ddb9f1016dcb3487ae0db02b6be503344c7d066e2050bcd01d411b5ee78c7ed450f0ff9da5ce228f774cbf41240361df53d9c6078159d16f4d34379ab7dedf6186489397c83b44b964251a2ebb42b7c4689a521271b1056d3b5a5fa8f28ba64fb8ce5e2226c33c45d27ba3f632dc266c12abf582b8438c2abcf3eae9de9f31152b4158ace0ef33435c20eb809f1b3988131db6e5a1442f2617c3491d9565fedb3e320e8df4236200d3bd265e47934aa578f84d0d1a5efeb49b39907e876452c46996d0feff9404b41aa5631b4482175d843d5512ded45e12a514690646492191e7add434afce63dbff8f0b03ec0c"; expectedRepo.description = "A local FDroid repo generated from apps installed on Android-Nexus-7-20139453"; - processFile("smallRepo.xml"); - handlerTestSuite(expectedRepo, 12, 12); - assertEquals(actualRepo.maxage, 14); - assertEquals(actualRepo.version, -1); - checkIncludedApps(new String[] { + RepoDetails actualDetails = getFromFile("smallRepo.xml"); + handlerTestSuite(expectedRepo, actualDetails, 12, 12, 14, -1); + checkIncludedApps(actualDetails.apps, new String[]{ "org.mozilla.firefox", "com.koushikdutta.superuser", "info.guardianproject.courier", @@ -103,11 +74,9 @@ public class RepoXMLHandlerTest extends AndroidTestCase { expectedRepo.name = "Guardian Project Official Releases"; expectedRepo.pubkey = "308205d8308203c0020900a397b4da7ecda034300d06092a864886f70d01010505003081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f301e170d3134303632363139333931385a170d3431313131303139333931385a3081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f30820222300d06092a864886f70d01010105000382020f003082020a0282020100b3cd79121b9b883843be3c4482e320809106b0a23755f1dd3c7f46f7d315d7bb2e943486d61fc7c811b9294dcc6b5baac4340f8db2b0d5e14749e7f35e1fc211fdbc1071b38b4753db201c314811bef885bd8921ad86facd6cc3b8f74d30a0b6e2e6e576f906e9581ef23d9c03e926e06d1f033f28bd1e21cfa6a0e3ff5c9d8246cf108d82b488b9fdd55d7de7ebb6a7f64b19e0d6b2ab1380a6f9d42361770d1956701a7f80e2de568acd0bb4527324b1e0973e89595d91c8cc102d9248525ae092e2c9b69f7414f724195b81427f28b1d3d09a51acfe354387915fd9521e8c890c125fc41a12bf34d2a1b304067ab7251e0e9ef41833ce109e76963b0b256395b16b886bca21b831f1408f836146019e7908829e716e72b81006610a2af08301de5d067c9e114a1e5759db8a6be6a3cc2806bcfe6fafd41b5bc9ddddb3dc33d6f605b1ca7d8a9e0ecdd6390d38906649e68a90a717bea80fa220170eea0c86fc78a7e10dac7b74b8e62045a3ecca54e035281fdc9fe5920a855fde3c0be522e3aef0c087524f13d973dff3768158b01a5800a060c06b451ec98d627dd052eda804d0556f60dbc490d94e6e9dea62ffcafb5beffbd9fc38fb2f0d7050004fe56b4dda0a27bc47554e1e0a7d764e17622e71f83a475db286bc7862deee1327e2028955d978272ea76bf0b88e70a18621aba59ff0c5993ef5f0e5d6b6b98e68b70203010001300d06092a864886f70d0101050500038202010079c79c8ef408a20d243d8bd8249fb9a48350dc19663b5e0fce67a8dbcb7de296c5ae7bbf72e98a2020fb78f2db29b54b0e24b181aa1c1d333cc0303685d6120b03216a913f96b96eb838f9bff125306ae3120af838c9fc07ebb5100125436bd24ec6d994d0bff5d065221871f8410daf536766757239bf594e61c5432c9817281b985263bada8381292e543a49814061ae11c92a316e7dc100327b59e3da90302c5ada68c6a50201bda1fcce800b53f381059665dbabeeb0b50eb22b2d7d2d9b0aa7488ca70e67ac6c518adb8e78454a466501e89d81a45bf1ebc350896f2c3ae4b6679ecfbf9d32960d4f5b493125c7876ef36158562371193f600bc511000a67bdb7c664d018f99d9e589868d103d7e0994f166b2ba18ff7e67d8c4da749e44dfae1d930ae5397083a51675c409049dfb626a96246c0015ca696e94ebb767a20147834bf78b07fece3f0872b057c1c519ff882501995237d8206b0b3832f78753ebd8dcbd1d3d9f5ba733538113af6b407d960ec4353c50eb38ab29888238da843cd404ed8f4952f59e4bbc0035fc77a54846a9d419179c46af1b4a3b7fc98e4d312aaa29b9b7d79e739703dc0fa41c7280d5587709277ffa11c3620f5fba985b82c238ba19b17ebd027af9424be0941719919f620dd3bb3c3f11638363708aa11f858e153cf3a69bce69978b90e4a273836100aa1e617ba455cd00426847f"; expectedRepo.description = "The official app repository of The Guardian Project. Applications in this repository are official binaries build by the original application developers and signed by the same key as the APKs that are released in the Google Play store."; - processFile("mediumRepo.xml"); - handlerTestSuite(expectedRepo, 15, 36); - assertEquals(expectedRepo.maxage, 60); - assertEquals(expectedRepo.version, 12); - checkIncludedApps(new String[] { + RepoDetails actualDetails = getFromFile("mediumRepo.xml"); + handlerTestSuite(expectedRepo, actualDetails, 15, 36, 60, 12); + checkIncludedApps(actualDetails.apps, new String[]{ "info.guardianproject.cacert", "info.guardianproject.otr.app.im", "info.guardianproject.soundrecorder", @@ -131,15 +100,13 @@ public class RepoXMLHandlerTest extends AndroidTestCase { expectedRepo.name = "F-Droid"; expectedRepo.pubkey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef"; expectedRepo.description = "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time."; - processFile("largeRepo.xml"); - handlerTestSuite(expectedRepo, 1211, 2381); - assertEquals("Repo max age", 14, expectedRepo.maxage); - assertEquals("Repo version", 12, expectedRepo.version); + RepoDetails actualDetails = getFromFile("largeRepo.xml"); + handlerTestSuite(expectedRepo, actualDetails, 1211, 2381, 14, 12); /* * generated using: sed 's, actualApps, String[] expctedAppIds) { assertNotNull(actualApps); - assertNotNull(packageNames); - assertEquals(actualApps.size(), packageNames.length); - for (String id : packageNames) { + assertNotNull(expctedAppIds); + assertEquals(actualApps.size(), expctedAppIds.length); + for (String id : expctedAppIds) { boolean thisAppMissing = true; for (App app : actualApps) { if (TextUtils.equals(app.id, id)) { @@ -638,53 +605,84 @@ public class RepoXMLHandlerTest extends AndroidTestCase { } } - private void handlerTestSuite(Repo expectedRepo, int appCount, int apkCount) { - assertFalse(TextUtils.isEmpty(actualRepo.pubkey)); - assertEquals(expectedRepo.pubkey.length(), actualRepo.pubkey.length()); - assertEquals(expectedRepo.pubkey, actualRepo.pubkey); - assertFalse(FAKE_PUBKEY.equals(actualRepo.pubkey)); + private void handlerTestSuite(Repo expectedRepo, RepoDetails actualDetails, int appCount, int apkCount, int maxAge, int version) { + assertNotNull(actualDetails); + assertFalse(TextUtils.isEmpty(actualDetails.signingCert)); + assertEquals(expectedRepo.pubkey.length(), actualDetails.signingCert.length()); + assertEquals(expectedRepo.pubkey, actualDetails.signingCert); + assertFalse(FAKE_PUBKEY.equals(actualDetails.signingCert)); - assertFalse(TextUtils.isEmpty(actualRepo.name)); - assertEquals(expectedRepo.name.length(), actualRepo.name.length()); - assertEquals(expectedRepo.name, actualRepo.name); + assertFalse(TextUtils.isEmpty(actualDetails.name)); + assertEquals(expectedRepo.name.length(), actualDetails.name.length()); + assertEquals(expectedRepo.name, actualDetails.name); - assertFalse(TextUtils.isEmpty(actualRepo.description)); - assertEquals(expectedRepo.description.length(), actualRepo.description.length()); - assertEquals(expectedRepo.description, actualRepo.description); + assertFalse(TextUtils.isEmpty(actualDetails.description)); + assertEquals(expectedRepo.description.length(), actualDetails.description.length()); + assertEquals(expectedRepo.description, actualDetails.description); - assertNotNull(actualApps); - assertEquals(actualApps.size(), appCount); + assertEquals(actualDetails.maxAge, maxAge); + assertEquals(actualDetails.version, version); - List apks = actualApks; + List apps = actualDetails.apps; + assertNotNull(apps); + assertEquals(apps.size(), appCount); + + List apks = actualDetails.apks; assertNotNull(apks); assertEquals(apks.size(), apkCount); } - private static class MockRepo extends Repo { + private static class RepoDetails implements RepoXMLHandler.IndexReceiver { + + public String name; + public String description; + public String signingCert; + public int maxAge; + public int version; + + public List apks = new ArrayList<>(); + public List apps = new ArrayList<>(); + @Override - public long getId() { - return 10000; + public void receiveRepo(String name, String description, String signingCert, int maxage, int version) { + this.name = name; + this.description = description; + this.signingCert = signingCert; + this.maxAge = maxage; + this.version = version; } + + @Override + public void receiveApp(App app, List packages) { + apks.addAll(packages); + apps.add(app); + } + } - private RepoXMLHandler processFile(String indexFilename) { + @NonNull + private RepoDetails getFromFile(String indexFilename) { SAXParser parser; try { parser = SAXParserFactory.newInstance().newSAXParser(); XMLReader reader = parser.getXMLReader(); - RepoXMLHandler handler = new RepoXMLHandler(new MockRepo(), indexReceiver); + RepoDetails repoDetails = new RepoDetails(); + RepoXMLHandler handler = new RepoXMLHandler(new MockRepo(100), repoDetails); reader.setContentHandler(handler); String resName = "assets/" + indexFilename; Log.i(TAG, "test file: " + getClass().getClassLoader().getResource(resName)); InputStream input = getClass().getClassLoader().getResourceAsStream(resName); InputSource is = new InputSource(new BufferedInputStream(input)); reader.parse(is); - return handler; + return repoDetails; } catch (ParserConfigurationException | SAXException | IOException e) { e.printStackTrace(); fail(); + + // Satisfies the compiler, but fail() will always throw a runtime exception so we never + // reach this return statement. + return null; } - return null; } } From ad6a8e5b4ea3e1627f6d751117cc91ccf38f517d Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 9 Nov 2015 18:02:18 +1100 Subject: [PATCH 06/17] Extracted RepoPersister class from RepoUpdater to separate security and DB logic. --- .../src/org/fdroid/fdroid/RepoUpdater.java | 261 +--------------- .../org/fdroid/fdroid/data/RepoPersister.java | 287 ++++++++++++++++++ 2 files changed, 294 insertions(+), 254 deletions(-) create mode 100644 F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 493b1cdb1..1fe11220e 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -1,21 +1,16 @@ package org.fdroid.fdroid; -import android.content.ContentProviderOperation; import android.content.ContentValues; import android.content.Context; -import android.content.OperationApplicationException; -import android.net.Uri; -import android.os.RemoteException; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.text.TextUtils; import android.util.Log; import org.fdroid.fdroid.data.Apk; -import org.fdroid.fdroid.data.ApkProvider; import org.fdroid.fdroid.data.App; -import org.fdroid.fdroid.data.AppProvider; import org.fdroid.fdroid.data.Repo; +import org.fdroid.fdroid.data.RepoPersister; import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.data.TempApkProvider; import org.fdroid.fdroid.data.TempAppProvider; @@ -33,11 +28,8 @@ import java.net.URL; import java.security.CodeSigner; import java.security.cert.Certificate; import java.security.cert.X509Certificate; -import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -63,20 +55,6 @@ public class RepoUpdater { public static final String PROGRESS_TYPE_PROCESS_XML = "processingXml"; public static final String PROGRESS_DATA_REPO_ADDRESS = "repoAddress"; - /** - * When an app already exists in the db, and we are updating it on the off chance that some - * values changed in the index, some fields should not be updated. Rather, they should be - * ignored, because they were explicitly set by the user, and hence can't be automatically - * overridden by the index. - * - * NOTE: In the future, these attributes will be moved to a join table, so that the app table - * is essentially completely transient, and can be nuked at any time. - */ - private static final String[] APP_FIELDS_TO_IGNORE = { - AppProvider.DataColumns.IGNORE_ALLUPDATES, - AppProvider.DataColumns.IGNORE_THISUPDATE, - }; - @NonNull protected final Context context; @NonNull @@ -87,6 +65,9 @@ public class RepoUpdater { private String cacheTag; private X509Certificate signingCertFromJar; + @NonNull + private final RepoPersister persister; + /** * Updates an app repo as read out of the database into a {@link Repo} instance. * @@ -95,6 +76,7 @@ public class RepoUpdater { public RepoUpdater(@NonNull Context context, @NonNull Repo repo) { this.context = context; this.repo = repo; + this.persister = new RepoPersister(context, repo); } public void setProgressListener(@Nullable ProgressListener progressListener) { @@ -177,7 +159,7 @@ public class RepoUpdater { @Override public void receiveApp(App app, List packages) { try { - saveToDb(app, packages); + persister.saveToDb(app, packages); } catch (UpdateException e) { throw new RuntimeException("Error while saving repo details to database.", e); } @@ -185,231 +167,6 @@ public class RepoUpdater { }; } - /** - * My crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get: - * * 25 = 37 seconds - * * 50 = 33 seconds - * * 100 = 30 seconds - * * 200 = 32 seconds - * Raising this means more memory consumption, so we'd like it to be low, but not - * so low that it takes too long. - */ - private static final int MAX_APP_BUFFER = 50; - - private List appsToSave = new ArrayList<>(); - private Map> apksToSave = new HashMap<>(); - - private void saveToDb(App app, List packages) throws UpdateException { - appsToSave.add(app); - apksToSave.put(app.id, packages); - - if (appsToSave.size() >= MAX_APP_BUFFER) { - flushBufferToDb(); - } - } - - private void flushBufferToDb() throws UpdateException { - if (apksToSave.size() > 0 || appsToSave.size() > 0) { - Log.d(TAG, "Flushing details of up to " + MAX_APP_BUFFER + " apps and their packages to the database."); - flushAppsToDbInBatch(); - flushApksToDbInBatch(); - apksToSave.clear(); - appsToSave.clear(); - } - } - - private void flushApksToDbInBatch() throws UpdateException { - List apksToSaveList = new ArrayList<>(); - for (Map.Entry> entries : apksToSave.entrySet()) { - apksToSaveList.addAll(entries.getValue()); - } - - calcApkCompatibilityFlags(apksToSaveList); - - ArrayList apkOperations = new ArrayList<>(); - ContentProviderOperation clearOrphans = deleteOrphanedApks(appsToSave, apksToSave); - if (clearOrphans != null) { - apkOperations.add(clearOrphans); - } - apkOperations.addAll(insertOrUpdateApks(apksToSaveList)); - - try { - context.getContentResolver().applyBatch(TempApkProvider.getAuthority(), apkOperations); - } catch (RemoteException | OperationApplicationException e) { - throw new UpdateException(repo, "An internal error occured while updating the database", e); - } - } - - private void flushAppsToDbInBatch() throws UpdateException { - ArrayList appOperations = insertOrUpdateApps(appsToSave); - - try { - context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations); - } catch (RemoteException | OperationApplicationException e) { - throw new UpdateException(repo, "An internal error occured while updating the database", e); - } - } - - /** - * Depending on whether the {@link App}s have been added to the database previously, this - * will queue up an update or an insert {@link ContentProviderOperation} for each app. - */ - private ArrayList insertOrUpdateApps(List apps) { - ArrayList operations = new ArrayList<>(apps.size()); - for (App app : apps) { - if (isAppInDatabase(app)) { - operations.add(updateExistingApp(app)); - } else { - operations.add(insertNewApp(app)); - } - } - return operations; - } - - /** - * Depending on whether the .apks have been added to the database previously, this - * will queue up an update or an insert {@link ContentProviderOperation} for each package. - */ - private ArrayList insertOrUpdateApks(List packages) { - List existingApks = ApkProvider.Helper.knownApks(context, packages, new String[]{ApkProvider.DataColumns.VERSION_CODE}); - ArrayList operations = new ArrayList<>(packages.size()); - for (Apk apk : packages) { - boolean exists = false; - for (Apk existing : existingApks) { - if (existing.vercode == apk.vercode) { - exists = true; - break; - } - } - - if (exists) { - operations.add(updateExistingApk(apk)); - } else { - operations.add(insertNewApk(apk)); - } - } - - return operations; - } - - /** - * Creates an update {@link ContentProviderOperation} for the {@link App} in question. - * Does not do any checks to see if the app already exists or not. - */ - private ContentProviderOperation updateExistingApp(App app) { - Uri uri = TempAppProvider.getAppUri(app); - ContentValues values = app.toContentValues(); - for (final String toIgnore : APP_FIELDS_TO_IGNORE) { - if (values.containsKey(toIgnore)) { - values.remove(toIgnore); - } - } - return ContentProviderOperation.newUpdate(uri).withValues(values).build(); - } - - /** - * Creates an insert {@link ContentProviderOperation} for the {@link App} in question. - * Does not do any checks to see if the app already exists or not. - */ - private ContentProviderOperation insertNewApp(App app) { - ContentValues values = app.toContentValues(); - Uri uri = TempAppProvider.getContentUri(); - return ContentProviderOperation.newInsert(uri).withValues(values).build(); - } - - /** - * Looks in the database to see which apps we already know about. Only - * returns ids of apps that are in the database if they are in the "apps" - * array. - */ - private boolean isAppInDatabase(App app) { - String[] fields = {AppProvider.DataColumns.APP_ID}; - App found = AppProvider.Helper.findById(context.getContentResolver(), app.id, fields); - return found != null; - } - - /** - * Creates an update {@link ContentProviderOperation} for the {@link Apk} in question. - * Does not do any checks to see if the apk already exists or not. - */ - private ContentProviderOperation updateExistingApk(final Apk apk) { - Uri uri = TempApkProvider.getApkUri(apk); - ContentValues values = apk.toContentValues(); - return ContentProviderOperation.newUpdate(uri).withValues(values).build(); - } - - /** - * Creates an insert {@link ContentProviderOperation} for the {@link Apk} in question. - * Does not do any checks to see if the apk already exists or not. - */ - private ContentProviderOperation insertNewApk(final Apk apk) { - ContentValues values = apk.toContentValues(); - Uri uri = TempApkProvider.getContentUri(); - return ContentProviderOperation.newInsert(uri).withValues(values).build(); - } - - /** - * Finds all apks from the repo we are currently updating, that belong to the specified app, - * and delete them as they are no longer provided by that repo. - */ - @Nullable - private ContentProviderOperation deleteOrphanedApks(List apps, Map> packages) { - - String[] projection = new String[]{ApkProvider.DataColumns.APK_ID, ApkProvider.DataColumns.VERSION_CODE}; - List existing = ApkProvider.Helper.find(context, repo, apps, projection); - - List toDelete = new ArrayList<>(); - - for (Apk existingApk : existing) { - - boolean shouldStay = false; - - for (Map.Entry> entry : packages.entrySet()) { - for (Apk newApk : entry.getValue()) { - if (newApk.vercode == existingApk.vercode) { - shouldStay = true; - break; - } - } - - if (shouldStay) { - break; - } - } - - if (!shouldStay) { - toDelete.add(existingApk); - } - } - - if (toDelete.size() > 0) { - Uri uri = TempApkProvider.getApksUri(repo, toDelete); - return ContentProviderOperation.newDelete(uri).build(); - } else { - return null; - } - } - - /** - * This cannot be offloaded to the database (as we did with the query which - * updates apps, depending on whether their apks are compatible or not). - * The reason is that we need to interact with the CompatibilityChecker - * in order to see if, and why an apk is not compatible. - */ - public void calcApkCompatibilityFlags(List apks) { - final CompatibilityChecker checker = new CompatibilityChecker(context); - for (final Apk apk : apks) { - final List reasons = checker.getIncompatibleReasons(apk); - if (reasons.size() > 0) { - apk.compatible = false; - apk.incompatibleReasons = Utils.CommaSeparatedList.make(reasons); - } else { - apk.compatible = true; - apk.incompatibleReasons = null; - } - } - } - public void processDownloadedFile(File downloadedFile) throws UpdateException { InputStream indexInputStream = null; try { @@ -441,8 +198,6 @@ public class RepoUpdater { reader.setContentHandler(repoXMLHandler); reader.parse(new InputSource(indexInputStream)); - flushBufferToDb(); - signingCertFromJar = getSigningCertFromJar(indexEntry); // JarEntry can only read certificates after the file represented by that JarEntry @@ -450,9 +205,7 @@ public class RepoUpdater { assertSigningCertFromXmlCorrect(); Log.i(TAG, "Repo signature verified, saving app metadata to database."); - TempAppProvider.Helper.commit(context); - TempApkProvider.Helper.commit(context); - RepoProvider.Helper.update(context, repo, repoDetailsToSave); + persister.commit(repoDetailsToSave); } catch (SAXException | ParserConfigurationException | IOException e) { throw new UpdateException(repo, "Error parsing index", e); diff --git a/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java b/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java new file mode 100644 index 000000000..f1166b20a --- /dev/null +++ b/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java @@ -0,0 +1,287 @@ +package org.fdroid.fdroid.data; + +import android.content.ContentProviderOperation; +import android.content.ContentValues; +import android.content.Context; +import android.content.OperationApplicationException; +import android.net.Uri; +import android.os.RemoteException; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.util.Log; + +import org.fdroid.fdroid.CompatibilityChecker; +import org.fdroid.fdroid.RepoUpdater; +import org.fdroid.fdroid.Utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RepoPersister { + + private static final String TAG = "RepoPersister"; + + /** + * When an app already exists in the db, and we are updating it on the off chance that some + * values changed in the index, some fields should not be updated. Rather, they should be + * ignored, because they were explicitly set by the user, and hence can't be automatically + * overridden by the index. + * + * NOTE: In the future, these attributes will be moved to a join table, so that the app table + * is essentially completely transient, and can be nuked at any time. + */ + private static final String[] APP_FIELDS_TO_IGNORE = { + AppProvider.DataColumns.IGNORE_ALLUPDATES, + AppProvider.DataColumns.IGNORE_THISUPDATE, + }; + + /** + * Crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get: + * * 25 = 37 seconds + * * 50 = 33 seconds + * * 100 = 30 seconds + * * 200 = 32 seconds + * Raising this means more memory consumption, so we'd like it to be low, but not + * so low that it takes too long. + */ + private static final int MAX_APP_BUFFER = 50; + + @NonNull + private final Repo repo; + + @NonNull + private final Context context; + + @NonNull + private final List appsToSave = new ArrayList<>(); + + @NonNull + private final Map> apksToSave = new HashMap<>(); + + public RepoPersister(@NonNull Context context, @NonNull Repo repo) { + this.repo = repo; + this.context = context; + } + + public void saveToDb(App app, List packages) throws RepoUpdater.UpdateException { + appsToSave.add(app); + apksToSave.put(app.id, packages); + + if (appsToSave.size() >= MAX_APP_BUFFER) { + flushBufferToDb(); + } + } + + public void commit(ContentValues repoDetailsToSave) throws RepoUpdater.UpdateException { + flushBufferToDb(); + TempAppProvider.Helper.commit(context); + TempApkProvider.Helper.commit(context); + RepoProvider.Helper.update(context, repo, repoDetailsToSave); + } + + private void flushBufferToDb() throws RepoUpdater.UpdateException { + if (apksToSave.size() > 0 || appsToSave.size() > 0) { + Log.d(TAG, "Flushing details of up to " + MAX_APP_BUFFER + " apps and their packages to the database."); + flushAppsToDbInBatch(); + flushApksToDbInBatch(); + apksToSave.clear(); + appsToSave.clear(); + } + } + + private void flushApksToDbInBatch() throws RepoUpdater.UpdateException { + List apksToSaveList = new ArrayList<>(); + for (Map.Entry> entries : apksToSave.entrySet()) { + apksToSaveList.addAll(entries.getValue()); + } + + calcApkCompatibilityFlags(apksToSaveList); + + ArrayList apkOperations = new ArrayList<>(); + ContentProviderOperation clearOrphans = deleteOrphanedApks(appsToSave, apksToSave); + if (clearOrphans != null) { + apkOperations.add(clearOrphans); + } + apkOperations.addAll(insertOrUpdateApks(apksToSaveList)); + + try { + context.getContentResolver().applyBatch(TempApkProvider.getAuthority(), apkOperations); + } catch (RemoteException | OperationApplicationException e) { + throw new RepoUpdater.UpdateException(repo, "An internal error occured while updating the database", e); + } + } + + private void flushAppsToDbInBatch() throws RepoUpdater.UpdateException { + ArrayList appOperations = insertOrUpdateApps(appsToSave); + + try { + context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations); + } catch (RemoteException | OperationApplicationException e) { + throw new RepoUpdater.UpdateException(repo, "An internal error occured while updating the database", e); + } + } + + /** + * Depending on whether the {@link App}s have been added to the database previously, this + * will queue up an update or an insert {@link ContentProviderOperation} for each app. + */ + private ArrayList insertOrUpdateApps(List apps) { + ArrayList operations = new ArrayList<>(apps.size()); + for (App app : apps) { + if (isAppInDatabase(app)) { + operations.add(updateExistingApp(app)); + } else { + operations.add(insertNewApp(app)); + } + } + return operations; + } + + /** + * Depending on whether the .apks have been added to the database previously, this + * will queue up an update or an insert {@link ContentProviderOperation} for each package. + */ + private ArrayList insertOrUpdateApks(List packages) { + List existingApks = ApkProvider.Helper.knownApks(context, packages, new String[]{ApkProvider.DataColumns.VERSION_CODE}); + ArrayList operations = new ArrayList<>(packages.size()); + for (Apk apk : packages) { + boolean exists = false; + for (Apk existing : existingApks) { + if (existing.vercode == apk.vercode) { + exists = true; + break; + } + } + + if (exists) { + operations.add(updateExistingApk(apk)); + } else { + operations.add(insertNewApk(apk)); + } + } + + return operations; + } + + /** + * Creates an update {@link ContentProviderOperation} for the {@link App} in question. + * Does not do any checks to see if the app already exists or not. + */ + private ContentProviderOperation updateExistingApp(App app) { + Uri uri = TempAppProvider.getAppUri(app); + ContentValues values = app.toContentValues(); + for (final String toIgnore : APP_FIELDS_TO_IGNORE) { + if (values.containsKey(toIgnore)) { + values.remove(toIgnore); + } + } + return ContentProviderOperation.newUpdate(uri).withValues(values).build(); + } + + /** + * Creates an insert {@link ContentProviderOperation} for the {@link App} in question. + * Does not do any checks to see if the app already exists or not. + */ + private ContentProviderOperation insertNewApp(App app) { + ContentValues values = app.toContentValues(); + Uri uri = TempAppProvider.getContentUri(); + return ContentProviderOperation.newInsert(uri).withValues(values).build(); + } + + /** + * Looks in the database to see which apps we already know about. Only + * returns ids of apps that are in the database if they are in the "apps" + * array. + */ + private boolean isAppInDatabase(App app) { + String[] fields = {AppProvider.DataColumns.APP_ID}; + App found = AppProvider.Helper.findById(context.getContentResolver(), app.id, fields); + return found != null; + } + + /** + * Creates an update {@link ContentProviderOperation} for the {@link Apk} in question. + * Does not do any checks to see if the apk already exists or not. + */ + private ContentProviderOperation updateExistingApk(final Apk apk) { + Uri uri = TempApkProvider.getApkUri(apk); + ContentValues values = apk.toContentValues(); + return ContentProviderOperation.newUpdate(uri).withValues(values).build(); + } + + /** + * Creates an insert {@link ContentProviderOperation} for the {@link Apk} in question. + * Does not do any checks to see if the apk already exists or not. + */ + private ContentProviderOperation insertNewApk(final Apk apk) { + ContentValues values = apk.toContentValues(); + Uri uri = TempApkProvider.getContentUri(); + return ContentProviderOperation.newInsert(uri).withValues(values).build(); + } + + /** + * Finds all apks from the repo we are currently updating, that belong to the specified app, + * and delete them as they are no longer provided by that repo. + */ + @Nullable + private ContentProviderOperation deleteOrphanedApks(List apps, Map> packages) { + + String[] projection = new String[]{ApkProvider.DataColumns.APK_ID, ApkProvider.DataColumns.VERSION_CODE}; + List existing = ApkProvider.Helper.find(context, repo, apps, projection); + + List toDelete = new ArrayList<>(); + + for (Apk existingApk : existing) { + + boolean shouldStay = false; + + for (Map.Entry> entry : packages.entrySet()) { + for (Apk newApk : entry.getValue()) { + if (newApk.vercode == existingApk.vercode) { + shouldStay = true; + break; + } + } + + if (shouldStay) { + break; + } + } + + if (!shouldStay) { + toDelete.add(existingApk); + } + } + + if (toDelete.size() > 0) { + Uri uri = TempApkProvider.getApksUri(repo, toDelete); + return ContentProviderOperation.newDelete(uri).build(); + } else { + return null; + } + } + + /** + * This cannot be offloaded to the database (as we did with the query which + * updates apps, depending on whether their apks are compatible or not). + * The reason is that we need to interact with the CompatibilityChecker + * in order to see if, and why an apk is not compatible. + */ + private void calcApkCompatibilityFlags(List apks) { + final CompatibilityChecker checker = new CompatibilityChecker(context); + for (final Apk apk : apks) { + final List reasons = checker.getIncompatibleReasons(apk); + if (reasons.size() > 0) { + apk.compatible = false; + apk.incompatibleReasons = Utils.CommaSeparatedList.make(reasons); + } else { + apk.compatible = true; + apk.incompatibleReasons = null; + } + } + } + + +} From b22b39ea26d9a2eec81054b63ad2faafb5d963e2 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 9 Nov 2015 20:05:55 +1100 Subject: [PATCH 07/17] Fix bug with SQL generation (missing . char) from bung rebase. --- F-Droid/src/org/fdroid/fdroid/data/AppProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java index ee4812949..804fd1db2 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java @@ -776,7 +776,7 @@ public class AppProvider extends FDroidProvider { } if (AppProvider.DataColumns.NAME.equals(sortOrder)) { - sortOrder = getTableName() + sortOrder + " COLLATE LOCALIZED "; + sortOrder = getTableName() + "." + sortOrder + " COLLATE LOCALIZED "; } Query query = new Query(); From 71f641860aa6bae844b736ad74c16d742e3b16a9 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 9 Nov 2015 22:40:47 +1100 Subject: [PATCH 08/17] More refactoring of AppProvider to use getTableName() instead of constant. Although not used by the temp provider, it seemed strange having some of the code always using the `DBHelper.TABLE_APP` and other code using `getTableName()` where all of it could have used `getTableName()`. Also moved commiting of the temp tables to the real tables into the `RepoPersiter` instead of in `RepoUpdater`. --- .../src/org/fdroid/fdroid/RepoUpdater.java | 8 --- .../org/fdroid/fdroid/data/AppProvider.java | 65 ++++++++++--------- .../org/fdroid/fdroid/data/RepoPersister.java | 13 ++++ .../fdroid/fdroid/data/TempApkProvider.java | 4 +- .../fdroid/fdroid/data/TempAppProvider.java | 12 +++- 5 files changed, 59 insertions(+), 43 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 1fe11220e..1d36baee5 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -173,14 +173,6 @@ public class RepoUpdater { if (downloadedFile == null || !downloadedFile.exists()) throw new UpdateException(repo, downloadedFile + " does not exist!"); - // This is where we will store all of the metadata before commiting at the - // end of the process. This is due to the fact that we can't verify the cert - // the index was signed with until we've finished reading it - and we don't - // want to put stuff in the real database until we are sure it is from a - // trusted source. - TempAppProvider.Helper.init(context); - TempApkProvider.Helper.init(context); - // Due to a bug in Android 5.0 Lollipop, the inclusion of spongycastle causes // breakage when verifying the signature of the downloaded .jar. For more // details, check out https://gitlab.com/fdroid/fdroidclient/issues/111. diff --git a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java index 804fd1db2..b56616f0e 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java @@ -24,8 +24,6 @@ public class AppProvider extends FDroidProvider { private static final String TAG = "AppProvider"; - public static final int MAX_APPS_TO_QUERY = 900; - public static final class Helper { private Helper() { } @@ -165,7 +163,7 @@ public class AppProvider extends FDroidProvider { static final class UpgradeHelper { public static void updateIconUrls(Context context, SQLiteDatabase db) { - AppProvider.updateIconUrls(context, db); + AppProvider.updateIconUrls(context, db, DBHelper.TABLE_APP, DBHelper.TABLE_APK); } } @@ -295,8 +293,8 @@ public class AppProvider extends FDroidProvider { @Override protected String getRequiredTables() { - final String app = DBHelper.TABLE_APP; - final String apk = DBHelper.TABLE_APK; + final String app = getTableName(); + final String apk = getApkTableName(); final String repo = DBHelper.TABLE_REPO; return app + @@ -312,7 +310,7 @@ public class AppProvider extends FDroidProvider { @Override protected String groupBy() { // If the count field has been requested, then we want to group all rows together. - return countFieldAppended ? null : DBHelper.TABLE_APP + ".id"; + return countFieldAppended ? null : getTableName() + ".id"; } public void addSelection(AppQuerySelection selection) { @@ -329,7 +327,7 @@ public class AppProvider extends FDroidProvider { join( DBHelper.TABLE_INSTALLED_APP, "installed", - "installed." + InstalledAppProvider.DataColumns.APP_ID + " = " + DBHelper.TABLE_APP + ".id"); + "installed." + InstalledAppProvider.DataColumns.APP_ID + " = " + getTableName() + ".id"); requiresInstalledTable = true; } } @@ -339,7 +337,7 @@ public class AppProvider extends FDroidProvider { leftJoin( DBHelper.TABLE_INSTALLED_APP, "installed", - "installed." + InstalledAppProvider.DataColumns.APP_ID + " = " + DBHelper.TABLE_APP + ".id"); + "installed." + InstalledAppProvider.DataColumns.APP_ID + " = " + getTableName() + ".id"); requiresInstalledTable = true; } } @@ -378,15 +376,15 @@ public class AppProvider extends FDroidProvider { private void addSuggestedApkVersionField() { addSuggestedApkField( - ApkProvider.DataColumns.VERSION, - DataColumns.SuggestedApk.VERSION); + ApkProvider.DataColumns.VERSION, + DataColumns.SuggestedApk.VERSION); } private void addSuggestedApkField(String fieldName, String alias) { if (!isSuggestedApkTableAdded) { isSuggestedApkTableAdded = true; leftJoin( - DBHelper.TABLE_APK, + getApkTableName(), "suggestedApk", getTableName() + ".suggestedVercode = suggestedApk.vercode AND " + getTableName() + ".id = suggestedApk.id"); } @@ -395,8 +393,8 @@ public class AppProvider extends FDroidProvider { private void addInstalledAppVersionName() { addInstalledAppField( - InstalledAppProvider.DataColumns.VERSION_NAME, - DataColumns.InstalledApp.VERSION_NAME + InstalledAppProvider.DataColumns.VERSION_NAME, + DataColumns.InstalledApp.VERSION_NAME ); } @@ -555,6 +553,10 @@ public class AppProvider extends FDroidProvider { return DBHelper.TABLE_APP; } + protected String getApkTableName() { + return DBHelper.TABLE_APK; + } + @Override protected String getProviderName() { return "AppProvider"; @@ -578,7 +580,7 @@ public class AppProvider extends FDroidProvider { } private AppQuerySelection queryRepo(long repoId) { - final String selection = DBHelper.TABLE_APK + ".repo = ? "; + final String selection = getApkTableName() + ".repo = ? "; final String[] args = {String.valueOf(repoId)}; return new AppQuerySelection(selection, args); } @@ -660,7 +662,8 @@ public class AppProvider extends FDroidProvider { } private AppQuerySelection queryRecentlyUpdated() { - final String selection = getTableName() + ".added != fdroid_app.lastUpdated AND fdroid_app.lastUpdated > ?"; + final String app = getTableName(); + final String selection = app + ".added != " + app + ".lastUpdated AND " + app + ".lastUpdated > ?"; final String[] args = {Utils.formatDate(Preferences.get().calcMaxHistory(), "")}; return new AppQuerySelection(selection, args); } @@ -683,7 +686,7 @@ public class AppProvider extends FDroidProvider { } private AppQuerySelection queryNoApks() { - String selection = "(SELECT COUNT(*) FROM " + DBHelper.TABLE_APK + " WHERE " + DBHelper.TABLE_APK + ".id = " + getTableName() + ".id) = 0"; + String selection = "(SELECT COUNT(*) FROM " + getApkTableName() + " WHERE " + getApkTableName() + ".id = " + getTableName() + ".id) = 0"; return new AppQuerySelection(selection); } @@ -843,11 +846,11 @@ public class AppProvider extends FDroidProvider { return count; } - private void updateAppDetails() { + protected void updateAppDetails() { updateCompatibleFlags(); updateSuggestedFromUpstream(); updateSuggestedFromLatest(); - updateIconUrls(getContext(), write()); + updateIconUrls(getContext(), write(), getTableName(), getApkTableName()); } /** @@ -865,8 +868,8 @@ public class AppProvider extends FDroidProvider { Utils.debugLog(TAG, "Calculating whether apps are compatible, based on whether any of their apks are compatible"); - final String apk = DBHelper.TABLE_APK; - final String app = DBHelper.TABLE_APP; + final String apk = getApkTableName(); + final String app = getTableName(); String updateSql = "UPDATE " + app + " SET compatible = ( " + @@ -900,8 +903,8 @@ public class AppProvider extends FDroidProvider { Utils.debugLog(TAG, "Calculating suggested versions for all apps which specify an upstream version code."); - final String apk = DBHelper.TABLE_APK; - final String app = DBHelper.TABLE_APP; + final String apk = getApkTableName(); + final String app = getTableName(); final boolean unstableUpdates = Preferences.get().getUnstableUpdates(); String restrictToStable = unstableUpdates ? "" : (apk + ".vercode <= " + app + ".upstreamVercode AND "); @@ -941,8 +944,8 @@ public class AppProvider extends FDroidProvider { Utils.debugLog(TAG, "Calculating suggested versions for all apps which don't specify an upstream version code."); - final String apk = DBHelper.TABLE_APK; - final String app = DBHelper.TABLE_APP; + final String apk = getApkTableName(); + final String app = getTableName(); String updateSql = "UPDATE " + app + " SET suggestedVercode = ( " + @@ -961,7 +964,7 @@ public class AppProvider extends FDroidProvider { * it without instantiating an {@link AppProvider}. This is also the reason it needs to accept * the context and database as arguments. */ - private static void updateIconUrls(Context context, SQLiteDatabase db) { + private static void updateIconUrls(Context context, SQLiteDatabase db, String appTable, String apkTable) { final String iconsDir = Utils.getIconsDir(context, 1.0); final String iconsDirLarge = Utils.getIconsDir(context, 1.5); String repoVersion = Integer.toString(Repo.VERSION_DENSITY_SPECIFIC_ICONS); @@ -969,7 +972,7 @@ public class AppProvider extends FDroidProvider { + repoVersion); Utils.debugLog(TAG, "Using icons dir '" + iconsDir + "'"); Utils.debugLog(TAG, "Using large icons dir '" + iconsDirLarge + "'"); - String query = getIconUpdateQuery(); + String query = getIconUpdateQuery(appTable, apkTable); final String[] params = { repoVersion, iconsDir, Utils.FALLBACK_ICONS_DIR, repoVersion, iconsDirLarge, Utils.FALLBACK_ICONS_DIR, @@ -982,10 +985,8 @@ public class AppProvider extends FDroidProvider { * 1) The repo version that introduced density specific icons * 2) The dir to density specific icons for the current device. */ - private static String getIconUpdateQuery() { + private static String getIconUpdateQuery(String app, String apk) { - final String apk = DBHelper.TABLE_APK; - final String app = DBHelper.TABLE_APP; final String repo = DBHelper.TABLE_REPO; final String iconUrlQuery = @@ -1022,9 +1023,9 @@ public class AppProvider extends FDroidProvider { // then join onto that instead. This will save from doing // a futher sub query for each app. " SELECT MAX(inner_apk.vercode) " + - " FROM fdroid_apk as inner_apk " + - " WHERE inner_apk.id = fdroid_apk.id ) " + - " AND fdroid_apk.repo = fdroid_repo._id "; + " FROM " + apk + " as inner_apk " + + " WHERE inner_apk.id = " + apk + ".id ) " + + " AND " + apk + ".repo = fdroid_repo._id "; return " UPDATE " + app + " SET " + diff --git a/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java b/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java index f1166b20a..858746e07 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java +++ b/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java @@ -51,6 +51,8 @@ public class RepoPersister { @NonNull private final Repo repo; + private boolean hasBeenInitialized; + @NonNull private final Context context; @@ -82,6 +84,17 @@ public class RepoPersister { } private void flushBufferToDb() throws RepoUpdater.UpdateException { + if (!hasBeenInitialized) { + // This is where we will store all of the metadata before commiting at the + // end of the process. This is due to the fact that we can't verify the cert + // the index was signed with until we've finished reading it - and we don't + // want to put stuff in the real database until we are sure it is from a + // trusted source. + TempAppProvider.Helper.init(context); + TempApkProvider.Helper.init(context); + hasBeenInitialized = true; + } + if (apksToSave.size() > 0 || appsToSave.size() > 0) { Log.d(TAG, "Flushing details of up to " + MAX_APP_BUFFER + " apps and their packages to the database."); flushAppsToDbInBatch(); diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java index 0079487b7..66148704f 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -17,6 +17,8 @@ public class TempApkProvider extends ApkProvider { private static final String PROVIDER_NAME = "TempApkProvider"; + static final String TABLE_TEMP_APK = "temp_fdroid_apk"; + private static final String PATH_INIT = "init"; private static final String PATH_COMMIT = "commit"; @@ -135,7 +137,7 @@ public class TempApkProvider extends ApkProvider { private void initTable() { write().execSQL("DROP TABLE IF EXISTS " + getTableName()); - write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APK); + write().execSQL("CREATE TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APK); } private void commitTable() { diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java index 2e6737ac9..a307a08f5 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java @@ -15,6 +15,8 @@ public class TempAppProvider extends AppProvider { private static final String PROVIDER_NAME = "TempAppProvider"; + private static final String TABLE_TEMP_APP = "temp_fdroid_app"; + private static final String PATH_INIT = "init"; private static final String PATH_COMMIT = "commit"; @@ -31,7 +33,7 @@ public class TempAppProvider extends AppProvider { @Override protected String getTableName() { - return "temp_" + super.getTableName(); + return TABLE_TEMP_APP; } public static String getAuthority() { @@ -68,6 +70,11 @@ public class TempAppProvider extends AppProvider { } + @Override + protected String getApkTableName() { + return TempApkProvider.TABLE_TEMP_APK; + } + @Override public Uri insert(Uri uri, ContentValues values) { int code = matcher.match(uri); @@ -76,6 +83,7 @@ public class TempAppProvider extends AppProvider { initTable(); return null; } else if (code == CODE_COMMIT) { + updateAppDetails(); commitTable(); return null; } else { @@ -104,7 +112,7 @@ public class TempAppProvider extends AppProvider { private void initTable() { write().execSQL("DROP TABLE IF EXISTS " + getTableName()); - write().execSQL("CREATE TEMPORARY TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APP); + write().execSQL("CREATE TABLE " + getTableName() + " AS SELECT * FROM " + DBHelper.TABLE_APP); } private void commitTable() { From 90290f830a7fcbb2b4ef0e109937846fc43bac11 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 9 Nov 2015 23:01:36 +1100 Subject: [PATCH 09/17] Send "inserting" message to notifications while committing temp provider. Right now it says "50%" always, will need to think whether to ditch the percentage completely, or to have the temp app/apk providers emit progress events some how too. --- F-Droid/src/org/fdroid/fdroid/RepoUpdater.java | 1 + F-Droid/src/org/fdroid/fdroid/UpdateService.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 1d36baee5..2658b4514 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -53,6 +53,7 @@ public class RepoUpdater { private static final String TAG = "RepoUpdater"; public static final String PROGRESS_TYPE_PROCESS_XML = "processingXml"; + public static final String PROGRESS_COMMITTING = "committing"; public static final String PROGRESS_DATA_REPO_ADDRESS = "repoAddress"; @NonNull diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index b618a2455..11effd97a 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -505,8 +505,6 @@ public class UpdateService extends IntentService implements ProgressListener { @Override public void onProgress(ProgressListener.Event event) { String message = ""; - // TODO: Switch to passing through Bundles of data with the event, rather than a repo address. They are - // now much more general purpose then just repo downloading. String repoAddress = event.getData().getString(RepoUpdater.PROGRESS_DATA_REPO_ADDRESS); String downloadedSize = Utils.getFriendlySize(event.progress); String totalSize = Utils.getFriendlySize(event.total); @@ -515,6 +513,9 @@ public class UpdateService extends IntentService implements ProgressListener { case RepoUpdater.PROGRESS_TYPE_PROCESS_XML: message = getString(R.string.status_processing_xml_percent, repoAddress, downloadedSize, totalSize, percent); break; + case RepoUpdater.PROGRESS_COMMITTING: + message = getString(R.string.status_inserting, 50); + break; } sendStatus(this, STATUS_INFO, message, percent); } From 6969dcb90edf14f1ad0a864a5289de2943461202 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Wed, 25 Nov 2015 17:57:21 +1100 Subject: [PATCH 10/17] Change "inserting" string to not include progress for last phase of update. --- F-Droid/res/values/strings.xml | 2 +- F-Droid/src/org/fdroid/fdroid/UpdateService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/F-Droid/res/values/strings.xml b/F-Droid/res/values/strings.xml index 0c1b7fb40..906e0b8c2 100644 --- a/F-Droid/res/values/strings.xml +++ b/F-Droid/res/values/strings.xml @@ -190,7 +190,7 @@ Processing %2$s / %3$s (%4$d%%) from %1$s Connecting to\n%1$s Checking apps compatibility with your device… - Saving application details (%1$d%%) + Saving application details All repositories are up to date All other repos didn\'t create errors. Error during update: %s diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index 11effd97a..fc1d26af5 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -514,7 +514,7 @@ public class UpdateService extends IntentService implements ProgressListener { message = getString(R.string.status_processing_xml_percent, repoAddress, downloadedSize, totalSize, percent); break; case RepoUpdater.PROGRESS_COMMITTING: - message = getString(R.string.status_inserting, 50); + message = getString(R.string.status_inserting_apps); break; } sendStatus(this, STATUS_INFO, message, percent); From 8a6a62833b2491e854193ddc1a8915da1711da72 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Tue, 17 Nov 2015 18:54:26 +1100 Subject: [PATCH 11/17] Alert update notification when saving details to DB. --- F-Droid/src/org/fdroid/fdroid/RepoUpdater.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 2658b4514..f4919884d 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -196,10 +196,7 @@ public class RepoUpdater { // JarEntry can only read certificates after the file represented by that JarEntry // has been read completely, so verification cannot run until now... assertSigningCertFromXmlCorrect(); - - Log.i(TAG, "Repo signature verified, saving app metadata to database."); - persister.commit(repoDetailsToSave); - + commitToDb(); } catch (SAXException | ParserConfigurationException | IOException e) { throw new UpdateException(repo, "Error parsing index", e); } finally { @@ -213,6 +210,14 @@ public class RepoUpdater { } } + private void commitToDb() throws UpdateException { + Log.i(TAG, "Repo signature verified, saving app metadata to database."); + if (progressListener != null) { + progressListener.onProgress(new ProgressListener.Event(PROGRESS_COMMITTING)); + } + persister.commit(repoDetailsToSave); + } + private void assertSigningCertFromXmlCorrect() throws SigningException { // no signing cert read from database, this is the first use From c5bf2a131b335c2a02f2c3f9cacbebf965aef5a3 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Wed, 18 Nov 2015 06:19:21 +1100 Subject: [PATCH 12/17] Remove unused imports. --- F-Droid/src/org/fdroid/fdroid/RepoUpdater.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index f4919884d..8e3e64316 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -12,8 +12,6 @@ import org.fdroid.fdroid.data.App; import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoPersister; import org.fdroid.fdroid.data.RepoProvider; -import org.fdroid.fdroid.data.TempApkProvider; -import org.fdroid.fdroid.data.TempAppProvider; import org.fdroid.fdroid.net.Downloader; import org.fdroid.fdroid.net.DownloaderFactory; import org.xml.sax.InputSource; From ab33eccaa2e765048a99220740dc2e72e7341e7d Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 30 Nov 2015 18:02:38 +1100 Subject: [PATCH 13/17] Replaced if with switch. Fixed typo in error. --- .../org/fdroid/fdroid/data/RepoPersister.java | 4 ++-- .../fdroid/fdroid/data/TempApkProvider.java | 18 ++++++++-------- .../fdroid/fdroid/data/TempAppProvider.java | 21 +++++++++---------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java b/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java index 858746e07..c60913939 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java +++ b/F-Droid/src/org/fdroid/fdroid/data/RepoPersister.java @@ -122,7 +122,7 @@ public class RepoPersister { try { context.getContentResolver().applyBatch(TempApkProvider.getAuthority(), apkOperations); } catch (RemoteException | OperationApplicationException e) { - throw new RepoUpdater.UpdateException(repo, "An internal error occured while updating the database", e); + throw new RepoUpdater.UpdateException(repo, "An internal error occurred while updating the database", e); } } @@ -132,7 +132,7 @@ public class RepoPersister { try { context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations); } catch (RemoteException | OperationApplicationException e) { - throw new RepoUpdater.UpdateException(repo, "An internal error occured while updating the database", e); + throw new RepoUpdater.UpdateException(repo, "An internal error occurred while updating the database", e); } } diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java index 66148704f..352c341b1 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -89,15 +89,15 @@ public class TempApkProvider extends ApkProvider { @Override public Uri insert(Uri uri, ContentValues values) { - int code = matcher.match(uri); - if (code == CODE_INIT) { - initTable(); - return null; - } else if (code == CODE_COMMIT) { - commitTable(); - return null; - } else { - return super.insert(uri, values); + switch (matcher.match(uri)) { + case CODE_INIT: + initTable(); + return null; + case CODE_COMMIT: + commitTable(); + return null; + default: + return super.insert(uri, values); } } diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java index a307a08f5..7e5d67702 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java @@ -77,17 +77,16 @@ public class TempAppProvider extends AppProvider { @Override public Uri insert(Uri uri, ContentValues values) { - int code = matcher.match(uri); - - if (code == CODE_INIT) { - initTable(); - return null; - } else if (code == CODE_COMMIT) { - updateAppDetails(); - commitTable(); - return null; - } else { - return super.insert(uri, values); + switch (matcher.match(uri)) { + case CODE_INIT: + initTable(); + return null; + case CODE_COMMIT: + updateAppDetails(); + commitTable(); + return null; + default: + return super.insert(uri, values); } } From a3c91e4eae7e97f096f4503b1adbed2d01a66e69 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 30 Nov 2015 18:11:17 +1100 Subject: [PATCH 14/17] Reformatted some annotations to be inline. --- F-Droid/src/org/fdroid/fdroid/RepoUpdater.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 8e3e64316..d363fb0de 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -54,18 +54,14 @@ public class RepoUpdater { public static final String PROGRESS_COMMITTING = "committing"; public static final String PROGRESS_DATA_REPO_ADDRESS = "repoAddress"; - @NonNull - protected final Context context; - @NonNull - protected final Repo repo; + @NonNull protected final Context context; + @NonNull protected final Repo repo; protected boolean hasChanged; - @Nullable - protected ProgressListener progressListener; + @Nullable protected ProgressListener progressListener; private String cacheTag; private X509Certificate signingCertFromJar; - @NonNull - private final RepoPersister persister; + @NonNull private final RepoPersister persister; /** * Updates an app repo as read out of the database into a {@link Repo} instance. From 096b9c20d12e27fd5c10b7c77284a9a75a6c6981 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 30 Nov 2015 18:13:12 +1100 Subject: [PATCH 15/17] Changed logging levels as per CR. --- F-Droid/src/org/fdroid/fdroid/RepoUpdater.java | 4 ++-- F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java | 2 +- F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index d363fb0de..aedbd07c8 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -111,7 +111,7 @@ public class RepoUpdater { } catch (IOException e) { if (downloader != null && downloader.getFile() != null) { if (!downloader.getFile().delete()) { - Log.i(TAG, "Couldn't delete file: " + downloader.getFile().getAbsolutePath()); + Log.w(TAG, "Couldn't delete file: " + downloader.getFile().getAbsolutePath()); } } @@ -198,7 +198,7 @@ public class RepoUpdater { Utils.closeQuietly(indexInputStream); if (downloadedFile != null) { if (!downloadedFile.delete()) { - Log.i(TAG, "Couldn't delete file: " + downloadedFile.getAbsolutePath()); + Log.w(TAG, "Couldn't delete file: " + downloadedFile.getAbsolutePath()); } } } diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java index 352c341b1..995a6f025 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempApkProvider.java @@ -141,7 +141,7 @@ public class TempApkProvider extends ApkProvider { } private void commitTable() { - Log.d(TAG, "Deleting all apks from " + DBHelper.TABLE_APK + " so they can be copied from " + getTableName()); + Log.i(TAG, "Deleting all apks from " + DBHelper.TABLE_APK + " so they can be copied from " + getTableName()); write().execSQL("DELETE FROM " + DBHelper.TABLE_APK); write().execSQL("INSERT INTO " + DBHelper.TABLE_APK + " SELECT * FROM " + getTableName()); getContext().getContentResolver().notifyChange(ApkProvider.getContentUri(), null); diff --git a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java index 7e5d67702..391265d01 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/TempAppProvider.java @@ -115,7 +115,7 @@ public class TempAppProvider extends AppProvider { } private void commitTable() { - Log.d(TAG, "Deleting all apks from " + DBHelper.TABLE_APP + " so they can be copied from " + getTableName()); + Log.i(TAG, "Deleting all apks from " + DBHelper.TABLE_APP + " so they can be copied from " + getTableName()); write().execSQL("DELETE FROM " + DBHelper.TABLE_APP); write().execSQL("INSERT INTO " + DBHelper.TABLE_APP + " SELECT * FROM " + getTableName()); getContext().getContentResolver().notifyChange(AppProvider.getContentUri(), null); From 77ee4296a597bfe488cf5f7e3d5d393f75e5aa74 Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 30 Nov 2015 18:19:34 +1100 Subject: [PATCH 16/17] Guard against divide by zero exceptions during progress events. --- F-Droid/src/org/fdroid/fdroid/UpdateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index fc1d26af5..aca75b31a 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -508,7 +508,7 @@ public class UpdateService extends IntentService implements ProgressListener { String repoAddress = event.getData().getString(RepoUpdater.PROGRESS_DATA_REPO_ADDRESS); String downloadedSize = Utils.getFriendlySize(event.progress); String totalSize = Utils.getFriendlySize(event.total); - int percent = (int) ((double) event.progress / event.total * 100); + int percent = event.total > 0 ? (int) ((double) event.progress / event.total * 100) : -1; switch (event.type) { case RepoUpdater.PROGRESS_TYPE_PROCESS_XML: message = getString(R.string.status_processing_xml_percent, repoAddress, downloadedSize, totalSize, percent); From 959fe9f2e3aaba6184ed3bfcdfac3df499a8398d Mon Sep 17 00:00:00 2001 From: Peter Serwylo Date: Mon, 30 Nov 2015 18:46:09 +1100 Subject: [PATCH 17/17] Show indeterminate progress when persisting the temp apk table. Will also appear as indeterminate if: * The repo being downloaded from doesn't send a Content-Length header. * While connecting to the HTTP server to begin downloading. --- F-Droid/src/org/fdroid/fdroid/UpdateService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index aca75b31a..6ad914c59 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -237,6 +237,8 @@ public class UpdateService extends IntentService implements ProgressListener { .setCategory(NotificationCompat.CATEGORY_SERVICE); if (progress != -1) { notificationBuilder.setProgress(100, progress, false); + } else { + notificationBuilder.setProgress(100, 0, true); } notificationManager.notify(NOTIFY_ID_UPDATING, notificationBuilder.build()); break;