diff --git a/F-Droid/src/org/fdroid/fdroid/RepoPersister.java b/F-Droid/src/org/fdroid/fdroid/RepoPersister.java deleted file mode 100644 index 2c98929f9..000000000 --- a/F-Droid/src/org/fdroid/fdroid/RepoPersister.java +++ /dev/null @@ -1,329 +0,0 @@ -package org.fdroid.fdroid; - -import android.content.ContentProviderOperation; -import android.content.ContentValues; -import android.content.Context; -import android.content.OperationApplicationException; -import android.database.Cursor; -import android.net.Uri; -import android.os.RemoteException; -import android.support.annotation.NonNull; -import android.util.Log; - -import org.fdroid.fdroid.data.Apk; -import org.fdroid.fdroid.data.ApkProvider; -import org.fdroid.fdroid.data.App; -import org.fdroid.fdroid.data.AppProvider; -import org.fdroid.fdroid.data.Repo; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Saves app and apk information to the database after a {@link RepoUpdater} has processed the - * relevant index file. - */ -public class RepoPersister { - - private static final String TAG = "RepoPersister"; - - /** - * When an app already exists in the db, and we are updating it on the off chance that some - * values changed in the index, some fields should not be updated. Rather, they should be - * ignored, because they were explicitly set by the user, and hence can't be automatically - * overridden by the index. - * - * NOTE: In the future, these attributes will be moved to a join table, so that the app table - * is essentially completely transient, and can be nuked at any time. - */ - private static final String[] APP_FIELDS_TO_IGNORE = { - AppProvider.DataColumns.IGNORE_ALLUPDATES, - AppProvider.DataColumns.IGNORE_THISUPDATE, - }; - - @NonNull - private final Context context; - - private final Map appsToUpdate = new HashMap<>(); - private final List apksToUpdate = new ArrayList<>(); - private final List repos = new ArrayList<>(); - - public RepoPersister(@NonNull Context context) { - this.context = context; - } - - public RepoPersister queueUpdater(RepoUpdater updater) { - queueApps(updater.getApps()); - queueApks(updater.getApks()); - repos.add(updater.repo); - return this; - } - - private void queueApps(List apps) { - for (final App app : apps) { - appsToUpdate.put(app.id, app); - } - } - - private void queueApks(List apks) { - apksToUpdate.addAll(apks); - } - - public void save(List disabledRepos) { - - List listOfAppsToUpdate = new ArrayList<>(); - listOfAppsToUpdate.addAll(appsToUpdate.values()); - - calcApkCompatibilityFlags(apksToUpdate); - - // Need to do this BEFORE updating the apks, otherwise when it continually - // calls "get existing apks for repo X" then it will be getting the newly - // created apks, rather than those from the fresh, juicy index we just processed. - removeApksNoLongerInRepo(apksToUpdate, repos); - - int totalInsertsUpdates = listOfAppsToUpdate.size() + apksToUpdate.size(); - updateOrInsertApps(listOfAppsToUpdate, totalInsertsUpdates, 0); - updateOrInsertApks(apksToUpdate, totalInsertsUpdates, listOfAppsToUpdate.size()); - removeApksFromRepos(disabledRepos); - removeAppsWithoutApks(); - - // This will sort out the icon urls, compatibility flags. and suggested version - // for each app. It used to happen here in Java code, but was moved to SQL when - // it became apparant we don't always have enough info (depending on which repos - // were updated). - AppProvider.Helper.calcDetailsFromIndex(context); - - } - - /** - * This cannot be offloaded to the database (as we did with the query which - * updates apps, depending on whether their apks are compatible or not). - * The reason is that we need to interact with the CompatibilityChecker - * in order to see if, and why an apk is not compatible. - */ - private void calcApkCompatibilityFlags(List apks) { - final CompatibilityChecker checker = new CompatibilityChecker(context); - for (final Apk apk : apks) { - final List reasons = checker.getIncompatibleReasons(apk); - if (reasons.size() > 0) { - apk.compatible = false; - apk.incompatibleReasons = Utils.CommaSeparatedList.make(reasons); - } else { - apk.compatible = true; - apk.incompatibleReasons = null; - } - } - } - - /** - * If a repo was updated (i.e. it is in use, and the index has changed - * since last time we did an update), then we want to remove any apks that - * belong to the repo which are not in the current list of apks that were - * retrieved. - */ - private void removeApksNoLongerInRepo(List apksToUpdate, List updatedRepos) { - - long startTime = System.currentTimeMillis(); - List toRemove = new ArrayList<>(); - - final String[] fields = { - ApkProvider.DataColumns.APK_ID, - ApkProvider.DataColumns.VERSION_CODE, - ApkProvider.DataColumns.VERSION, - }; - - for (final Repo repo : updatedRepos) { - final List existingApks = ApkProvider.Helper.findByRepo(context, repo, fields); - for (final Apk existingApk : existingApks) { - if (!isApkToBeUpdated(existingApk, apksToUpdate)) { - toRemove.add(existingApk); - } - } - } - - long duration = System.currentTimeMillis() - startTime; - Utils.debugLog(TAG, "Found " + toRemove.size() + " apks no longer in the updated repos (took " + duration + "ms)"); - - if (toRemove.size() > 0) { - ApkProvider.Helper.deleteApks(context, toRemove); - } - } - - private void updateOrInsertApps(List appsToUpdate, int totalUpdateCount, int currentCount) { - - List operations = new ArrayList<>(); - List knownAppIds = getKnownAppIds(appsToUpdate); - for (final App app : appsToUpdate) { - if (knownAppIds.contains(app.id)) { - operations.add(updateExistingApp(app)); - } else { - operations.add(insertNewApp(app)); - } - } - - Utils.debugLog(TAG, "Updating/inserting " + operations.size() + " apps."); - try { - executeBatchWithStatus(AppProvider.getAuthority(), operations, currentCount, totalUpdateCount); - } catch (RemoteException | OperationApplicationException e) { - Log.e(TAG, "Could not update or insert apps", e); - } - } - - private void executeBatchWithStatus(String providerAuthority, - List operations, - int currentCount, - int totalUpdateCount) - throws RemoteException, OperationApplicationException { - int i = 0; - while (i < operations.size()) { - int count = Math.min(operations.size() - i, 100); - int progress = (int) ((double) (currentCount + i) / totalUpdateCount * 100); - ArrayList o = new ArrayList<>(operations.subList(i, i + count)); - UpdateService.sendStatus(context, UpdateService.STATUS_INFO, context.getString( - R.string.status_inserting, progress), progress); - context.getContentResolver().applyBatch(providerAuthority, o); - i += 100; - } - } - - /** - * Return list of apps from the "apks" argument which are already in the database. - */ - private List getKnownApks(List apks) { - final String[] fields = { - ApkProvider.DataColumns.APK_ID, - ApkProvider.DataColumns.VERSION, - ApkProvider.DataColumns.VERSION_CODE, - }; - return ApkProvider.Helper.knownApks(context, apks, fields); - } - - private void updateOrInsertApks(List apksToUpdate, int totalApksAppsCount, int currentCount) { - - List operations = new ArrayList<>(); - - List knownApks = getKnownApks(apksToUpdate); - for (final Apk apk : apksToUpdate) { - boolean known = false; - for (final Apk knownApk : knownApks) { - if (knownApk.id.equals(apk.id) && knownApk.vercode == apk.vercode) { - known = true; - break; - } - } - - if (known) { - operations.add(updateExistingApk(apk)); - } else { - operations.add(insertNewApk(apk)); - knownApks.add(apk); // In case another repo has the same version/id combo for this apk. - } - } - - Utils.debugLog(TAG, "Updating/inserting " + operations.size() + " apks."); - try { - executeBatchWithStatus(ApkProvider.getAuthority(), operations, currentCount, totalApksAppsCount); - } catch (RemoteException | OperationApplicationException e) { - Log.e(TAG, "Could not update/insert apps", e); - } - } - - private ContentProviderOperation updateExistingApk(final Apk apk) { - Uri uri = ApkProvider.getContentUri(apk); - ContentValues values = apk.toContentValues(); - return ContentProviderOperation.newUpdate(uri).withValues(values).build(); - } - - private ContentProviderOperation insertNewApk(final Apk apk) { - ContentValues values = apk.toContentValues(); - Uri uri = ApkProvider.getContentUri(); - return ContentProviderOperation.newInsert(uri).withValues(values).build(); - } - - private ContentProviderOperation updateExistingApp(App app) { - Uri uri = AppProvider.getContentUri(app); - ContentValues values = app.toContentValues(); - for (final String toIgnore : APP_FIELDS_TO_IGNORE) { - if (values.containsKey(toIgnore)) { - values.remove(toIgnore); - } - } - return ContentProviderOperation.newUpdate(uri).withValues(values).build(); - } - - private ContentProviderOperation insertNewApp(App app) { - ContentValues values = app.toContentValues(); - Uri uri = AppProvider.getContentUri(); - return ContentProviderOperation.newInsert(uri).withValues(values).build(); - } - - private static boolean isApkToBeUpdated(Apk existingApk, List apksToUpdate) { - for (final Apk apkToUpdate : apksToUpdate) { - if (apkToUpdate.vercode == existingApk.vercode && apkToUpdate.id.equals(existingApk.id)) { - return true; - } - } - return false; - } - - private void removeApksFromRepos(List repos) { - for (final Repo repo : repos) { - Uri uri = ApkProvider.getRepoUri(repo.getId()); - int numDeleted = context.getContentResolver().delete(uri, null, null); - Utils.debugLog(TAG, "Removing " + numDeleted + " apks from repo " + repo.address); - } - } - - private void removeAppsWithoutApks() { - int numDeleted = context.getContentResolver().delete(AppProvider.getNoApksUri(), null, null); - Utils.debugLog(TAG, "Removing " + numDeleted + " apks that don't have any apks"); - } - - private List getKnownAppIds(List apps) { - List knownAppIds = new ArrayList<>(); - if (apps.isEmpty()) { - return knownAppIds; - } - if (apps.size() > AppProvider.MAX_APPS_TO_QUERY) { - int middle = apps.size() / 2; - List apps1 = apps.subList(0, middle); - List apps2 = apps.subList(middle, apps.size()); - knownAppIds.addAll(getKnownAppIds(apps1)); - knownAppIds.addAll(getKnownAppIds(apps2)); - } else { - knownAppIds.addAll(getKnownAppIdsFromProvider(apps)); - } - return knownAppIds; - } - - /** - * Looks in the database to see which apps we already know about. Only - * returns ids of apps that are in the database if they are in the "apps" - * array. - */ - private List getKnownAppIdsFromProvider(List apps) { - - final Uri uri = AppProvider.getContentUri(apps); - final String[] fields = {AppProvider.DataColumns.APP_ID}; - Cursor cursor = context.getContentResolver().query(uri, fields, null, null, null); - - int knownIdCount = cursor != null ? cursor.getCount() : 0; - List knownIds = new ArrayList<>(knownIdCount); - if (cursor != null) { - if (knownIdCount > 0) { - cursor.moveToFirst(); - while (!cursor.isAfterLast()) { - knownIds.add(cursor.getString(0)); - cursor.moveToNext(); - } - } - cursor.close(); - } - - return knownIds; - } - -} - diff --git a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java index 45d90a9d0..97c935e7a 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoUpdater.java @@ -1,13 +1,20 @@ package org.fdroid.fdroid; +import android.content.ContentProviderOperation; import android.content.ContentValues; import android.content.Context; +import android.content.OperationApplicationException; +import android.net.Uri; +import android.os.RemoteException; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.text.TextUtils; +import android.util.Log; import org.fdroid.fdroid.data.Apk; +import org.fdroid.fdroid.data.ApkProvider; import org.fdroid.fdroid.data.App; +import org.fdroid.fdroid.data.AppProvider; import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.net.Downloader; @@ -26,7 +33,9 @@ import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -35,8 +44,12 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; /** - * Handles getting the index metadata for an app repo, then verifying the - * signature on the index metdata, implementing as a JAR signature. + * + * Responsible for updating an individual repository. This will: + * * Download the index.jar + * * Verify that it is signed correctly and by the correct certificate + * * Parse the index.xml from the .jar file + * * Save the resulting repo, apps, and apks to the database. * * WARNING: this class is the central piece of the entire security model of * FDroid! Avoid modifying it when possible, if you absolutely must, be very, @@ -49,13 +62,26 @@ public class RepoUpdater { public static final String PROGRESS_TYPE_PROCESS_XML = "processingXml"; public static final String PROGRESS_DATA_REPO_ADDRESS = "repoAddress"; + /** + * When an app already exists in the db, and we are updating it on the off chance that some + * values changed in the index, some fields should not be updated. Rather, they should be + * ignored, because they were explicitly set by the user, and hence can't be automatically + * overridden by the index. + * + * NOTE: In the future, these attributes will be moved to a join table, so that the app table + * is essentially completely transient, and can be nuked at any time. + */ + private static final String[] APP_FIELDS_TO_IGNORE = { + AppProvider.DataColumns.IGNORE_ALLUPDATES, + AppProvider.DataColumns.IGNORE_THISUPDATE + }; + @NonNull protected final Context context; @NonNull protected final Repo repo; - private List apps = new ArrayList<>(); - private List apks = new ArrayList<>(); - private RepoUpdateRememberer rememberer; - protected boolean hasChanged; + protected boolean hasChanged = false; @Nullable protected ProgressListener progressListener; + private String cacheTag; + private X509Certificate signingCertFromJar; /** * Updates an app repo as read out of the database into a {@link Repo} instance. @@ -74,15 +100,7 @@ public class RepoUpdater { return hasChanged; } - public List getApps() { - return apps; - } - - public List getApks() { - return apks; - } - - private URL getIndexAddress() throws MalformedURLException { + protected URL getIndexAddress() throws MalformedURLException { String urlString = repo.address + "/index.jar"; String versionName = Utils.getVersionName(context); if (versionName != null) { @@ -110,7 +128,9 @@ public class RepoUpdater { } catch (IOException e) { if (downloader != null && downloader.getFile() != null) { - downloader.getFile().delete(); + if (!downloader.getFile().delete()) { + Log.i(TAG, "Couldn't delete file: " + downloader.getFile().getAbsolutePath()); + } } throw new UpdateException(repo, "Error getting index file", e); @@ -133,11 +153,257 @@ public class RepoUpdater { if (hasChanged) { // Don't worry about checking the status code for 200. If it was a // successful download, then we will have a file ready to use: - processDownloadedFile(downloader.getFile(), downloader.getCacheTag()); + cacheTag = downloader.getCacheTag(); + processDownloadedFile(downloader.getFile()); } } - protected void processDownloadedFile(File downloadedFile, String cacheTag) throws UpdateException { + private ContentValues repoDetailsToSave = null; + private String signingCertFromIndexXml = null; + + private RepoXMLHandler.IndexReceiver createIndexReceiver() { + return new RepoXMLHandler.IndexReceiver() { + @Override + public void receiveRepo(String name, String description, String signingCert, int maxAge, int version) { + signingCertFromIndexXml = signingCert; + repoDetailsToSave = prepareRepoDetailsForSaving(name, description, maxAge, version); + } + + @Override + public void receiveApp(App app, List packages) { + try { + saveToDb(app, packages); + } catch (UpdateException e) { + throw new RuntimeException("Error while saving repo details to database.", e); + } + } + }; + } + + /** + * My crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get: + * * 25 = { 39, 35 } seconds + * * 50 = { 36, 30 } seconds + * * 100 = { 33, 27 } seconds + * * 200 = { 30, 33 } seconds + */ + private static final int MAX_APP_BUFFER = 50; + + private List appsToSave = new ArrayList<>(); + private Map> apksToSave = new HashMap<>(); + + private void saveToDb(App app, List packages) throws UpdateException { + appsToSave.add(app); + apksToSave.put(app.id, packages); + + if (appsToSave.size() >= MAX_APP_BUFFER) { + flushBufferToDb(); + } + } + + private void flushBufferToDb() throws UpdateException { + Log.d(TAG, "Flushing details of " + MAX_APP_BUFFER + " and their packages to the database."); + flushAppsToDbInBatch(); + flushApksToDbInBatch(); + apksToSave.clear(); + appsToSave.clear(); + } + + private void flushApksToDbInBatch() throws UpdateException { + List apksToSaveList = new ArrayList<>(); + for (Map.Entry> entries : apksToSave.entrySet()) { + apksToSaveList.addAll(entries.getValue()); + } + + calcApkCompatibilityFlags(apksToSaveList); + + ArrayList apkOperations = new ArrayList<>(); + ContentProviderOperation clearOrphans = deleteOrphanedApks(appsToSave, apksToSave); + if (clearOrphans != null) { + apkOperations.add(clearOrphans); + } + apkOperations.addAll(insertOrUpdateApks(apksToSaveList)); + + try { + context.getContentResolver().applyBatch(ApkProvider.getAuthority(), apkOperations); + } catch (RemoteException | OperationApplicationException e) { + throw new UpdateException(repo, "An internal error occured while updating the database", e); + } + } + + private void flushAppsToDbInBatch() throws UpdateException { + ArrayList appOperations = insertOrUpdateApps(appsToSave); + + try { + context.getContentResolver().applyBatch(AppProvider.getAuthority(), appOperations); + } catch (RemoteException|OperationApplicationException e) { + Log.e(TAG, "Error updating apps", e); + throw new UpdateException(repo, "Error updating apps: " + e.getMessage(), e); + } + } + + /** + * Depending on whether the {@link App}s have been added to the database previously, this + * will queue up an update or an insert {@link ContentProviderOperation} for each app. + */ + private ArrayList insertOrUpdateApps(List apps) { + ArrayList operations = new ArrayList<>(apps.size()); + for (App app : apps) { + if (isAppInDatabase(app)) { + operations.add(updateExistingApp(app)); + } else { + operations.add(insertNewApp(app)); + } + } + return operations; + } + + /** + * Depending on whether the .apks have been added to the database previously, this + * will queue up an update or an insert {@link ContentProviderOperation} for each package. + */ + private ArrayList insertOrUpdateApks(List packages) { + List existingApks = ApkProvider.Helper.knownApks(context, packages, new String[]{ApkProvider.DataColumns.VERSION_CODE}); + ArrayList operations = new ArrayList<>(packages.size()); + for (Apk apk : packages) { + boolean exists = false; + for (Apk existing : existingApks) { + if (existing.vercode == apk.vercode) { + exists = true; + break; + } + } + + if (exists) { + operations.add(updateExistingApk(apk)); + } else { + operations.add(insertNewApk(apk)); + } + } + + return operations; + } + + /** + * Creates an update {@link ContentProviderOperation} for the {@link App} in question. + * Does not do any checks to see if the app already exists or not. + */ + private ContentProviderOperation updateExistingApp(App app) { + Uri uri = AppProvider.getContentUri(app); + ContentValues values = app.toContentValues(); + for (final String toIgnore : APP_FIELDS_TO_IGNORE) { + if (values.containsKey(toIgnore)) { + values.remove(toIgnore); + } + } + return ContentProviderOperation.newUpdate(uri).withValues(values).build(); + } + + /** + * Creates an insert {@link ContentProviderOperation} for the {@link App} in question. + * Does not do any checks to see if the app already exists or not. + */ + private ContentProviderOperation insertNewApp(App app) { + ContentValues values = app.toContentValues(); + Uri uri = AppProvider.getContentUri(); + return ContentProviderOperation.newInsert(uri).withValues(values).build(); + } + + /** + * Looks in the database to see which apps we already know about. Only + * returns ids of apps that are in the database if they are in the "apps" + * array. + */ + private boolean isAppInDatabase(App app) { + String[] fields = { AppProvider.DataColumns.APP_ID }; + App found = AppProvider.Helper.findById(context.getContentResolver(), app.id, fields); + return found != null; + } + + /** + * Creates an update {@link ContentProviderOperation} for the {@link Apk} in question. + * Does not do any checks to see if the apk already exists or not. + */ + private ContentProviderOperation updateExistingApk(final Apk apk) { + Uri uri = ApkProvider.getContentUri(apk); + ContentValues values = apk.toContentValues(); + return ContentProviderOperation.newUpdate(uri).withValues(values).build(); + } + + /** + * Creates an insert {@link ContentProviderOperation} for the {@link Apk} in question. + * Does not do any checks to see if the apk already exists or not. + */ + private ContentProviderOperation insertNewApk(final Apk apk) { + ContentValues values = apk.toContentValues(); + Uri uri = ApkProvider.getContentUri(); + return ContentProviderOperation.newInsert(uri).withValues(values).build(); + } + + /** + * Finds all apks from the repo we are currently updating, that belong to the specified app, + * and delete them as they are no longer provided by that repo. + */ + @Nullable + private ContentProviderOperation deleteOrphanedApks(List apps, Map> packages) { + + String[] projection = new String[]{ApkProvider.DataColumns.APK_ID, ApkProvider.DataColumns.VERSION_CODE}; + List existing = ApkProvider.Helper.find(context, repo, apps, projection); + + List toDelete = new ArrayList<>(); + + for (Apk existingApk : existing) { + + boolean shouldStay = false; + + for (Map.Entry> entry : packages.entrySet()) { + for (Apk newApk : entry.getValue()) { + if (newApk.vercode == existingApk.vercode) { + shouldStay = true; + break; + } + } + + if (shouldStay) { + break; + } + } + + if (!shouldStay) { + toDelete.add(existingApk); + } + } + + // TODO: Deal with more than MAX_QUERY_PARAMS... + if (toDelete.size() > 0) { + Uri uri = ApkProvider.getContentUriForApks(repo, toDelete); + return ContentProviderOperation.newDelete(uri).build(); + } else { + return null; + } + } + + /** + * This cannot be offloaded to the database (as we did with the query which + * updates apps, depending on whether their apks are compatible or not). + * The reason is that we need to interact with the CompatibilityChecker + * in order to see if, and why an apk is not compatible. + */ + public void calcApkCompatibilityFlags(List apks) { + final CompatibilityChecker checker = new CompatibilityChecker(context); + for (final Apk apk : apks) { + final List reasons = checker.getIncompatibleReasons(apk); + if (reasons.size() > 0) { + apk.compatible = false; + apk.incompatibleReasons = Utils.CommaSeparatedList.make(reasons); + } else { + apk.compatible = true; + apk.incompatibleReasons = null; + } + } + } + + public void processDownloadedFile(File downloadedFile) throws UpdateException { InputStream indexInputStream = null; try { if (downloadedFile == null || !downloadedFile.exists()) @@ -153,94 +419,77 @@ public class RepoUpdater { indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry), progressListener, repo, (int) indexEntry.getSize()); + /* JarEntry can only read certificates after the file represented by that JarEntry + * has been read completely, so verification cannot run until now... */ + // Process the index... final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); final XMLReader reader = parser.getXMLReader(); - final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo); + final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo, createIndexReceiver()); reader.setContentHandler(repoXMLHandler); reader.parse(new InputSource(indexInputStream)); + signingCertFromJar = getSigningCertFromJar(indexEntry); - /* JarEntry can only read certificates after the file represented by that JarEntry - * has been read completely, so verification cannot run until now... */ - X509Certificate certFromJar = getSigningCertFromJar(indexEntry); + assertSigningCertFromXmlCorrect(); + RepoProvider.Helper.update(context, repo, repoDetailsToSave); - String certFromIndexXml = repoXMLHandler.getSigningCertFromIndexXml(); - - // no signing cert read from database, this is the first use - if (repo.pubkey == null) { - verifyAndStoreTOFUCerts(certFromIndexXml, certFromJar); - } - verifyCerts(certFromIndexXml, certFromJar); - - apps = repoXMLHandler.getApps(); - apks = repoXMLHandler.getApks(); - - rememberer = new RepoUpdateRememberer(); - rememberer.context = context; - rememberer.repo = repo; - rememberer.values = prepareRepoDetailsForSaving(repoXMLHandler, cacheTag); } catch (SAXException | ParserConfigurationException | IOException e) { throw new UpdateException(repo, "Error parsing index", e); } finally { FDroidApp.enableSpongyCastleOnLollipop(); Utils.closeQuietly(indexInputStream); if (downloadedFile != null) { - downloadedFile.delete(); + if (!downloadedFile.delete()) { + Log.i(TAG, "Couldn't delete file: " + downloadedFile.getAbsolutePath()); + } } } } + private void assertSigningCertFromXmlCorrect() throws SigningException { + + // no signing cert read from database, this is the first use + if (repo.pubkey == null) { + verifyAndStoreTOFUCerts(signingCertFromIndexXml, signingCertFromJar); + } + verifyCerts(signingCertFromIndexXml, signingCertFromJar); + + } + /** * Update tracking data for the repo represented by this instance (index version, etag, * description, human-readable name, etc. */ - private ContentValues prepareRepoDetailsForSaving(RepoXMLHandler handler, String etag) { + private ContentValues prepareRepoDetailsForSaving(String name, String description, int maxAge, int version) { ContentValues values = new ContentValues(); values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatTime(new Date(), "")); - if (repo.lastetag == null || !repo.lastetag.equals(etag)) { - values.put(RepoProvider.DataColumns.LAST_ETAG, etag); + if (repo.lastetag == null || !repo.lastetag.equals(cacheTag)) { + values.put(RepoProvider.DataColumns.LAST_ETAG, cacheTag); } - if (handler.getVersion() != -1 && handler.getVersion() != repo.version) { - Utils.debugLog(TAG, "Repo specified a new version: from " - + repo.version + " to " + handler.getVersion()); - values.put(RepoProvider.DataColumns.VERSION, handler.getVersion()); + if (version != -1 && version != repo.version) { + Utils.debugLog(TAG, "Repo specified a new version: from " + repo.version + " to " + version); + values.put(RepoProvider.DataColumns.VERSION, version); } - if (handler.getMaxAge() != -1 && handler.getMaxAge() != repo.maxage) { + if (maxAge != -1 && maxAge != repo.maxage) { Utils.debugLog(TAG, "Repo specified a new maximum age - updated"); - values.put(RepoProvider.DataColumns.MAX_AGE, handler.getMaxAge()); + values.put(RepoProvider.DataColumns.MAX_AGE, maxAge); } - if (handler.getDescription() != null && !handler.getDescription().equals(repo.description)) { - values.put(RepoProvider.DataColumns.DESCRIPTION, handler.getDescription()); + if (description != null && !description.equals(repo.description)) { + values.put(RepoProvider.DataColumns.DESCRIPTION, description); } - if (handler.getName() != null && !handler.getName().equals(repo.name)) { - values.put(RepoProvider.DataColumns.NAME, handler.getName()); + if (name != null && !name.equals(repo.name)) { + values.put(RepoProvider.DataColumns.NAME, name); } return values; } - public RepoUpdateRememberer getRememberer() { - return rememberer; - } - - public static class RepoUpdateRememberer { - - private Context context; - private Repo repo; - private ContentValues values; - - public void rememberUpdate() { - RepoProvider.Helper.update(context, repo, values); - } - - } - public static class UpdateException extends Exception { private static final long serialVersionUID = -4492452418826132803L; @@ -257,23 +506,29 @@ public class RepoUpdater { } } + public static class SigningException extends UpdateException { + public SigningException(Repo repo, String message) { + super(repo, "Repository was not signed correctly: " + message); + } + } + /** * FDroid's index.jar is signed using a particular format and does not allow lots of * signing setups that would be valid for a regular jar. This validates those * restrictions. */ - private X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws UpdateException { + private X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws SigningException { final CodeSigner[] codeSigners = jarEntry.getCodeSigners(); if (codeSigners == null || codeSigners.length == 0) { - throw new UpdateException(repo, "No signature found in index"); + throw new SigningException(repo, "No signature found in index"); } /* we could in theory support more than 1, but as of now we do not */ if (codeSigners.length > 1) { - throw new UpdateException(repo, "index.jar must be signed by a single code signer!"); + throw new SigningException(repo, "index.jar must be signed by a single code signer!"); } List certs = codeSigners[0].getSignerCertPath().getCertificates(); if (certs.size() != 1) { - throw new UpdateException(repo, "index.jar code signers must only have a single certificate!"); + throw new SigningException(repo, "index.jar code signers must only have a single certificate!"); } return (X509Certificate) certs.get(0); } @@ -285,7 +540,7 @@ public class RepoUpdater { * check that the signing certificate in the jar matches that fingerprint. */ private void verifyAndStoreTOFUCerts(String certFromIndexXml, X509Certificate rawCertFromJar) - throws UpdateException { + throws SigningException { if (repo.pubkey != null) return; // there is a repo.pubkey already, nothing to TOFU @@ -293,30 +548,20 @@ public class RepoUpdater { * fingerprint. In that case, check that fingerprint against what is * actually in the index.jar itself. If no fingerprint, just store the * signing certificate */ - boolean trustNewSigningCertificate; - // If the fingerprint has never been set, it will be null (never "" or something else) - if (repo.fingerprint == null) { - // no info to check things are valid, so just Trust On First Use - trustNewSigningCertificate = true; - } else { + if (repo.fingerprint != null) { String fingerprintFromIndexXml = Utils.calcFingerprint(certFromIndexXml); String fingerprintFromJar = Utils.calcFingerprint(rawCertFromJar); - if (repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml) - && repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) { - trustNewSigningCertificate = true; - } else { - throw new UpdateException(repo, "Supplied certificate fingerprint does not match: '" - + repo.fingerprint + "' '" + fingerprintFromIndexXml + "' '" + fingerprintFromJar + "'"); + if (!repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml) + || !repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) { + throw new SigningException(repo, "Supplied certificate fingerprint does not match!"); } - } + } // else - no info to check things are valid, so just Trust On First Use - if (trustNewSigningCertificate) { - Utils.debugLog(TAG, "Saving new signing certificate in the database for " + repo.address); - ContentValues values = new ContentValues(2); - values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatTime(new Date(), "")); - values.put(RepoProvider.DataColumns.PUBLIC_KEY, Hasher.hex(rawCertFromJar)); - RepoProvider.Helper.update(context, repo, values); - } + Utils.debugLog(TAG, "Saving new signing certificate in the database for " + repo.address); + ContentValues values = new ContentValues(2); + values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatDate(new Date(), "")); + values.put(RepoProvider.DataColumns.PUBLIC_KEY, Hasher.hex(rawCertFromJar)); + RepoProvider.Helper.update(context, repo, values); } /** @@ -331,16 +576,15 @@ public class RepoUpdater { * @param certFromIndexXml the cert written into the header of the index XML * @param rawCertFromJar the {@link X509Certificate} embedded in the downloaded jar */ - private void verifyCerts(String certFromIndexXml, X509Certificate rawCertFromJar) throws UpdateException { + private void verifyCerts(String certFromIndexXml, X509Certificate rawCertFromJar) throws SigningException { // convert binary data to string version that is used in FDroid's database String certFromJar = Hasher.hex(rawCertFromJar); // repo and repo.pubkey must be pre-loaded from the database - if (repo == null - || TextUtils.isEmpty(repo.pubkey) + if (TextUtils.isEmpty(repo.pubkey) || TextUtils.isEmpty(certFromJar) || TextUtils.isEmpty(certFromIndexXml)) - throw new UpdateException(repo, "A empty repo or signing certificate is invalid!"); + throw new SigningException(repo, "A empty repo or signing certificate is invalid!"); // though its called repo.pubkey, its actually a X509 certificate if (repo.pubkey.equals(certFromJar) @@ -348,7 +592,7 @@ public class RepoUpdater { && certFromIndexXml.equals(certFromJar)) { return; // we have a match! } - throw new UpdateException(repo, "Signing certificate does not match!"); + throw new SigningException(repo, "Signing certificate does not match!"); } } diff --git a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java index 210bc714b..c0dd841f0 100644 --- a/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java +++ b/F-Droid/src/org/fdroid/fdroid/RepoXMLHandler.java @@ -19,6 +19,9 @@ package org.fdroid.fdroid; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + import org.fdroid.fdroid.data.Apk; import org.fdroid.fdroid.data.App; import org.fdroid.fdroid.data.Repo; @@ -31,61 +34,52 @@ import java.util.List; /** * Parses the index.xml into Java data structures. + * + * For streaming apks from an index file, it is helpful if the index has the tag before + * any tags. This means that apps and apks can be saved instantly by the RepoUpdater, + * without having to buffer them at all, saving memory. The XML spec doesn't mandate order like + * this, though it is almost always a fair assumption: + * + * http://www.ibm.com/developerworks/library/x-eleord/index.html + * + * This is doubly so, as repo indices are likely from fdroidserver, which will output everybodys + * repo the same way. Having said that, this also should not be _forced_ upon people, but we can + * at least consider rejecting malformed indexes. */ public class RepoXMLHandler extends DefaultHandler { // The repo we're processing. private final Repo repo; - private final List apps = new ArrayList<>(); - private final List apksList = new ArrayList<>(); + private List apksList = new ArrayList<>(); private App curapp; private Apk curapk; - private final StringBuilder curchars = new StringBuilder(); + + private String currentApkHashType = null; // After processing the XML, these will be -1 if the index didn't specify // them - otherwise it will be the value specified. - private int version = -1; - private int maxage = -1; + private int repoMaxAge = -1; + private int repoVersion = 0; + private String repoDescription; + private String repoName; - /** the X.509 signing certificate stored in the header of index.xml */ - private String signingCertFromIndexXml; + // the X.509 signing certificate stored in the header of index.xml + private String repoSigningCert; - private String name; - private String description; - private String hashType; + private final StringBuilder curchars = new StringBuilder(); - public RepoXMLHandler(Repo repo) { + interface IndexReceiver { + void receiveRepo(String name, String description, String signingCert, int maxage, int version); + void receiveApp(App app, List packages); + } + + private IndexReceiver receiver; + + public RepoXMLHandler(Repo repo, @NonNull IndexReceiver receiver) { this.repo = repo; - } - - public List getApps() { - return apps; - } - - public List getApks() { - return apksList; - } - - public int getMaxAge() { - return maxage; - } - - public int getVersion() { - return version; - } - - public String getDescription() { - return description; - } - - public String getName() { - return name; - } - - public String getSigningCertFromIndexXml() { - return signingCertFromIndexXml; + this.receiver = receiver; } @Override @@ -97,22 +91,13 @@ public class RepoXMLHandler extends DefaultHandler { public void endElement(String uri, String localName, String qName) throws SAXException { - super.endElement(uri, localName, qName); - if ("application".equals(localName) && curapp != null) { - apps.add(curapp); - curapp = null; - // If the app id is already present in this apps list, then it - // means the same index file has a duplicate app, which should - // not be allowed. - // However, I'm thinking that it should be unefined behaviour, - // because it is probably a bug in the fdroid server that made it - // happen, and I don't *think* it will crash the client, because - // the first app will insert, the second one will update the newly - // inserted one. + onApplicationParsed(); } else if ("package".equals(localName) && curapk != null && curapp != null) { apksList.add(curapk); curapk = null; + } else if ("repo".equals(localName)) { + onRepoParsed(); } else if (curchars.length() == 0) { // All options below require non-empty content return; @@ -120,53 +105,53 @@ public class RepoXMLHandler extends DefaultHandler { final String str = curchars.toString().trim(); if (curapk != null) { switch (localName) { - case "version": - curapk.version = str; - break; - case "versioncode": - curapk.vercode = Utils.parseInt(str, -1); - break; - case "size": - curapk.size = Utils.parseInt(str, 0); - break; - case "hash": - if (hashType == null || "md5".equals(hashType)) { - if (curapk.hash == null) { - curapk.hash = str; - curapk.hashType = "MD5"; - } - } else if ("sha256".equals(hashType)) { + case "version": + curapk.version = str; + break; + case "versioncode": + curapk.vercode = Utils.parseInt(str, -1); + break; + case "size": + curapk.size = Utils.parseInt(str, 0); + break; + case "hash": + if (currentApkHashType == null || currentApkHashType.equals("md5")) { + if (curapk.hash == null) { curapk.hash = str; curapk.hashType = "SHA-256"; } - break; - case "sig": - curapk.sig = str; - break; - case "srcname": - curapk.srcname = str; - break; - case "apkname": - curapk.apkName = str; - break; - case "sdkver": - curapk.minSdkVersion = Utils.parseInt(str, 0); - break; - case "maxsdkver": - curapk.maxSdkVersion = Utils.parseInt(str, 0); - break; - case "added": - curapk.added = Utils.parseDate(str, null); - break; - case "permissions": - curapk.permissions = Utils.CommaSeparatedList.make(str); - break; - case "features": - curapk.features = Utils.CommaSeparatedList.make(str); - break; - case "nativecode": - curapk.nativecode = Utils.CommaSeparatedList.make(str); - break; + } else if (currentApkHashType.equals("sha256")) { + curapk.hash = str; + curapk.hashType = "SHA-256"; + } + break; + case "sig": + curapk.sig = str; + break; + case "srcname": + curapk.srcname = str; + break; + case "apkname": + curapk.apkName = str; + break; + case "sdkver": + curapk.minSdkVersion = Utils.parseInt(str, 0); + break; + case "maxsdkver": + curapk.maxSdkVersion = Utils.parseInt(str, 0); + break; + case "added": + curapk.added = Utils.parseDate(str, null); + break; + case "permissions": + curapk.permissions = Utils.CommaSeparatedList.make(str); + break; + case "features": + curapk.features = Utils.CommaSeparatedList.make(str); + break; + case "nativecode": + curapk.nativecode = Utils.CommaSeparatedList.make(str); + break; } } else if (curapp != null) { switch (localName) { @@ -239,27 +224,39 @@ public class RepoXMLHandler extends DefaultHandler { break; } } else if ("description".equals(localName)) { - description = cleanWhiteSpace(str); + repoDescription = cleanWhiteSpace(str); } } + private void onApplicationParsed() { + receiver.receiveApp(curapp, apksList); + curapp = null; + apksList = new ArrayList<>(); + // If the app id is already present in this apps list, then it + // means the same index file has a duplicate app, which should + // not be allowed. + // However, I'm thinking that it should be undefined behaviour, + // because it is probably a bug in the fdroid server that made it + // happen, and I don't *think* it will crash the client, because + // the first app will insert, the second one will update the newly + // inserted one. + } + + private void onRepoParsed() { + receiver.receiveRepo(repoName, repoDescription, repoSigningCert, repoMaxAge, repoVersion); + } + @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { super.startElement(uri, localName, qName, attributes); if ("repo".equals(localName)) { - signingCertFromIndexXml = attributes.getValue("", "pubkey"); - maxage = Utils.parseInt(attributes.getValue("", "maxage"), -1); - version = Utils.parseInt(attributes.getValue("", "version"), -1); - - final String nm = attributes.getValue("", "name"); - if (nm != null) - name = cleanWhiteSpace(nm); - final String dc = attributes.getValue("", "description"); - if (dc != null) - description = cleanWhiteSpace(dc); - + repoSigningCert = attributes.getValue("", "pubkey"); + repoMaxAge = Utils.parseInt(attributes.getValue("", "maxage"), -1); + repoVersion = Utils.parseInt(attributes.getValue("", "version"), -1); + repoName = cleanWhiteSpace(attributes.getValue("", "name")); + repoDescription = cleanWhiteSpace(attributes.getValue("", "description")); } else if ("application".equals(localName) && curapp == null) { curapp = new App(); curapp.id = attributes.getValue("", "id"); @@ -267,15 +264,15 @@ public class RepoXMLHandler extends DefaultHandler { curapk = new Apk(); curapk.id = curapp.id; curapk.repo = repo.getId(); - hashType = null; + currentApkHashType = null; } else if ("hash".equals(localName) && curapk != null) { - hashType = attributes.getValue("", "type"); + currentApkHashType = attributes.getValue("", "type"); } curchars.setLength(0); } - private String cleanWhiteSpace(String str) { - return str.replaceAll("\n", " ").replaceAll(" ", " "); + private String cleanWhiteSpace(@Nullable String str) { + return str == null ? null : str.replaceAll("\\s", " "); } } diff --git a/F-Droid/src/org/fdroid/fdroid/UpdateService.java b/F-Droid/src/org/fdroid/fdroid/UpdateService.java index 0e19d7e7d..3105b5851 100644 --- a/F-Droid/src/org/fdroid/fdroid/UpdateService.java +++ b/F-Droid/src/org/fdroid/fdroid/UpdateService.java @@ -139,11 +139,16 @@ public class UpdateService extends IntentService implements ProgressListener { .setOngoing(true) .setCategory(NotificationCompat.CATEGORY_SERVICE) .setContentTitle(getString(R.string.update_notification_title)); - - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH) { - Intent intent = new Intent(this, FDroid.class); - // TODO: Is this the correct FLAG? - notificationBuilder.setContentIntent(PendingIntent.getActivity(this, 0, intent, PendingIntent.FLAG_UPDATE_CURRENT)); + + // Android docs are a little sketchy, however it seems that Gingerbread is the last + // sdk that made a content intent mandatory: + // + // http://stackoverflow.com/a/20032920 + // + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.GINGERBREAD_MR1) { + Intent pendingIntent = new Intent(this, FDroid.class); + pendingIntent.addFlags (Intent.FLAG_ACTIVITY_NEW_TASK); + notificationBuilder.setContentIntent(PendingIntent.getActivity(this, 0, pendingIntent, PendingIntent.FLAG_UPDATE_CURRENT)); } notificationManager.notify(NOTIFY_ID_UPDATING, notificationBuilder.build()); @@ -324,6 +329,7 @@ public class UpdateService extends IntentService implements ProgressListener { @Override protected void onHandleIntent(Intent intent) { + final long startTime = System.currentTimeMillis(); String address = intent.getStringExtra(EXTRA_ADDRESS); boolean manualUpdate = intent.getBooleanExtra(EXTRA_MANUAL_UPDATE, false); @@ -341,16 +347,12 @@ public class UpdateService extends IntentService implements ProgressListener { // database while we do all the downloading, etc... List repos = RepoProvider.Helper.all(this); - // Process each repo... - RepoPersister appSaver = new RepoPersister(this); - //List swapRepos = new ArrayList<>(); List unchangedRepos = new ArrayList<>(); List updatedRepos = new ArrayList<>(); List disabledRepos = new ArrayList<>(); List errorRepos = new ArrayList<>(); ArrayList repoErrors = new ArrayList<>(); - List repoUpdateRememberers = new ArrayList<>(); boolean changes = false; boolean singleRepoUpdate = !TextUtils.isEmpty(address); for (final Repo repo : repos) { @@ -374,10 +376,8 @@ public class UpdateService extends IntentService implements ProgressListener { try { updater.update(); if (updater.hasChanged()) { - appSaver.queueUpdater(updater); updatedRepos.add(repo); changes = true; - repoUpdateRememberers.add(updater.getRememberer()); } else { unchangedRepos.add(repo); } @@ -392,16 +392,8 @@ public class UpdateService extends IntentService implements ProgressListener { Utils.debugLog(TAG, "Not checking app details or compatibility, because all repos were up to date."); } else { sendStatus(this, STATUS_INFO, getString(R.string.status_checking_compatibility)); - - appSaver.save(disabledRepos); - notifyContentProviders(); - //we only remember the update if everything has gone well - for (RepoUpdater.RepoUpdateRememberer rememberer : repoUpdateRememberers) { - rememberer.rememberUpdate(); - } - if (prefs.getBoolean(Preferences.PREF_UPD_NOTIFY, true)) { performUpdateNotification(); } @@ -428,6 +420,9 @@ public class UpdateService extends IntentService implements ProgressListener { Log.e(TAG, "Exception during update processing", e); sendStatus(this, STATUS_ERROR_GLOBAL, e.getMessage()); } + + long time = System.currentTimeMillis() - startTime; + Log.i(TAG, "Updating repo(s) complete, took " + time / 1000 + " seconds to complete."); } private void notifyContentProviders() { diff --git a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java index 26d5bb83d..495689c17 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/ApkProvider.java @@ -88,6 +88,24 @@ public class ApkProvider extends FDroidProvider { return find(context, id, versionCode, DataColumns.ALL); } + /** + * Find all apks for a particular app, but limit it to those originating from the + * specified repo. + */ + public static List find(Context context, Repo repo, List apps, String[] projection) { + ContentResolver resolver = context.getContentResolver(); + final Uri uri = getContentUriForApps(repo, apps); + Cursor cursor = resolver.query(uri, projection, null, null, null); + return cursorToList(cursor); + } + + /** + * @see org.fdroid.fdroid.data.ApkProvider.Helper#find(Context, Repo, List, String[]) + */ + public static List find(Context context, Repo repo, List apps) { + return find(context, repo, apps, DataColumns.ALL); + } + public static Apk find(Context context, String id, int versionCode, String[] projection) { ContentResolver resolver = context.getContentResolver(); final Uri uri = getContentUri(id, versionCode); @@ -208,12 +226,16 @@ public class ApkProvider extends FDroidProvider { private static final int CODE_APP = CODE_SINGLE + 1; private static final int CODE_REPO = CODE_APP + 1; private static final int CODE_APKS = CODE_REPO + 1; + private static final int CODE_REPO_APPS = CODE_APKS + 1; + private static final int CODE_REPO_APK = CODE_REPO_APPS + 1; - private static final String PROVIDER_NAME = "ApkProvider"; - private static final String PATH_APK = "apk"; - private static final String PATH_APKS = "apks"; - private static final String PATH_APP = "app"; - private static final String PATH_REPO = "repo"; + private static final String PROVIDER_NAME = "ApkProvider"; + private static final String PATH_APK = "apk"; + private static final String PATH_APKS = "apks"; + private static final String PATH_APP = "app"; + private static final String PATH_REPO = "repo"; + private static final String PATH_REPO_APPS = "repo-apps"; + private static final String PATH_REPO_APK = "repo-apk"; private static final UriMatcher matcher = new UriMatcher(-1); @@ -227,6 +249,8 @@ public class ApkProvider extends FDroidProvider { matcher.addURI(getAuthority(), PATH_APK + "/#/*", CODE_SINGLE); matcher.addURI(getAuthority(), PATH_APKS + "/*", CODE_APKS); matcher.addURI(getAuthority(), PATH_APP + "/*", CODE_APP); + matcher.addURI(getAuthority(), PATH_REPO_APPS + "/#/*", CODE_REPO_APPS); + matcher.addURI(getAuthority(), PATH_REPO_APK + "/#/*", CODE_REPO_APK); matcher.addURI(getAuthority(), null, CODE_LIST); } @@ -267,6 +291,24 @@ public class ApkProvider extends FDroidProvider { .build(); } + public static Uri getContentUriForApps(Repo repo, List apps) { + return getContentUri() + .buildUpon() + .appendPath(PATH_REPO_APPS) + .appendPath(Long.toString(repo.id)) + .appendPath(buildAppString(apps)) + .build(); + } + + public static Uri getContentUriForApks(Repo repo, List apks) { + return getContentUri() + .buildUpon() + .appendPath(PATH_REPO_APK) + .appendPath(Long.toString(repo.id)) + .appendPath(buildApkString(apks)) + .build(); + } + /** * Intentionally left protected because it will break if apks is larger than * {@link org.fdroid.fdroid.data.ApkProvider#MAX_APKS_TO_QUERY}. Instead of using @@ -274,6 +316,13 @@ public class ApkProvider extends FDroidProvider { * {@link org.fdroid.fdroid.data.ApkProvider.Helper#knownApks(android.content.Context, java.util.List, String[])} */ protected static Uri getContentUri(List apks) { + return getContentUri().buildUpon() + .appendPath(PATH_APKS) + .appendPath(buildApkString(apks)) + .build(); + } + + private static String buildApkString(List apks) { StringBuilder builder = new StringBuilder(); for (int i = 0; i < apks.size(); i++) { if (i != 0) { @@ -282,10 +331,18 @@ public class ApkProvider extends FDroidProvider { final Apk a = apks.get(i); builder.append(a.id).append(':').append(a.vercode); } - return getContentUri().buildUpon() - .appendPath(PATH_APKS) - .appendPath(builder.toString()) - .build(); + return builder.toString(); + } + + private static String buildAppString(List apks) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < apks.size(); i++) { + if (i != 0) { + builder.append(','); + } + builder.append(apks.get(0).id); + } + return builder.toString(); } @Override @@ -360,6 +417,10 @@ public class ApkProvider extends FDroidProvider { return new QuerySelection(selection, args); } + private QuerySelection queryRepoApps(long repoId, String appIds) { + return queryRepo(repoId).add(AppProvider.queryApps(appIds, DataColumns.APK_ID)); + } + private QuerySelection queryApks(String apkKeys) { final String[] apkDetails = apkKeys.split(","); if (apkDetails.length > MAX_APKS_TO_QUERY) { @@ -408,6 +469,11 @@ public class ApkProvider extends FDroidProvider { query = query.add(queryRepo(Long.parseLong(uri.getLastPathSegment()))); break; + case CODE_REPO_APPS: + List pathSegments = uri.getPathSegments(); + query = query.add(queryRepoApps(Long.parseLong(pathSegments.get(1)), pathSegments.get(2))); + break; + default: Log.e(TAG, "Invalid URI for apk content provider: " + uri); throw new UnsupportedOperationException("Invalid URI for apk content provider: " + uri); @@ -469,6 +535,12 @@ public class ApkProvider extends FDroidProvider { case CODE_APKS: query = query.add(queryApks(uri.getLastPathSegment())); break; + + // TODO: Add tests for this. + case CODE_REPO_APK: + List pathSegments = uri.getPathSegments(); + query = query.add(queryRepo(Long.parseLong(pathSegments.get(1)))).add(queryApks(pathSegments.get(2))); + break; case CODE_LIST: throw new UnsupportedOperationException("Can't delete all apks."); diff --git a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java index 1033e799f..8ddde3c53 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/AppProvider.java @@ -686,12 +686,16 @@ public class AppProvider extends FDroidProvider { return new AppQuerySelection(selection); } - private AppQuerySelection queryApps(String appIds) { + static AppQuerySelection queryApps(String appIds, String idField) { String[] args = appIds.split(","); - String selection = "fdroid_app.id IN (" + generateQuestionMarksForInClause(args.length) + ")"; + String selection = idField + " IN (" + generateQuestionMarksForInClause(args.length) + ")"; return new AppQuerySelection(selection, args); } + private static AppQuerySelection queryApps(String appIds) { + return queryApps(appIds, "fdroid_app.id"); + } + @Override public Cursor query(Uri uri, String[] projection, String customSelection, String[] selectionArgs, String sortOrder) { AppQuerySelection selection = new AppQuerySelection(customSelection, selectionArgs); diff --git a/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java b/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java index ab5210adc..61267cde3 100644 --- a/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java +++ b/F-Droid/src/org/fdroid/fdroid/data/FDroidProvider.java @@ -99,7 +99,7 @@ public abstract class FDroidProvider extends ContentProvider { protected abstract UriMatcher getMatcher(); - protected String generateQuestionMarksForInClause(int num) { + protected static String generateQuestionMarksForInClause(int num) { StringBuilder sb = new StringBuilder(num * 2); for (int i = 0; i < num; i++) { if (i != 0) { diff --git a/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java b/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java index 71c6e43ff..b5471f92c 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/MultiRepoUpdaterTest.java @@ -22,7 +22,6 @@ import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; import java.io.File; -import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -38,7 +37,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { private RepoUpdater mainRepoUpdater; private RepoUpdater archiveRepoUpdater; private File testFilesDir; - private RepoPersister persister; private static final String PUB_KEY = "3082050b308202f3a003020102020420d8f212300d06092a864886f70d01010b050030363110300e0603" + @@ -137,8 +135,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { RepoProvider.Helper.remove(context, 3); RepoProvider.Helper.remove(context, 4); - persister = new RepoPersister(context); - conflictingRepoUpdater = createUpdater(REPO_CONFLICTING, context); mainRepoUpdater = createUpdater(REPO_MAIN, context); archiveRepoUpdater = createUpdater(REPO_ARCHIVE, context); @@ -285,7 +281,7 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { } } - assertTrue("Found app " + appId + ", v" + versionCode, found); + assertTrue("Couldn't find app " + appId + ", v" + versionCode, found); } } @@ -303,10 +299,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { } } - private void persistData() { - persister.save(new ArrayList(0)); - } - /* At time fo writing, the following tests did not pass. This is because the multi-repo support in F-Droid was not sufficient. When working on proper multi repo support than this should be ucommented and all these tests should pass: @@ -314,7 +306,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectConflictingThenMainThenArchive() throws UpdateException { assertEmpty(); if (updateConflicting() && updateMain() && updateArchive()) { - persistData(); assertExpected(); } } @@ -322,7 +313,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectConflictingThenArchiveThenMain() throws UpdateException { assertEmpty(); if (updateConflicting() && updateArchive() && updateMain()) { - persistData(); assertExpected(); } } @@ -330,7 +320,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectArchiveThenMainThenConflicting() throws UpdateException { assertEmpty(); if (updateArchive() && updateMain() && updateConflicting()) { - persistData(); assertExpected(); } } @@ -338,7 +327,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectArchiveThenConflictingThenMain() throws UpdateException { assertEmpty(); if (updateArchive() && updateConflicting() && updateMain()) { - persistData(); assertExpected(); } } @@ -346,7 +334,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectMainThenArchiveThenConflicting() throws UpdateException { assertEmpty(); if (updateMain() && updateArchive() && updateConflicting()) { - persistData(); assertExpected(); } } @@ -354,7 +341,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testCorrectMainThenConflictingThenArchive() throws UpdateException { assertEmpty(); if (updateMain() && updateConflicting() && updateArchive()) { - persistData(); assertExpected(); } } @@ -364,7 +350,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableConflictingThenMainThenArchive() throws UpdateException { assertEmpty(); if (updateConflicting() && updateMain() && updateArchive()) { - persistData(); assertSomewhatAcceptable(); } } @@ -372,7 +357,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableConflictingThenArchiveThenMain() throws UpdateException { assertEmpty(); if (updateConflicting() && updateArchive() && updateMain()) { - persistData(); assertSomewhatAcceptable(); } } @@ -380,7 +364,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableArchiveThenMainThenConflicting() throws UpdateException { assertEmpty(); if (updateArchive() && updateMain() && updateConflicting()) { - persistData(); assertSomewhatAcceptable(); } } @@ -388,7 +371,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableArchiveThenConflictingThenMain() throws UpdateException { assertEmpty(); if (updateArchive() && updateConflicting() && updateMain()) { - persistData(); assertSomewhatAcceptable(); } } @@ -396,7 +378,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableMainThenArchiveThenConflicting() throws UpdateException { assertEmpty(); if (updateMain() && updateArchive() && updateConflicting()) { - persistData(); assertSomewhatAcceptable(); } } @@ -404,7 +385,6 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { public void testAcceptableMainThenConflictingThenArchive() throws UpdateException { assertEmpty(); if (updateMain() && updateConflicting() && updateArchive()) { - persistData(); assertSomewhatAcceptable(); } } @@ -444,8 +424,7 @@ public class MultiRepoUpdaterTest extends InstrumentationTestCase { return false; File indexJar = TestUtils.copyAssetToDir(context, indexJarPath, testFilesDir); - updater.processDownloadedFile(indexJar, UUID.randomUUID().toString()); - persister.queueUpdater(updater); + updater.processDownloadedFile(indexJar); return true; } diff --git a/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java b/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java index a588df536..c8692105b 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/RepoUpdaterTest.java @@ -35,7 +35,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // these are supposed to succeed try { - repoUpdater.processDownloadedFile(simpleIndexJar, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(simpleIndexJar); } catch (UpdateException e) { e.printStackTrace(); fail(); @@ -48,7 +48,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithoutSignature.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { // success! @@ -61,7 +61,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedManifest.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -77,7 +77,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedSignature.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -93,7 +93,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedCertificate.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -109,7 +109,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedEverything.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException e) { e.printStackTrace(); @@ -125,7 +125,7 @@ public class RepoUpdaterTest extends InstrumentationTestCase { // this is supposed to fail try { File jarFile = TestUtils.copyAssetToDir(context, "masterKeyIndex.jar", testFilesDir); - repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString()); + repoUpdater.processDownloadedFile(jarFile); fail(); } catch (UpdateException | SecurityException e) { // success! diff --git a/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java b/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java index ee37d748a..c9be7f43c 100644 --- a/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java +++ b/F-Droid/test/src/org/fdroid/fdroid/RepoXMLHandlerTest.java @@ -15,6 +15,7 @@ import org.xml.sax.XMLReader; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.List; import javax.xml.parsers.ParserConfigurationException; @@ -24,9 +25,34 @@ import javax.xml.parsers.SAXParserFactory; public class RepoXMLHandlerTest extends AndroidTestCase { private static final String TAG = "RepoXMLHandlerTest"; - private Repo repo; + private final Repo actualRepo = new Repo(); - private String fakePubkey = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345"; + public final List actualApps = new ArrayList<>(); + public final List actualApks = new ArrayList<>(); + + private final static String FAKE_PUBKEY = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345"; + + private RepoXMLHandler.IndexReceiver indexReceiver = new RepoXMLHandler.IndexReceiver() { + + private boolean hasReceivedRepo; + + @Override + public void receiveRepo(String name, String description, String signingCert, int maxage, int version) { + assertFalse("Repo XML contains more than one .", hasReceivedRepo); + actualRepo.name = name; + actualRepo.description = description; + actualRepo.pubkey = signingCert; + actualRepo.maxage = maxage; + actualRepo.version = version; + hasReceivedRepo = true; + } + + @Override + public void receiveApp(App app, List packages) { + actualApps.add(app); + actualApks.addAll(packages); + } + }; public RepoXMLHandlerTest() { } @@ -34,28 +60,29 @@ public class RepoXMLHandlerTest extends AndroidTestCase { @Override protected void setUp() throws Exception { super.setUp(); - repo = new Repo(); } public void testSimpleIndex() { - repo.name = "F-Droid"; - repo.pubkey = "308201ee30820157a0030201020204300d845b300d06092a864886f70d01010b0500302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e301e170d3134303432373030303633315a170d3431303931323030303633315a302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e30819f300d06092a864886f70d010101050003818d0030818902818100a439472e4b6d01141bfc94ecfe131c7c728fdda670bb14c57ca60bd1c38a8b8bc0879d22a0a2d0bc0d6fdd4cb98d1d607c2caefbe250a0bd0322aedeb365caf9b236992fac13e6675d3184a6c7c6f07f73410209e399a9da8d5d7512bbd870508eebacff8b57c3852457419434d34701ccbf692267cbc3f42f1c5d1e23762d790203010001a321301f301d0603551d0e041604140b1840691dab909746fde4bfe28207d1cae15786300d06092a864886f70d01010b05000381810062424c928ffd1b6fd419b44daafef01ca982e09341f7077fb865905087aeac882534b3bd679b51fdfb98892cef38b63131c567ed26c9d5d9163afc775ac98ad88c405d211d6187bde0b0d236381cc574ba06ef9080721a92ae5a103a7301b2c397eecc141cc850dd3e123813ebc41c59d31ddbcb6e984168280c53272f6a442b"; - repo.description = "The official repository of the F-Droid client. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitorious.org/f-droid."; - RepoXMLHandler handler = getFromFile(repo, "simpleIndex.xml"); - handlerTestSuite(repo, handler, 0, 0); - assertEquals(handler.getMaxAge(), -1); - assertEquals(handler.getVersion(), 12); + Repo expectedRepo = new Repo(); + expectedRepo.name = "F-Droid"; + expectedRepo.pubkey = "308201ee30820157a0030201020204300d845b300d06092a864886f70d01010b0500302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e301e170d3134303432373030303633315a170d3431303931323030303633315a302a3110300e060355040b1307462d44726f6964311630140603550403130d70616c6174736368696e6b656e30819f300d06092a864886f70d010101050003818d0030818902818100a439472e4b6d01141bfc94ecfe131c7c728fdda670bb14c57ca60bd1c38a8b8bc0879d22a0a2d0bc0d6fdd4cb98d1d607c2caefbe250a0bd0322aedeb365caf9b236992fac13e6675d3184a6c7c6f07f73410209e399a9da8d5d7512bbd870508eebacff8b57c3852457419434d34701ccbf692267cbc3f42f1c5d1e23762d790203010001a321301f301d0603551d0e041604140b1840691dab909746fde4bfe28207d1cae15786300d06092a864886f70d01010b05000381810062424c928ffd1b6fd419b44daafef01ca982e09341f7077fb865905087aeac882534b3bd679b51fdfb98892cef38b63131c567ed26c9d5d9163afc775ac98ad88c405d211d6187bde0b0d236381cc574ba06ef9080721a92ae5a103a7301b2c397eecc141cc850dd3e123813ebc41c59d31ddbcb6e984168280c53272f6a442b"; + expectedRepo.description = "The official repository of the F-Droid client. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitorious.org/f-droid."; + processFile("simpleIndex.xml"); + handlerTestSuite(expectedRepo, 0, 0); + assertEquals(actualRepo.maxage, -1); + assertEquals(actualRepo.version, 12); } public void testSmallRepo() { - repo.name = "Android-Nexus-7-20139453 on UNSET"; - repo.pubkey = "308202da308201c2a00302010202080eb08c796fec91aa300d06092a864886f70d0101050500302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a656374301e170d3134313030333135303631325a170d3135313030333135303631325a302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a65637430820122300d06092a864886f70d01010105000382010f003082010a0282010100c7ab44b130be5c00eedcc3625462f6f6ac26e502641cd641f3e30cbb0ff1ba325158611e7fc2448a35b6a6df30dc6e23602cf6909448befcf11e2fe486b580f1e76fe5887d159050d00afd2c4079f6538896bb200627f4b3e874f011ce5df0fef5d150fcb0b377b531254e436eaf4083ea72fe3b8c3ef450789fa858f2be8f6c5335bb326aff3dda689fbc7b5ba98dea53651dbea7452c38d294985ac5dd8a9e491a695de92c706d682d6911411fcaef3b0a08a030fe8a84e47acaab0b7edcda9d190ce39e810b79b1d8732eca22b15f0d048c8d6f00503a7ee81ab6e08919ff465883432304d95238b95e95c5f74e0a421809e2a6a85825aed680e0d6939e8f0203010001300d06092a864886f70d010105050003820101006d17aad3271b8b2c299dbdb7b1182849b0d5ddb9f1016dcb3487ae0db02b6be503344c7d066e2050bcd01d411b5ee78c7ed450f0ff9da5ce228f774cbf41240361df53d9c6078159d16f4d34379ab7dedf6186489397c83b44b964251a2ebb42b7c4689a521271b1056d3b5a5fa8f28ba64fb8ce5e2226c33c45d27ba3f632dc266c12abf582b8438c2abcf3eae9de9f31152b4158ace0ef33435c20eb809f1b3988131db6e5a1442f2617c3491d9565fedb3e320e8df4236200d3bd265e47934aa578f84d0d1a5efeb49b39907e876452c46996d0feff9404b41aa5631b4482175d843d5512ded45e12a514690646492191e7add434afce63dbff8f0b03ec0c"; - repo.description = "A local FDroid repo generated from apps installed on Android-Nexus-7-20139453"; - RepoXMLHandler handler = getFromFile(repo, "smallRepo.xml"); - handlerTestSuite(repo, handler, 12, 12); - assertEquals(handler.getMaxAge(), 14); - assertEquals(handler.getVersion(), -1); - checkIncludedApps(handler.getApps(), new String[] { + Repo expectedRepo = new Repo(); + expectedRepo.name = "Android-Nexus-7-20139453 on UNSET"; + expectedRepo.pubkey = "308202da308201c2a00302010202080eb08c796fec91aa300d06092a864886f70d0101050500302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a656374301e170d3134313030333135303631325a170d3135313030333135303631325a302d3111300f060355040a0c084b6572706c61707031183016060355040b0c0f477561726469616e50726f6a65637430820122300d06092a864886f70d01010105000382010f003082010a0282010100c7ab44b130be5c00eedcc3625462f6f6ac26e502641cd641f3e30cbb0ff1ba325158611e7fc2448a35b6a6df30dc6e23602cf6909448befcf11e2fe486b580f1e76fe5887d159050d00afd2c4079f6538896bb200627f4b3e874f011ce5df0fef5d150fcb0b377b531254e436eaf4083ea72fe3b8c3ef450789fa858f2be8f6c5335bb326aff3dda689fbc7b5ba98dea53651dbea7452c38d294985ac5dd8a9e491a695de92c706d682d6911411fcaef3b0a08a030fe8a84e47acaab0b7edcda9d190ce39e810b79b1d8732eca22b15f0d048c8d6f00503a7ee81ab6e08919ff465883432304d95238b95e95c5f74e0a421809e2a6a85825aed680e0d6939e8f0203010001300d06092a864886f70d010105050003820101006d17aad3271b8b2c299dbdb7b1182849b0d5ddb9f1016dcb3487ae0db02b6be503344c7d066e2050bcd01d411b5ee78c7ed450f0ff9da5ce228f774cbf41240361df53d9c6078159d16f4d34379ab7dedf6186489397c83b44b964251a2ebb42b7c4689a521271b1056d3b5a5fa8f28ba64fb8ce5e2226c33c45d27ba3f632dc266c12abf582b8438c2abcf3eae9de9f31152b4158ace0ef33435c20eb809f1b3988131db6e5a1442f2617c3491d9565fedb3e320e8df4236200d3bd265e47934aa578f84d0d1a5efeb49b39907e876452c46996d0feff9404b41aa5631b4482175d843d5512ded45e12a514690646492191e7add434afce63dbff8f0b03ec0c"; + expectedRepo.description = "A local FDroid repo generated from apps installed on Android-Nexus-7-20139453"; + processFile("smallRepo.xml"); + handlerTestSuite(expectedRepo, 12, 12); + assertEquals(actualRepo.maxage, 14); + assertEquals(actualRepo.version, -1); + checkIncludedApps(new String[] { "org.mozilla.firefox", "com.koushikdutta.superuser", "info.guardianproject.courier", @@ -72,14 +99,15 @@ public class RepoXMLHandlerTest extends AndroidTestCase { } public void testMediumRepo() { - repo.name = "Guardian Project Official Releases"; - repo.pubkey = "308205d8308203c0020900a397b4da7ecda034300d06092a864886f70d01010505003081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f301e170d3134303632363139333931385a170d3431313131303139333931385a3081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f30820222300d06092a864886f70d01010105000382020f003082020a0282020100b3cd79121b9b883843be3c4482e320809106b0a23755f1dd3c7f46f7d315d7bb2e943486d61fc7c811b9294dcc6b5baac4340f8db2b0d5e14749e7f35e1fc211fdbc1071b38b4753db201c314811bef885bd8921ad86facd6cc3b8f74d30a0b6e2e6e576f906e9581ef23d9c03e926e06d1f033f28bd1e21cfa6a0e3ff5c9d8246cf108d82b488b9fdd55d7de7ebb6a7f64b19e0d6b2ab1380a6f9d42361770d1956701a7f80e2de568acd0bb4527324b1e0973e89595d91c8cc102d9248525ae092e2c9b69f7414f724195b81427f28b1d3d09a51acfe354387915fd9521e8c890c125fc41a12bf34d2a1b304067ab7251e0e9ef41833ce109e76963b0b256395b16b886bca21b831f1408f836146019e7908829e716e72b81006610a2af08301de5d067c9e114a1e5759db8a6be6a3cc2806bcfe6fafd41b5bc9ddddb3dc33d6f605b1ca7d8a9e0ecdd6390d38906649e68a90a717bea80fa220170eea0c86fc78a7e10dac7b74b8e62045a3ecca54e035281fdc9fe5920a855fde3c0be522e3aef0c087524f13d973dff3768158b01a5800a060c06b451ec98d627dd052eda804d0556f60dbc490d94e6e9dea62ffcafb5beffbd9fc38fb2f0d7050004fe56b4dda0a27bc47554e1e0a7d764e17622e71f83a475db286bc7862deee1327e2028955d978272ea76bf0b88e70a18621aba59ff0c5993ef5f0e5d6b6b98e68b70203010001300d06092a864886f70d0101050500038202010079c79c8ef408a20d243d8bd8249fb9a48350dc19663b5e0fce67a8dbcb7de296c5ae7bbf72e98a2020fb78f2db29b54b0e24b181aa1c1d333cc0303685d6120b03216a913f96b96eb838f9bff125306ae3120af838c9fc07ebb5100125436bd24ec6d994d0bff5d065221871f8410daf536766757239bf594e61c5432c9817281b985263bada8381292e543a49814061ae11c92a316e7dc100327b59e3da90302c5ada68c6a50201bda1fcce800b53f381059665dbabeeb0b50eb22b2d7d2d9b0aa7488ca70e67ac6c518adb8e78454a466501e89d81a45bf1ebc350896f2c3ae4b6679ecfbf9d32960d4f5b493125c7876ef36158562371193f600bc511000a67bdb7c664d018f99d9e589868d103d7e0994f166b2ba18ff7e67d8c4da749e44dfae1d930ae5397083a51675c409049dfb626a96246c0015ca696e94ebb767a20147834bf78b07fece3f0872b057c1c519ff882501995237d8206b0b3832f78753ebd8dcbd1d3d9f5ba733538113af6b407d960ec4353c50eb38ab29888238da843cd404ed8f4952f59e4bbc0035fc77a54846a9d419179c46af1b4a3b7fc98e4d312aaa29b9b7d79e739703dc0fa41c7280d5587709277ffa11c3620f5fba985b82c238ba19b17ebd027af9424be0941719919f620dd3bb3c3f11638363708aa11f858e153cf3a69bce69978b90e4a273836100aa1e617ba455cd00426847f"; - repo.description = "The official app repository of The Guardian Project. Applications in this repository are official binaries build by the original application developers and signed by the same key as the APKs that are released in the Google Play store."; - RepoXMLHandler handler = getFromFile(repo, "mediumRepo.xml"); - handlerTestSuite(repo, handler, 15, 36); - assertEquals(handler.getMaxAge(), 60); - assertEquals(handler.getVersion(), 12); - checkIncludedApps(handler.getApps(), new String[] { + Repo expectedRepo = new Repo(); + expectedRepo.name = "Guardian Project Official Releases"; + expectedRepo.pubkey = "308205d8308203c0020900a397b4da7ecda034300d06092a864886f70d01010505003081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f301e170d3134303632363139333931385a170d3431313131303139333931385a3081ad310b30090603550406130255533111300f06035504080c084e657720596f726b3111300f06035504070c084e657720596f726b31143012060355040b0c0b4644726f6964205265706f31193017060355040a0c10477561726469616e2050726f6a656374311d301b06035504030c14677561726469616e70726f6a6563742e696e666f3128302606092a864886f70d0109011619726f6f7440677561726469616e70726f6a6563742e696e666f30820222300d06092a864886f70d01010105000382020f003082020a0282020100b3cd79121b9b883843be3c4482e320809106b0a23755f1dd3c7f46f7d315d7bb2e943486d61fc7c811b9294dcc6b5baac4340f8db2b0d5e14749e7f35e1fc211fdbc1071b38b4753db201c314811bef885bd8921ad86facd6cc3b8f74d30a0b6e2e6e576f906e9581ef23d9c03e926e06d1f033f28bd1e21cfa6a0e3ff5c9d8246cf108d82b488b9fdd55d7de7ebb6a7f64b19e0d6b2ab1380a6f9d42361770d1956701a7f80e2de568acd0bb4527324b1e0973e89595d91c8cc102d9248525ae092e2c9b69f7414f724195b81427f28b1d3d09a51acfe354387915fd9521e8c890c125fc41a12bf34d2a1b304067ab7251e0e9ef41833ce109e76963b0b256395b16b886bca21b831f1408f836146019e7908829e716e72b81006610a2af08301de5d067c9e114a1e5759db8a6be6a3cc2806bcfe6fafd41b5bc9ddddb3dc33d6f605b1ca7d8a9e0ecdd6390d38906649e68a90a717bea80fa220170eea0c86fc78a7e10dac7b74b8e62045a3ecca54e035281fdc9fe5920a855fde3c0be522e3aef0c087524f13d973dff3768158b01a5800a060c06b451ec98d627dd052eda804d0556f60dbc490d94e6e9dea62ffcafb5beffbd9fc38fb2f0d7050004fe56b4dda0a27bc47554e1e0a7d764e17622e71f83a475db286bc7862deee1327e2028955d978272ea76bf0b88e70a18621aba59ff0c5993ef5f0e5d6b6b98e68b70203010001300d06092a864886f70d0101050500038202010079c79c8ef408a20d243d8bd8249fb9a48350dc19663b5e0fce67a8dbcb7de296c5ae7bbf72e98a2020fb78f2db29b54b0e24b181aa1c1d333cc0303685d6120b03216a913f96b96eb838f9bff125306ae3120af838c9fc07ebb5100125436bd24ec6d994d0bff5d065221871f8410daf536766757239bf594e61c5432c9817281b985263bada8381292e543a49814061ae11c92a316e7dc100327b59e3da90302c5ada68c6a50201bda1fcce800b53f381059665dbabeeb0b50eb22b2d7d2d9b0aa7488ca70e67ac6c518adb8e78454a466501e89d81a45bf1ebc350896f2c3ae4b6679ecfbf9d32960d4f5b493125c7876ef36158562371193f600bc511000a67bdb7c664d018f99d9e589868d103d7e0994f166b2ba18ff7e67d8c4da749e44dfae1d930ae5397083a51675c409049dfb626a96246c0015ca696e94ebb767a20147834bf78b07fece3f0872b057c1c519ff882501995237d8206b0b3832f78753ebd8dcbd1d3d9f5ba733538113af6b407d960ec4353c50eb38ab29888238da843cd404ed8f4952f59e4bbc0035fc77a54846a9d419179c46af1b4a3b7fc98e4d312aaa29b9b7d79e739703dc0fa41c7280d5587709277ffa11c3620f5fba985b82c238ba19b17ebd027af9424be0941719919f620dd3bb3c3f11638363708aa11f858e153cf3a69bce69978b90e4a273836100aa1e617ba455cd00426847f"; + expectedRepo.description = "The official app repository of The Guardian Project. Applications in this repository are official binaries build by the original application developers and signed by the same key as the APKs that are released in the Google Play store."; + processFile("mediumRepo.xml"); + handlerTestSuite(expectedRepo, 15, 36); + assertEquals(expectedRepo.maxage, 60); + assertEquals(expectedRepo.version, 12); + checkIncludedApps(new String[] { "info.guardianproject.cacert", "info.guardianproject.otr.app.im", "info.guardianproject.soundrecorder", @@ -99,18 +127,19 @@ public class RepoXMLHandlerTest extends AndroidTestCase { } public void testLargeRepo() { - repo.name = "F-Droid"; - repo.pubkey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef"; - repo.description = "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time."; - RepoXMLHandler handler = getFromFile(repo, "largeRepo.xml"); - handlerTestSuite(repo, handler, 1211, 2381); - assertEquals(handler.getMaxAge(), 14); - assertEquals(handler.getVersion(), 12); + Repo expectedRepo = new Repo(); + expectedRepo.name = "F-Droid"; + expectedRepo.pubkey = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d01010505003071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b73301e170d3130303732333137313032345a170d3337313230383137313032345a3071310b300906035504061302554b3110300e06035504081307556e6b6e6f776e3111300f0603550407130857657468657262793110300e060355040a1307556e6b6e6f776e3110300e060355040b1307556e6b6e6f776e311930170603550403131043696172616e2047756c746e69656b7330820122300d06092a864886f70d01010105000382010f003082010a028201010096d075e47c014e7822c89fd67f795d23203e2a8843f53ba4e6b1bf5f2fd0e225938267cfcae7fbf4fe596346afbaf4070fdb91f66fbcdf2348a3d92430502824f80517b156fab00809bdc8e631bfa9afd42d9045ab5fd6d28d9e140afc1300917b19b7c6c4df4a494cf1f7cb4a63c80d734265d735af9e4f09455f427aa65a53563f87b336ca2c19d244fcbba617ba0b19e56ed34afe0b253ab91e2fdb1271f1b9e3c3232027ed8862a112f0706e234cf236914b939bcf959821ecb2a6c18057e070de3428046d94b175e1d89bd795e535499a091f5bc65a79d539a8d43891ec504058acb28c08393b5718b57600a211e803f4a634e5c57f25b9b8c4422c6fd90203010001300d06092a864886f70d0101050500038201010008e4ef699e9807677ff56753da73efb2390d5ae2c17e4db691d5df7a7b60fc071ae509c5414be7d5da74df2811e83d3668c4a0b1abc84b9fa7d96b4cdf30bba68517ad2a93e233b042972ac0553a4801c9ebe07bf57ebe9a3b3d6d663965260e50f3b8f46db0531761e60340a2bddc3426098397fda54044a17e5244549f9869b460ca5e6e216b6f6a2db0580b480ca2afe6ec6b46eedacfa4aa45038809ece0c5978653d6c85f678e7f5a2156d1bedd8117751e64a4b0dcd140f3040b021821a8d93aed8d01ba36db6c82372211fed714d9a32607038cdfd565bd529ffc637212aaa2c224ef22b603eccefb5bf1e085c191d4b24fe742b17ab3f55d4e6f05ef"; + expectedRepo.description = "The official FDroid repository. Applications in this repository are mostly built directory from the source code. Some are official binaries built by the original application developers - these will be replaced by source-built versions over time."; + processFile("largeRepo.xml"); + handlerTestSuite(expectedRepo, 1211, 2381); + assertEquals("Repo max age", 14, expectedRepo.maxage); + assertEquals("Repo version", 12, expectedRepo.version); /* * generated using: sed 's, apps, String[] packageNames) { - assertNotNull(apps); + private void checkIncludedApps(String[] packageNames) { + assertNotNull(actualApps); assertNotNull(packageNames); - assertEquals(apps.size(), packageNames.length); + assertEquals(actualApps.size(), packageNames.length); for (String id : packageNames) { boolean thisAppMissing = true; - for (App app : apps) { - if (TextUtils.equals(app.id, id)) + for (App app : actualApps) { + if (TextUtils.equals(app.id, id)) { thisAppMissing = false; + break; + } } assertFalse(thisAppMissing); } } - private void handlerTestSuite(Repo repo, RepoXMLHandler handler, int appCount, int apkCount) { - assertNotNull(handler); - assertFalse(TextUtils.isEmpty(handler.getSigningCertFromIndexXml())); - assertEquals(repo.pubkey.length(), handler.getSigningCertFromIndexXml().length()); - assertEquals(repo.pubkey, handler.getSigningCertFromIndexXml()); - assertFalse(fakePubkey.equals(handler.getSigningCertFromIndexXml())); + private void handlerTestSuite(Repo expectedRepo, int appCount, int apkCount) { + assertFalse(TextUtils.isEmpty(actualRepo.pubkey)); + assertEquals(expectedRepo.pubkey.length(), actualRepo.pubkey.length()); + assertEquals(expectedRepo.pubkey, actualRepo.pubkey); + assertFalse(FAKE_PUBKEY.equals(actualRepo.pubkey)); - assertFalse(TextUtils.isEmpty(handler.getName())); - assertEquals(repo.name.length(), handler.getName().length()); - assertEquals(repo.name, handler.getName()); + assertFalse(TextUtils.isEmpty(actualRepo.name)); + assertEquals(expectedRepo.name.length(), actualRepo.name.length()); + assertEquals(expectedRepo.name, actualRepo.name); - assertFalse(TextUtils.isEmpty(handler.getDescription())); - assertEquals(repo.description.length(), handler.getDescription().length()); - assertEquals(repo.description, handler.getDescription()); + assertFalse(TextUtils.isEmpty(actualRepo.description)); + assertEquals(expectedRepo.description.length(), actualRepo.description.length()); + assertEquals(expectedRepo.description, actualRepo.description); - List apps = handler.getApps(); - assertNotNull(apps); - assertEquals(apps.size(), appCount); + assertNotNull(actualApps); + assertEquals(actualApps.size(), appCount); - List apks = handler.getApks(); + List apks = actualApks; assertNotNull(apks); assertEquals(apks.size(), apkCount); } - private RepoXMLHandler getFromFile(Repo repo, String indexFilename) { + private static class MockRepo extends Repo { + @Override + public long getId() { + return 10000; + } + } + + private RepoXMLHandler processFile(String indexFilename) { SAXParser parser; try { parser = SAXParserFactory.newInstance().newSAXParser(); XMLReader reader = parser.getXMLReader(); - RepoXMLHandler handler = new RepoXMLHandler(repo); + RepoXMLHandler handler = new RepoXMLHandler(new MockRepo(), indexReceiver); reader.setContentHandler(handler); String resName = "assets/" + indexFilename; Log.i(TAG, "test file: " + getClass().getClassLoader().getResource(resName));