Karl/BobStore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'issue-1054--index-v1-progress-via-broadcasts' into 'master'

Browse Source 
Add progress reporting for index-v1 (using `LocalBroadcastManager`)

Closes #1054

See merge request !550
This commit is contained in:
Hans-Christoph Steiner 2017-07-05 15:41:33 +00:00
commit 8ba7882a98
4 changed files with 107 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java
View File

@ -2,10 +2,8 @@ package org.fdroid.fdroid;
import android.content.ContentValues;
import android.content.Context;
import android.content.Intent;
import android.net.Uri;
import android.support.annotation.NonNull;
import android.support.v4.content.LocalBroadcastManager;
import android.text.TextUtils;
import android.util.Log;
@ -61,11 +59,13 @@ public class IndexV1Updater extends RepoUpdater {
private static final String SIGNED_FILE_NAME = "index-v1.jar";
public static final String DATA_FILE_NAME = "index-v1.json";
private final LocalBroadcastManager localBroadcastManager;
public IndexV1Updater(@NonNull Context context, @NonNull Repo repo) {
super(context, repo);
this.localBroadcastManager = LocalBroadcastManager.getInstance(this.context);
}
@Override
protected String getIndexUrl(@NonNull Repo repo) {
return Uri.parse(repo.address).buildUpon().appendPath(SIGNED_FILE_NAME).build().toString();
}
/**
@ -83,19 +83,10 @@ public class IndexV1Updater extends RepoUpdater {
InputStream indexInputStream = null;
try {
// read file name from file
final Uri dataUri = Uri.parse(repo.address).buildUpon().appendPath(SIGNED_FILE_NAME).build();
final Uri dataUri = Uri.parse(indexUrl);
downloader = DownloaderFactory.create(context, dataUri.toString());
downloader.setCacheTag(repo.lastetag);
downloader.setListener(new ProgressListener() {
@Override
public void onProgress(URL sourceUrl, int bytesRead, int totalBytes) {
Intent intent = new Intent(Downloader.ACTION_PROGRESS);
intent.setData(dataUri);
intent.putExtra(Downloader.EXTRA_BYTES_READ, bytesRead);
intent.putExtra(Downloader.EXTRA_TOTAL_BYTES, totalBytes);
localBroadcastManager.sendBroadcast(intent);
}
});
downloader.setListener(downloadListener);
downloader.download();
if (downloader.isNotFound()) {
return false;
@ -114,7 +105,7 @@ public class IndexV1Updater extends RepoUpdater {
JarFile jarFile = new JarFile(downloader.outputFile, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry(DATA_FILE_NAME);
indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry),
processXmlProgressListener, new URL(repo.address), (int) indexEntry.getSize());
processIndexListener, new URL(repo.address), (int) indexEntry.getSize());
processIndexV1(indexInputStream, indexEntry, downloader.getCacheTag());
} catch (IOException e) {
@ -156,6 +147,8 @@ public class IndexV1Updater extends RepoUpdater {
*/
public void processIndexV1(InputStream indexInputStream, JarEntry indexEntry, String cacheTag)
throws IOException, UpdateException {
Utils.Profiler profiler = new Utils.Profiler(TAG);
profiler.log("Starting to process index-v1.json");
ObjectMapper mapper = getObjectMapperInstance(repo.getId());
JsonFactory f = mapper.getFactory();
JsonParser parser = f.createParser(indexInputStream);
@ -186,6 +179,7 @@ public class IndexV1Updater extends RepoUpdater {
}
}
parser.close(); // ensure resources get cleaned up timely and properly
profiler.log("Finished processing index-v1.json. Now verifying certificate...");
if (repoMap == null) {
return;
@ -200,7 +194,8 @@ public class IndexV1Updater extends RepoUpdater {
X509Certificate certificate = getSigningCertFromJar(indexEntry);
verifySigningCertificate(certificate);
Utils.debugLog(TAG, "Repo signature verified, saving app metadata to database.");
profiler.log("Certificate verified. Now saving to database...");
// timestamp is absolutely required
repo.timestamp = timestamp;
@ -215,7 +210,9 @@ public class IndexV1Updater extends RepoUpdater {
RepoPersister repoPersister = new RepoPersister(context, repo);
if (apps != null && apps.length > 0) {
int appCount = 0;
for (App app : apps) {
appCount++;
List<Apk> apks = null;
if (packages != null) {
apks = packages.get(app.packageName);
@ -224,16 +221,25 @@ public class IndexV1Updater extends RepoUpdater {
Log.i(TAG, "processIndexV1 empty packages");
apks = new ArrayList<Apk>(0);
}
if (appCount % 50 == 0) {
notifyProcessingApps(appCount, apps.length);
}
repoPersister.saveToDb(app, apks);
}
}
// TODO send event saying moving on to committing to db
profiler.log("Saved to database, but only a temporary table. Now persisting to database...");
notifyCommittingToDb();
ContentValues values = prepareRepoDetailsForSaving(repo.name,
repo.description, repo.maxage, repo.version, repo.timestamp, repo.icon,
repo.mirrors, cacheTag);
repoPersister.commit(values);
profiler.log("Persited to database.");
// TODO RepoUpdater.processRepoPushRequests(context, repoPushRequestList);
Utils.debugLog(TAG, "Repo Push Requests: " + requests);

70
app/src/main/java/org/fdroid/fdroid/RepoUpdater.java
View File

@ -21,8 +21,6 @@
*/
package org.fdroid.fdroid;
// TODO move to org.fdroid.fdroid.updater
// TODO reduce visibility of methods once in .updater package (.e.g tests need it public now)
import android.content.ContentResolver;
import android.content.ContentValues;
@ -30,7 +28,6 @@ import android.content.Context;
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.text.TextUtils;
import android.util.Log;
@ -54,7 +51,6 @@ import org.xml.sax.XMLReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.CodeSigner;
import java.security.cert.Certificate;
@ -70,6 +66,9 @@ import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
// TODO move to org.fdroid.fdroid.updater
// TODO reduce visibility of methods once in .updater package (.e.g tests need it public now)
/**
* Updates the local database with a repository's app/apk metadata and verifying
* the JAR signature on the file received from the repository. As an overview:
@ -85,22 +84,15 @@ import javax.xml.parsers.SAXParserFactory;
* very careful with the changes that you are making!
*/
public class RepoUpdater {
//TODO rename RepoUpdater to IndexV0Updater
private static final String TAG = "RepoUpdater";
private final String indexUrl;
final String indexUrl;
@NonNull
final Context context;
@NonNull
final Repo repo;
boolean hasChanged;
@Nullable
ProgressListener downloadProgressListener;
ProgressListener committingProgressListener;
ProgressListener processXmlProgressListener;
private String cacheTag;
private X509Certificate signingCertFromJar;
@ -118,25 +110,16 @@ public class RepoUpdater {
this.context = context;
this.repo = repo;
this.persister = new RepoPersister(context, repo);
this.indexUrl = getIndexUrl(repo);
}
protected String getIndexUrl(@NonNull Repo repo) {
String url = repo.address + "/index.jar";
String versionName = Utils.getVersionName(context);
if (versionName != null) {
url += "?client_version=" + versionName;
}
this.indexUrl = url;
}
public void setDownloadProgressListener(ProgressListener progressListener) {
this.downloadProgressListener = progressListener;
}
public void setProcessXmlProgressListener(ProgressListener progressListener) {
this.processXmlProgressListener = progressListener;
}
public void setCommittingProgressListener(ProgressListener progressListener) {
this.committingProgressListener = progressListener;
return url;
}
public boolean hasChanged() {
@ -148,7 +131,7 @@ public class RepoUpdater {
try {
downloader = DownloaderFactory.create(context, indexUrl);
downloader.setCacheTag(repo.lastetag);
downloader.setListener(downloadProgressListener);
downloader.setListener(downloadListener);
downloader.download();
if (downloader.isCached()) {
@ -238,7 +221,7 @@ public class RepoUpdater {
JarFile jarFile = new JarFile(downloadedFile, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry("index.xml");
indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry),
processXmlProgressListener, new URL(repo.address), (int) indexEntry.getSize());
processIndexListener, new URL(repo.address), (int) indexEntry.getSize());
// Process the index...
SAXParserFactory factory = SAXParserFactory.newInstance();
@ -274,16 +257,31 @@ public class RepoUpdater {
}
}
protected final ProgressListener downloadListener = new ProgressListener() {
@Override
public void onProgress(URL sourceUrl, int bytesRead, int totalBytes) {
UpdateService.reportDownloadProgress(context, RepoUpdater.this, bytesRead, totalBytes);
}
};
protected final ProgressListener processIndexListener = new ProgressListener() {
@Override
public void onProgress(URL sourceUrl, int bytesRead, int totalBytes) {
UpdateService.reportProcessIndexProgress(context, RepoUpdater.this, bytesRead, totalBytes);
}
};
protected void notifyProcessingApps(int appsSaved, int totalApps) {
UpdateService.reportProcessingAppsProgress(context, this, appsSaved, totalApps);
}
protected void notifyCommittingToDb() {
notifyProcessingApps(0, -1);
}
private void commitToDb() throws UpdateException {
Log.i(TAG, "Repo signature verified, saving app metadata to database.");
if (committingProgressListener != null) {
try {
//TODO this should be an event, not a progress listener
committingProgressListener.onProgress(new URL(indexUrl), 0, -1);
} catch (MalformedURLException e) {
e.printStackTrace();
}
}
notifyCommittingToDb();
persister.commit(repoDetailsToSave);
}
@ -479,7 +477,7 @@ public class RepoUpdater {
// TODO: In the future, this needs to be able to specify which repository to get
// the package from. Better yet, we should be able to specify the hash of a package
// to install (especially when we move to using hashes more as identifiers than we
// do righ tnow).
// do right now).
App app = AppProvider.Helper.findHighestPriorityMetadata(cr, packageName);
if (app == null) {
Utils.debugLog(TAG, packageName + " not in local database, ignoring request to"

99
app/src/main/java/org/fdroid/fdroid/UpdateService.java
View File

@ -51,7 +51,6 @@ import org.fdroid.fdroid.data.Schema;
import org.fdroid.fdroid.installer.InstallManagerService;
import org.fdroid.fdroid.views.main.MainActivity;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -178,15 +177,15 @@ public class UpdateService extends IntentService {
LocalBroadcastManager.getInstance(this).unregisterReceiver(updateStatusReceiver);
}
private static void sendStatus(Context context, int statusCode) {
public static void sendStatus(Context context, int statusCode) {
sendStatus(context, statusCode, null, -1);
}
private static void sendStatus(Context context, int statusCode, String message) {
public static void sendStatus(Context context, int statusCode, String message) {
sendStatus(context, statusCode, message, -1);
}
private static void sendStatus(Context context, int statusCode, String message, int progress) {
public static void sendStatus(Context context, int statusCode, String message, int progress) {
Intent intent = new Intent(LOCAL_ACTION_STATUS);
intent.putExtra(EXTRA_STATUS_CODE, statusCode);
if (!TextUtils.isEmpty(message)) {
@ -408,12 +407,11 @@ public class UpdateService extends IntentService {
try {
RepoUpdater updater = new IndexV1Updater(this, repo);
//TODO setProgressListeners(updater);
if (Preferences.get().isForceOldIndexEnabled() || !updater.update()) {
updater = new RepoUpdater(getBaseContext(), repo);
setProgressListeners(updater);
updater.update();
}
if (updater.hasChanged()) {
updatedRepos++;
changes = true;
@ -501,53 +499,50 @@ public class UpdateService extends IntentService {
}
}
public static void reportDownloadProgress(Context context, RepoUpdater updater, int bytesRead, int totalBytes) {
Utils.debugLog(TAG, "Downloading " + updater.indexUrl + "(" + bytesRead + "/" + totalBytes + ")");
String downloadedSizeFriendly = Utils.getFriendlySize(bytesRead);
int percent = -1;
if (totalBytes > 0) {
percent = (int) ((double) bytesRead / totalBytes * 100);
}
String message;
if (totalBytes == -1) {
message = context.getString(R.string.status_download_unknown_size, updater.indexUrl, downloadedSizeFriendly);
percent = -1;
} else {
String totalSizeFriendly = Utils.getFriendlySize(totalBytes);
message = context.getString(R.string.status_download, updater.indexUrl, downloadedSizeFriendly, totalSizeFriendly, percent);
}
sendStatus(context, STATUS_INFO, message, percent);
}
public static void reportProcessIndexProgress(Context context, RepoUpdater updater, int bytesRead, int totalBytes) {
Utils.debugLog(TAG, "Processing " + updater.indexUrl + "(" + bytesRead + "/" + totalBytes + ")");
String downloadedSize = Utils.getFriendlySize(bytesRead);
String totalSize = Utils.getFriendlySize(totalBytes);
int percent = -1;
if (totalBytes > 0) {
percent = (int) ((double) bytesRead / totalBytes * 100);
}
String message = context.getString(R.string.status_processing_xml_percent, updater.indexUrl, downloadedSize, totalSize, percent);
sendStatus(context, STATUS_INFO, message, percent);
}
/**
* Set up the various {@link ProgressListener}s needed to get feedback to the UI.
* Note: {@code ProgressListener}s do not need to be unregistered, they can just
* be set again for each download.
* If an updater is unable to know how many apps it has to process (i.e. it is streaming apps to the database or
* performing a large database query which touches all apps, but is unable to report progress), then it call this
* listener with `totalBytes = 0`. Doing so will result in a message of "Saving app details" sent to the user. If
* you know how many apps you have processed, then a message of "Saving app details (x/total)" is displayed.
*/
private void setProgressListeners(RepoUpdater updater) {
updater.setDownloadProgressListener(new ProgressListener() {
@Override
public void onProgress(URL sourceUrl, int bytesRead, int totalBytes) {
Log.i(TAG, "downloadProgressReceiver " + sourceUrl);
String downloadedSizeFriendly = Utils.getFriendlySize(bytesRead);
int percent = -1;
if (totalBytes > 0) {
percent = (int) ((double) bytesRead / totalBytes * 100);
}
String message;
if (totalBytes == -1) {
message = getString(R.string.status_download_unknown_size, sourceUrl, downloadedSizeFriendly);
percent = -1;
} else {
String totalSizeFriendly = Utils.getFriendlySize(totalBytes);
message = getString(R.string.status_download, sourceUrl, downloadedSizeFriendly, totalSizeFriendly, percent);
}
sendStatus(getApplicationContext(), STATUS_INFO, message, percent);
}
});
updater.setProcessXmlProgressListener(new ProgressListener() {
@Override
public void onProgress(URL sourceUrl, int bytesRead, int totalBytes) {
String downloadedSize = Utils.getFriendlySize(bytesRead);
String totalSize = Utils.getFriendlySize(totalBytes);
int percent = -1;
if (totalBytes > 0) {
percent = (int) ((double) bytesRead / totalBytes * 100);
}
String message = getString(R.string.status_processing_xml_percent, sourceUrl, downloadedSize, totalSize, percent);
sendStatus(getApplicationContext(), STATUS_INFO, message, percent);
}
});
updater.setCommittingProgressListener(new ProgressListener() {
@Override
public void onProgress(URL sourceUrl, int bytesRead, int totalBytes) {
String message = getString(R.string.status_inserting_apps);
sendStatus(getApplicationContext(), STATUS_INFO, message);
}
});
public static void reportProcessingAppsProgress(Context context, RepoUpdater updater, int appsSaved, int totalApps) {
Utils.debugLog(TAG, "Committing " + updater.indexUrl + "(" + appsSaved + "/" + totalApps + ")");
if (totalApps > 0) {
String message = context.getString(R.string.status_inserting_x_apps, appsSaved, totalApps, updater.indexUrl);
sendStatus(context, STATUS_INFO, message, (int) ((double) appsSaved / totalApps * 100));
} else {
String message = context.getString(R.string.status_inserting_apps);
sendStatus(context, STATUS_INFO, message);
}
}
}

1
app/src/main/res/values/strings.xml
View File

@ -243,6 +243,7 @@
<string name="status_processing_xml_percent">Processing %2$s / %3$s (%4$d%%) from %1$s</string>
<string name="status_connecting_to_repo">Connecting to\n%1$s</string>
<string name="status_inserting_apps">Saving app details</string>
<string name="status_inserting_x_apps">Saving app details (%1$d/%2$d) from %3$s</string>
<string name="repos_unchanged">All repositories are up to date</string>
<string name="all_other_repos_fine">All other repos didn\'t create errors.</string>
<string name="global_error_updating_repos">Error during update: %s</string>