From dd14b9e315cbf7fe763c1aa946c985556c9f9b2e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Feb 2019 14:06:54 +0100 Subject: [PATCH] choose random mirror for each package/APK download This spreads downloads across all available mirrors randomly. This could definitely be improved, like choosing the fastest or nearest mirror, or only .onion addresses on Tor. This will improve the current situation and should reduce the load on f-droid.org a lot. fdroidclient#1696 --- .../java/org/fdroid/fdroid/FDroidApp.java | 47 +++++++++++-------- .../org/fdroid/fdroid/IndexV1Updater.java | 4 +- .../java/org/fdroid/fdroid/data/DBHelper.java | 2 +- .../java/org/fdroid/fdroid/data/Repo.java | 2 +- .../installer/InstallManagerService.java | 32 +++++++++++-- 5 files changed, 58 insertions(+), 29 deletions(-) diff --git a/app/src/main/java/org/fdroid/fdroid/FDroidApp.java b/app/src/main/java/org/fdroid/fdroid/FDroidApp.java index c828876e2..b7558cce6 100644 --- a/app/src/main/java/org/fdroid/fdroid/FDroidApp.java +++ b/app/src/main/java/org/fdroid/fdroid/FDroidApp.java @@ -39,6 +39,7 @@ import android.net.Uri; import android.os.Build; import android.os.Environment; import android.os.StrictMode; +import android.support.annotation.Nullable; import android.support.v4.util.LongSparseArray; import android.text.TextUtils; import android.util.Base64; @@ -66,7 +67,6 @@ import org.fdroid.fdroid.compat.PRNGFixes; import org.fdroid.fdroid.data.AppProvider; import org.fdroid.fdroid.data.InstalledAppProviderService; import org.fdroid.fdroid.data.Repo; -import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.installer.ApkFileProvider; import org.fdroid.fdroid.installer.InstallHistoryService; import org.fdroid.fdroid.localrepo.SDCardScannerService; @@ -245,13 +245,6 @@ public class FDroidApp extends Application { repo = new Repo(); } - /** - * @see #getMirror(String, Repo) - */ - public static String getMirror(String urlString, long repoId) throws IOException { - return getMirror(urlString, RepoProvider.Helper.findById(getInstance(), repoId)); - } - /** * Each time this is called, it will return a mirror from the pool of * mirrors. If it reaches the end of the list of mirrors, it will start @@ -260,17 +253,18 @@ public class FDroidApp extends Application { * again, it will do one last pass through the list with the timeout set to * {@link Downloader#LONGEST_TIMEOUT}. After that, this gives up with a * {@link IOException}. + *

+ * {@link #lastWorkingMirrorArray} is used to track the last mirror URL used, + * so it can be used in the string replacement operating when converting a + * download URL to point to a different mirror. Download URLs can be + * anything from {@code index-v1.jar} to APKs to icons to screenshots. * * @see #resetMirrorVars() * @see #getTimeout() * @see Repo#getRandomMirror(String) */ - public static String getMirror(String urlString, Repo repo2) throws IOException { + public static String getNewMirrorOnError(@Nullable String urlString, Repo repo2) throws IOException { if (repo2.hasMirrors()) { - String lastWorkingMirror = lastWorkingMirrorArray.get(repo2.getId()); - if (lastWorkingMirror == null) { - lastWorkingMirror = repo2.address; - } if (numTries <= 0) { if (timeout == Downloader.DEFAULT_TIMEOUT) { timeout = Downloader.SECOND_TIMEOUT; @@ -286,24 +280,37 @@ public class FDroidApp extends Application { if (numTries == Integer.MAX_VALUE) { numTries = repo2.getMirrorCount(); } - String mirror = repo2.getRandomMirror(lastWorkingMirror); - String newUrl = urlString.replace(lastWorkingMirror, mirror); - Utils.debugLog(TAG, "Trying mirror " + mirror + " after " + lastWorkingMirror + " failed," + - " timeout=" + timeout / 1000 + "s"); - lastWorkingMirrorArray.put(repo2.getId(), mirror); numTries--; - return newUrl; + return switchUrlToNewMirror(urlString, repo2); } else { throw new IOException("No mirrors available"); } } + /** + * Switch the URL in {@code urlString} to come from a random mirror. + */ + public static String switchUrlToNewMirror(@Nullable String urlString, Repo repo2) { + String lastWorkingMirror = lastWorkingMirrorArray.get(repo2.getId()); + if (lastWorkingMirror == null) { + lastWorkingMirror = repo2.address; + } + String mirror = repo2.getRandomMirror(lastWorkingMirror); + lastWorkingMirrorArray.put(repo2.getId(), mirror); + return urlString.replace(lastWorkingMirror, mirror); + } + public static int getTimeout() { return timeout; } + /** + * Reset the retry counter and timeout to defaults, and set the last + * working mirror to the canonical URL. + * + * @see #getNewMirrorOnError(String, Repo) + */ public static void resetMirrorVars() { - // Reset last working mirror, numtries, and timeout for (int i = 0; i < lastWorkingMirrorArray.size(); i++) { lastWorkingMirrorArray.removeAt(i); } diff --git a/app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java b/app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java index 8d57e185e..c53b52f3e 100644 --- a/app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java +++ b/app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java @@ -141,7 +141,7 @@ public class IndexV1Updater extends IndexUpdater { | SSLHandshakeException | SSLKeyException | SSLPeerUnverifiedException | SSLProtocolException | ProtocolException | UnknownHostException e) { // if the above list changes, also change below and in DownloaderService.handleIntent() - Utils.debugLog(TAG, "Trying to download the index from a mirror"); + Utils.debugLog(TAG, "Trying to download the index from a mirror: " + e.getMessage()); // Mirror logic here, so that the default download code is untouched. String mirrorUrl; String prevMirrorUrl = indexUrl; @@ -149,7 +149,7 @@ public class IndexV1Updater extends IndexUpdater { int n = repo.getMirrorCount() * 3; // 3 is the number of timeouts we have. 10s, 30s & 60s for (int i = 0; i <= n; i++) { try { - mirrorUrl = FDroidApp.getMirror(prevMirrorUrl, repo); + mirrorUrl = FDroidApp.getNewMirrorOnError(prevMirrorUrl, repo); prevMirrorUrl = mirrorUrl; downloader = DownloaderFactory.create(context, mirrorUrl); downloader.setCacheTag(repo.lastetag); diff --git a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java index e56a0d139..07df7502d 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java +++ b/app/src/main/java/org/fdroid/fdroid/data/DBHelper.java @@ -1397,7 +1397,7 @@ public class DBHelper extends SQLiteOpenHelper { /** * Insert a new repo into the database. This also initializes the list of * "mirror" URLs. There should always be at least one URL there, since the - * logic in {@link org.fdroid.fdroid.FDroidApp#getMirror(String, Repo)} + * logic in {@link org.fdroid.fdroid.FDroidApp#switchUrlToNewMirror(String, Repo)} * expects at least one entry in the mirrors list. */ private void insertRepo(SQLiteDatabase db, String name, String address, diff --git a/app/src/main/java/org/fdroid/fdroid/data/Repo.java b/app/src/main/java/org/fdroid/fdroid/data/Repo.java index 1333adc7c..2bebb1854 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/Repo.java +++ b/app/src/main/java/org/fdroid/fdroid/data/Repo.java @@ -395,7 +395,7 @@ public class Repo extends ValueObject { * or USB OTG drive. * * @see FDroidApp#resetMirrorVars() - * @see FDroidApp#getMirror(String, Repo) + * @see FDroidApp#switchUrlToNewMirror(String, Repo) * @see FDroidApp#getTimeout() */ public String getRandomMirror(String mirrorToSkip) { diff --git a/app/src/main/java/org/fdroid/fdroid/installer/InstallManagerService.java b/app/src/main/java/org/fdroid/fdroid/installer/InstallManagerService.java index f42ce380f..57302885f 100644 --- a/app/src/main/java/org/fdroid/fdroid/installer/InstallManagerService.java +++ b/app/src/main/java/org/fdroid/fdroid/installer/InstallManagerService.java @@ -11,6 +11,7 @@ import android.content.pm.PackageInfo; import android.net.Uri; import android.os.IBinder; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; import android.support.v4.content.LocalBroadcastManager; import android.text.TextUtils; import android.util.Log; @@ -23,6 +24,7 @@ import org.fdroid.fdroid.Utils; import org.fdroid.fdroid.compat.PackageManagerCompat; import org.fdroid.fdroid.data.Apk; import org.fdroid.fdroid.data.App; +import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.net.Downloader; import org.fdroid.fdroid.net.DownloaderService; @@ -211,7 +213,7 @@ public class InstallManagerService extends Service { long apkFileSize = apkFilePath.length(); if (!apkFilePath.exists() || apkFileSize < apk.size) { Utils.debugLog(TAG, "download " + urlString + " " + apkFilePath); - DownloaderService.queue(this, urlString, apk.repoId, urlString); + DownloaderService.queue(this, switchUrlToNewMirror(urlString, apk.repoId), apk.repoId, urlString); } else if (ApkCache.apkIsCached(apkFilePath, apk)) { Utils.debugLog(TAG, "skip download, we have it, straight to install " + urlString + " " + apkFilePath); sendBroadcast(intent.getData(), Downloader.ACTION_STARTED, apkFilePath); @@ -219,7 +221,7 @@ public class InstallManagerService extends Service { } else { Utils.debugLog(TAG, "delete and download again " + urlString + " " + apkFilePath); apkFilePath.delete(); - DownloaderService.queue(this, urlString, apk.repoId, urlString); + DownloaderService.queue(this, switchUrlToNewMirror(urlString, apk.repoId), apk.repoId, urlString); } return START_REDELIVER_INTENT; // if killed before completion, retry Intent @@ -232,6 +234,24 @@ public class InstallManagerService extends Service { localBroadcastManager.sendBroadcast(intent); } + /** + * Tries to return a version of {@code urlString} from a mirror, if there + * is an error, it just returns {@code urlString}. + * + * @see FDroidApp#getNewMirrorOnError(String, org.fdroid.fdroid.data.Repo) + */ + public String getNewMirrorOnError(@Nullable String urlString, long repoId) { + try { + return FDroidApp.getNewMirrorOnError(urlString, RepoProvider.Helper.findById(this, repoId)); + } catch (IOException e) { + return urlString; + } + } + + public String switchUrlToNewMirror(@Nullable String urlString, long repoId) { + return FDroidApp.switchUrlToNewMirror(urlString, RepoProvider.Helper.findById(this, repoId)); + } + /** * Check if any OBB files are available, and if so, download and install them. This * also deletes any obsolete OBB files, per the spec, since there can be only one @@ -290,13 +310,13 @@ public class InstallManagerService extends Service { } else if (Downloader.ACTION_INTERRUPTED.equals(action)) { localBroadcastManager.unregisterReceiver(this); } else if (Downloader.ACTION_CONNECTION_FAILED.equals(action)) { - DownloaderService.queue(context, urlString, 0, urlString); + DownloaderService.queue(context, getNewMirrorOnError(urlString, 0), 0, urlString); } else { throw new RuntimeException("intent action not handled!"); } } }; - DownloaderService.queue(this, obbUrlString, 0, obbUrlString); + DownloaderService.queue(this, switchUrlToNewMirror(obbUrlString, 0), 0, obbUrlString); localBroadcastManager.registerReceiver(downloadReceiver, DownloaderService.getIntentFilter(obbUrlString)); } @@ -354,7 +374,9 @@ public class InstallManagerService extends Service { break; case Downloader.ACTION_CONNECTION_FAILED: try { - DownloaderService.queue(context, FDroidApp.getMirror(mirrorUrlString, repoId), repoId, urlString); + String currentUrlString = FDroidApp.getNewMirrorOnError(mirrorUrlString, + RepoProvider.Helper.findById(InstallManagerService.this, repoId)); + DownloaderService.queue(context, currentUrlString, repoId, urlString); DownloaderService.setTimeout(FDroidApp.getTimeout()); } catch (IOException e) { appUpdateStatusManager.setDownloadError(urlString, intent.getStringExtra(Downloader.EXTRA_ERROR_MESSAGE));