Karl/BobStore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

choose random mirror for each package/APK download

Browse Source 
This spreads downloads across all available mirrors randomly.  This could
definitely be improved, like choosing the fastest or nearest mirror, or
only .onion addresses on Tor.  This will improve the current situation and
should reduce the load on f-droid.org a lot.

fdroidclient#1696
This commit is contained in:
Hans-Christoph Steiner 2019-02-19 14:06:54 +01:00
parent 89422e9c8f
commit dd14b9e315
5 changed files with 58 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
app/src/main/java/org/fdroid/fdroid/FDroidApp.java
View File

@ -39,6 +39,7 @@ import android.net.Uri;
import android.os.Build;
import android.os.Environment;
import android.os.StrictMode;
import android.support.annotation.Nullable;
import android.support.v4.util.LongSparseArray;
import android.text.TextUtils;
import android.util.Base64;
@ -66,7 +67,6 @@ import org.fdroid.fdroid.compat.PRNGFixes;
import org.fdroid.fdroid.data.AppProvider;
import org.fdroid.fdroid.data.InstalledAppProviderService;
import org.fdroid.fdroid.data.Repo;
import org.fdroid.fdroid.data.RepoProvider;
import org.fdroid.fdroid.installer.ApkFileProvider;
import org.fdroid.fdroid.installer.InstallHistoryService;
import org.fdroid.fdroid.localrepo.SDCardScannerService;
@ -245,13 +245,6 @@ public class FDroidApp extends Application {
repo = new Repo();
}
/**
* @see #getMirror(String, Repo)
*/
public static String getMirror(String urlString, long repoId) throws IOException {
return getMirror(urlString, RepoProvider.Helper.findById(getInstance(), repoId));
}
/**
* Each time this is called, it will return a mirror from the pool of
* mirrors. If it reaches the end of the list of mirrors, it will start
@ -260,17 +253,18 @@ public class FDroidApp extends Application {
* again, it will do one last pass through the list with the timeout set to
* {@link Downloader#LONGEST_TIMEOUT}. After that, this gives up with a
* {@link IOException}.
* <p>
* {@link #lastWorkingMirrorArray} is used to track the last mirror URL used,
* so it can be used in the string replacement operating when converting a
* download URL to point to a different mirror. Download URLs can be
* anything from {@code index-v1.jar} to APKs to icons to screenshots.
*
* @see #resetMirrorVars()
* @see #getTimeout()
* @see Repo#getRandomMirror(String)
*/
public static String getMirror(String urlString, Repo repo2) throws IOException {
public static String getNewMirrorOnError(@Nullable String urlString, Repo repo2) throws IOException {
if (repo2.hasMirrors()) {
String lastWorkingMirror = lastWorkingMirrorArray.get(repo2.getId());
if (lastWorkingMirror == null) {
lastWorkingMirror = repo2.address;
}
if (numTries <= 0) {
if (timeout == Downloader.DEFAULT_TIMEOUT) {
timeout = Downloader.SECOND_TIMEOUT;
@ -286,24 +280,37 @@ public class FDroidApp extends Application {
if (numTries == Integer.MAX_VALUE) {
numTries = repo2.getMirrorCount();
}
String mirror = repo2.getRandomMirror(lastWorkingMirror);
String newUrl = urlString.replace(lastWorkingMirror, mirror);
Utils.debugLog(TAG, "Trying mirror " + mirror + " after " + lastWorkingMirror + " failed," +
" timeout=" + timeout / 1000 + "s");
lastWorkingMirrorArray.put(repo2.getId(), mirror);
numTries--;
return newUrl;
return switchUrlToNewMirror(urlString, repo2);
} else {
throw new IOException("No mirrors available");
}
}
/**
* Switch the URL in {@code urlString} to come from a random mirror.
*/
public static String switchUrlToNewMirror(@Nullable String urlString, Repo repo2) {
String lastWorkingMirror = lastWorkingMirrorArray.get(repo2.getId());
if (lastWorkingMirror == null) {
lastWorkingMirror = repo2.address;
}
String mirror = repo2.getRandomMirror(lastWorkingMirror);
lastWorkingMirrorArray.put(repo2.getId(), mirror);
return urlString.replace(lastWorkingMirror, mirror);
}
public static int getTimeout() {
return timeout;
}
/**
* Reset the retry counter and timeout to defaults, and set the last
* working mirror to the canonical URL.
*
* @see #getNewMirrorOnError(String, Repo)
*/
public static void resetMirrorVars() {
// Reset last working mirror, numtries, and timeout
for (int i = 0; i < lastWorkingMirrorArray.size(); i++) {
lastWorkingMirrorArray.removeAt(i);
}

4
app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java
View File

@ -141,7 +141,7 @@ public class IndexV1Updater extends IndexUpdater {
| SSLHandshakeException | SSLKeyException | SSLPeerUnverifiedException | SSLProtocolException
| ProtocolException | UnknownHostException e) {
// if the above list changes, also change below and in DownloaderService.handleIntent()
Utils.debugLog(TAG, "Trying to download the index from a mirror");
Utils.debugLog(TAG, "Trying to download the index from a mirror: " + e.getMessage());
// Mirror logic here, so that the default download code is untouched.
String mirrorUrl;
String prevMirrorUrl = indexUrl;
@ -149,7 +149,7 @@ public class IndexV1Updater extends IndexUpdater {
int n = repo.getMirrorCount() * 3; // 3 is the number of timeouts we have. 10s, 30s & 60s
for (int i = 0; i <= n; i++) {
try {
mirrorUrl = FDroidApp.getMirror(prevMirrorUrl, repo);
mirrorUrl = FDroidApp.getNewMirrorOnError(prevMirrorUrl, repo);
prevMirrorUrl = mirrorUrl;
downloader = DownloaderFactory.create(context, mirrorUrl);
downloader.setCacheTag(repo.lastetag);

2
app/src/main/java/org/fdroid/fdroid/data/DBHelper.java
View File

@ -1397,7 +1397,7 @@ public class DBHelper extends SQLiteOpenHelper {
/**
* Insert a new repo into the database. This also initializes the list of
* "mirror" URLs. There should always be at least one URL there, since the
* logic in {@link org.fdroid.fdroid.FDroidApp#getMirror(String, Repo)}
* logic in {@link org.fdroid.fdroid.FDroidApp#switchUrlToNewMirror(String, Repo)}
* expects at least one entry in the mirrors list.
*/
private void insertRepo(SQLiteDatabase db, String name, String address,

2
app/src/main/java/org/fdroid/fdroid/data/Repo.java
View File

@ -395,7 +395,7 @@ public class Repo extends ValueObject {
* or USB OTG drive.
*
* @see FDroidApp#resetMirrorVars()
* @see FDroidApp#getMirror(String, Repo)
* @see FDroidApp#switchUrlToNewMirror(String, Repo)
* @see FDroidApp#getTimeout()
*/
public String getRandomMirror(String mirrorToSkip) {

32
app/src/main/java/org/fdroid/fdroid/installer/InstallManagerService.java
View File

@ -11,6 +11,7 @@ import android.content.pm.PackageInfo;
import android.net.Uri;
import android.os.IBinder;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.v4.content.LocalBroadcastManager;
import android.text.TextUtils;
import android.util.Log;
@ -23,6 +24,7 @@ import org.fdroid.fdroid.Utils;
import org.fdroid.fdroid.compat.PackageManagerCompat;
import org.fdroid.fdroid.data.Apk;
import org.fdroid.fdroid.data.App;
import org.fdroid.fdroid.data.RepoProvider;
import org.fdroid.fdroid.net.Downloader;
import org.fdroid.fdroid.net.DownloaderService;
@ -211,7 +213,7 @@ public class InstallManagerService extends Service {
long apkFileSize = apkFilePath.length();
if (!apkFilePath.exists() || apkFileSize < apk.size) {
Utils.debugLog(TAG, "download " + urlString + " " + apkFilePath);
DownloaderService.queue(this, urlString, apk.repoId, urlString);
DownloaderService.queue(this, switchUrlToNewMirror(urlString, apk.repoId), apk.repoId, urlString);
} else if (ApkCache.apkIsCached(apkFilePath, apk)) {
Utils.debugLog(TAG, "skip download, we have it, straight to install " + urlString + " " + apkFilePath);
sendBroadcast(intent.getData(), Downloader.ACTION_STARTED, apkFilePath);
@ -219,7 +221,7 @@ public class InstallManagerService extends Service {
} else {
Utils.debugLog(TAG, "delete and download again " + urlString + " " + apkFilePath);
apkFilePath.delete();
DownloaderService.queue(this, urlString, apk.repoId, urlString);
DownloaderService.queue(this, switchUrlToNewMirror(urlString, apk.repoId), apk.repoId, urlString);
}
return START_REDELIVER_INTENT; // if killed before completion, retry Intent
@ -232,6 +234,24 @@ public class InstallManagerService extends Service {
localBroadcastManager.sendBroadcast(intent);
}
/**
* Tries to return a version of {@code urlString} from a mirror, if there
* is an error, it just returns {@code urlString}.
*
* @see FDroidApp#getNewMirrorOnError(String, org.fdroid.fdroid.data.Repo)
*/
public String getNewMirrorOnError(@Nullable String urlString, long repoId) {
try {
return FDroidApp.getNewMirrorOnError(urlString, RepoProvider.Helper.findById(this, repoId));
} catch (IOException e) {
return urlString;
}
}
public String switchUrlToNewMirror(@Nullable String urlString, long repoId) {
return FDroidApp.switchUrlToNewMirror(urlString, RepoProvider.Helper.findById(this, repoId));
}
/**
* Check if any OBB files are available, and if so, download and install them. This
* also deletes any obsolete OBB files, per the spec, since there can be only one
@ -290,13 +310,13 @@ public class InstallManagerService extends Service {
} else if (Downloader.ACTION_INTERRUPTED.equals(action)) {
localBroadcastManager.unregisterReceiver(this);
} else if (Downloader.ACTION_CONNECTION_FAILED.equals(action)) {
DownloaderService.queue(context, urlString, 0, urlString);
DownloaderService.queue(context, getNewMirrorOnError(urlString, 0), 0, urlString);
} else {
throw new RuntimeException("intent action not handled!");
}
}
};
DownloaderService.queue(this, obbUrlString, 0, obbUrlString);
DownloaderService.queue(this, switchUrlToNewMirror(obbUrlString, 0), 0, obbUrlString);
localBroadcastManager.registerReceiver(downloadReceiver,
DownloaderService.getIntentFilter(obbUrlString));
}
@ -354,7 +374,9 @@ public class InstallManagerService extends Service {
break;
case Downloader.ACTION_CONNECTION_FAILED:
try {
DownloaderService.queue(context, FDroidApp.getMirror(mirrorUrlString, repoId), repoId, urlString);
String currentUrlString = FDroidApp.getNewMirrorOnError(mirrorUrlString,
RepoProvider.Helper.findById(InstallManagerService.this, repoId));
DownloaderService.queue(context, currentUrlString, repoId, urlString);
DownloaderService.setTimeout(FDroidApp.getTimeout());
} catch (IOException e) {
appUpdateStatusManager.setDownloadError(urlString, intent.getStringExtra(Downloader.EXTRA_ERROR_MESSAGE));