Karl/BobStore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'big-cache-update' into 'master'

Browse Source 
Big cache update

So I messed up the caching a bit with my update in the past, so this is a big update to fix lots of bugs (hopefully) and add a couple of nice cache clean up features.  This should move us towards making F-Droid maintain itself more and more.  More comments in the commits.

See merge request !378
This commit is contained in:
Hans-Christoph Steiner 2016-08-16 17:02:45 +00:00
commit 20a5f42359
9 changed files with 141 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
app/src/test/java/org/fdroid/fdroid/installer/ApkCacheTest.java → app/src/androidTest/java/org/fdroid/fdroid/CleanCacheServiceTest.java
View File

@ -1,11 +1,10 @@
package org.fdroid.fdroid.installer;
package org.fdroid.fdroid;
import android.support.test.runner.AndroidJUnit4;
import org.apache.commons.io.FileUtils;
import org.fdroid.fdroid.BuildConfig;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.RobolectricGradleTestRunner;
import org.robolectric.annotation.Config;
import java.io.File;
import java.io.IOException;
@ -13,10 +12,10 @@ import java.io.IOException;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
// TODO: Use sdk=24 when Robolectric supports this
@Config(constants = BuildConfig.class, sdk = 23)
@RunWith(RobolectricGradleTestRunner.class)
public class ApkCacheTest {
@RunWith(AndroidJUnit4.class)
public class CleanCacheServiceTest {
public static final String TAG = "CleanCacheServiceTest";
@Test
public void testClearOldFiles() throws IOException, InterruptedException {
@ -45,13 +44,18 @@ public class ApkCacheTest {
assertTrue(second.createNewFile());
assertTrue(second.exists());
ApkCache.clearOldFiles(dir, 3);
CleanCacheService.clearOldFiles(dir, 3000); // check all in dir
assertFalse(first.exists());
assertTrue(second.exists());
Thread.sleep(7000);
ApkCache.clearOldFiles(dir, 3);
CleanCacheService.clearOldFiles(second, 3000); // check just second file
assertFalse(first.exists());
assertFalse(second.exists());
// make sure it doesn't freak out on a non-existant file
File nonexistant = new File(tempDir, "nonexistant");
CleanCacheService.clearOldFiles(nonexistant, 1);
CleanCacheService.clearOldFiles(null, 1);
}
}

87
app/src/main/java/org/fdroid/fdroid/CleanCacheService.java
View File

@ -1,23 +1,36 @@
package org.fdroid.fdroid;
import android.annotation.TargetApi;
import android.app.AlarmManager;
import android.app.IntentService;
import android.app.PendingIntent;
import android.content.Context;
import android.content.Intent;
import android.os.Build;
import android.os.Process;
import android.os.SystemClock;
import android.system.ErrnoException;
import android.system.Os;
import android.system.StructStat;
import org.apache.commons.io.FileUtils;
import org.fdroid.fdroid.installer.ApkCache;
import java.io.File;
import java.util.concurrent.TimeUnit;
/**
* Handles cleaning up caches files that are not going to be used, and do not
* block the operation of the app itself. For things that must happen before
* F-Droid starts normal operation, that should go into
* {@link FDroidApp#onCreate()}
* {@link FDroidApp#onCreate()}.
* <p>
* These files should only be deleted when they are at least an hour-ish old,
* in case they are actively in use while {@code CleanCacheService} is running.
* {@link #clearOldFiles(File, long)} checks the file age using access time from
* {@link StructStat#st_atime} on {@link android.os.Build.VERSION_CODES#LOLLIPOP}
* and newer. On older Android, last modified time from {@link File#lastModified()}
* is used.
*/
public class CleanCacheService extends IntentService {
@ -28,9 +41,9 @@ public class CleanCacheService extends IntentService {
*/
public static void schedule(Context context) {
long keepTime = Preferences.get().getKeepCacheTime();
long interval = 604800000; // 1 day
long interval = TimeUnit.DAYS.toMillis(1);
if (keepTime < interval) {
interval = keepTime * 1000;
interval = keepTime;
}
Intent intent = new Intent(context, CleanCacheService.class);
@ -52,9 +65,20 @@ public class CleanCacheService extends IntentService {
return;
}
Process.setThreadPriority(Process.THREAD_PRIORITY_LOWEST);
ApkCache.clearApkCache(this);
deleteExpiredApksFromCache();
deleteStrayIndexFiles();
deleteOldInstallerFiles();
deleteOldIcons();
}
/**
* All downloaded APKs will be cached for a certain amount of time, which is
* specified by the user in the "Keep Cache Time" preference. This removes
* any APK in the cache that is older than that preference specifies.
*/
private void deleteExpiredApksFromCache() {
File cacheDir = ApkCache.getApkCacheDir(getBaseContext());
clearOldFiles(cacheDir, Preferences.get().getKeepCacheTime());
}
/**
@ -74,7 +98,7 @@ public class CleanCacheService extends IntentService {
for (File f : files) {
if (f.getName().startsWith("install-")) {
FileUtils.deleteQuietly(f);
clearOldFiles(f, TimeUnit.HOURS.toMillis(1));
}
}
}
@ -104,11 +128,58 @@ public class CleanCacheService extends IntentService {
for (File f : files) {
if (f.getName().startsWith("index-")) {
FileUtils.deleteQuietly(f);
clearOldFiles(f, TimeUnit.HOURS.toMillis(1));
}
if (f.getName().startsWith("dl-")) {
FileUtils.deleteQuietly(f);
clearOldFiles(f, TimeUnit.HOURS.toMillis(1));
}
}
}
}
/**
* Delete cached icons that have not been accessed in over a year.
*/
private void deleteOldIcons() {
clearOldFiles(Utils.getIconsCacheDir(this), TimeUnit.DAYS.toMillis(365));
}
/**
* Recursively delete files in {@code f} that were last used
* {@code millisAgo} milliseconds ago. On {@code android-21} and newer, this
* is based on the last access of the file, on older Android versions, it is
* based on the last time the file was modified, e.g. downloaded.
*
* @param f The file or directory to clean
* @param millisAgo The number of milliseconds old that marks a file for deletion.
*/
@TargetApi(21)
public static void clearOldFiles(File f, long millisAgo) {
if (f == null) {
return;
}
long olderThan = System.currentTimeMillis() - millisAgo;
if (f.isDirectory()) {
File[] files = f.listFiles();
if (files == null) {
return;
}
for (File file : files) {
clearOldFiles(file, millisAgo);
}
f.delete();
} else if (Build.VERSION.SDK_INT < 21) {
if (FileUtils.isFileOlder(f, olderThan)) {
f.delete();
}
} else {
try {
StructStat stat = Os.lstat(f.getAbsolutePath());
if ((stat.st_atime * 1000L) < olderThan) {
f.delete();
}
} catch (ErrnoException e) {
e.printStackTrace();
}
}
}
}

5
app/src/main/java/org/fdroid/fdroid/FDroidApp.java
View File

@ -45,7 +45,6 @@ import com.nostra13.universalimageloader.cache.disc.impl.LimitedAgeDiskCache;
import com.nostra13.universalimageloader.cache.disc.naming.FileNameGenerator;
import com.nostra13.universalimageloader.core.ImageLoader;
import com.nostra13.universalimageloader.core.ImageLoaderConfiguration;
import com.nostra13.universalimageloader.utils.StorageUtils;
import org.acra.ACRA;
import org.acra.ReportingInteractionMode;
@ -60,7 +59,6 @@ import org.fdroid.fdroid.data.Repo;
import org.fdroid.fdroid.net.IconDownloader;
import org.fdroid.fdroid.net.WifiStateChangeService;
import java.io.File;
import java.net.URL;
import java.net.URLStreamHandler;
import java.net.URLStreamHandlerFactory;
@ -262,8 +260,7 @@ public class FDroidApp extends Application {
ImageLoaderConfiguration config = new ImageLoaderConfiguration.Builder(getApplicationContext())
.imageDownloader(new IconDownloader(getApplicationContext()))
.diskCache(new LimitedAgeDiskCache(
new File(StorageUtils.getCacheDirectory(getApplicationContext(), true),
"icons"),
Utils.getIconsCacheDir(this),
null,
new FileNameGenerator() {
@Override

28
app/src/main/java/org/fdroid/fdroid/Preferences.java
View File

@ -16,6 +16,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import info.guardianproject.netcipher.NetCipher;
@ -30,11 +31,9 @@ public final class Preferences implements SharedPreferences.OnSharedPreferenceCh
private static final String TAG = "Preferences";
private final Context context;
private final SharedPreferences preferences;
private Preferences(Context context) {
this.context = context;
preferences = PreferenceManager.getDefaultSharedPreferences(context);
preferences.registerOnSharedPreferenceChangeListener(this);
if (preferences.getString(PREF_LOCAL_REPO_NAME, null) == null) {
@ -72,7 +71,7 @@ public final class Preferences implements SharedPreferences.OnSharedPreferenceCh
private static final int DEFAULT_UPD_HISTORY = 14;
private static final boolean DEFAULT_PRIVILEGED_INSTALLER = false;
//private static final boolean DEFAULT_LOCAL_REPO_BONJOUR = true;
private static final long DEFAULT_KEEP_CACHE_SECONDS = 86400; // one day
private static final long DEFAULT_KEEP_CACHE_TIME = TimeUnit.DAYS.toMillis(1);
private static final boolean DEFAULT_UNSTABLE_UPDATES = false;
//private static final boolean DEFAULT_LOCAL_REPO_HTTPS = false;
private static final boolean DEFAULT_INCOMP_VER = false;
@ -136,14 +135,29 @@ public final class Preferences implements SharedPreferences.OnSharedPreferenceCh
private static final String PREF_CACHE_APK = "cacheDownloaded";
/**
* Time in seconds to keep cached files. Anything that has been around longer will be deleted
* Time in millis to keep cached files. Anything that has been around longer will be deleted
*/
public long getKeepCacheTime() {
String value = preferences.getString(PREF_KEEP_CACHE_TIME, String.valueOf(DEFAULT_KEEP_CACHE_SECONDS));
String value = preferences.getString(PREF_KEEP_CACHE_TIME,
String.valueOf(DEFAULT_KEEP_CACHE_TIME));
// the first time this was migrated, it was botched, so reset to default
switch (value) {
case "3600":
case "86400":
case "604800":
case "2592000":
case "31449600":
case "2147483647":
SharedPreferences.Editor editor = preferences.edit();
editor.remove(PREF_KEEP_CACHE_TIME);
editor.apply();
return Preferences.DEFAULT_KEEP_CACHE_TIME;
}
if (preferences.contains(PREF_CACHE_APK)) {
if (preferences.getBoolean(PREF_CACHE_APK, false)) {
value = context.getString(R.string.keep_forever);
value = String.valueOf(Long.MAX_VALUE);
}
SharedPreferences.Editor editor = preferences.edit();
editor.remove(PREF_CACHE_APK);
@ -154,7 +168,7 @@ public final class Preferences implements SharedPreferences.OnSharedPreferenceCh
try {
return Long.parseLong(value);
} catch (NumberFormatException e) {
return DEFAULT_KEEP_CACHE_SECONDS;
return DEFAULT_KEEP_CACHE_TIME;
}
}

9
app/src/main/java/org/fdroid/fdroid/Utils.java
View File

@ -33,6 +33,7 @@ import android.util.Log;
import com.nostra13.universalimageloader.core.DisplayImageOptions;
import com.nostra13.universalimageloader.core.assist.ImageScaleType;
import com.nostra13.universalimageloader.core.display.FadeInBitmapDisplayer;
import com.nostra13.universalimageloader.utils.StorageUtils;
import org.fdroid.fdroid.compat.FileCompat;
import org.fdroid.fdroid.data.Repo;
@ -107,6 +108,14 @@ public final class Utils {
return "/icons-120/";
}
/**
* @return the directory where cached icons are stored
*/
public static File getIconsCacheDir(Context context) {
File cacheDir = StorageUtils.getCacheDirectory(context.getApplicationContext(), true);
return new File(cacheDir, "icons");
}
public static void copy(InputStream input, OutputStream output) throws IOException {
byte[] buffer = new byte[BUFFER_SIZE];
while (true) {

37
app/src/main/java/org/fdroid/fdroid/installer/ApkCache.java
View File

@ -26,7 +26,6 @@ import com.nostra13.universalimageloader.utils.StorageUtils;
import org.apache.commons.io.FileUtils;
import org.fdroid.fdroid.Hasher;
import org.fdroid.fdroid.Preferences;
import org.fdroid.fdroid.data.Apk;
import org.fdroid.fdroid.data.SanitizedFile;
@ -117,19 +116,14 @@ public class ApkCache {
}
}
public static void clearApkCache(Context context) {
clearOldFiles(getApkCacheDir(context), Preferences.get().getKeepCacheTime());
}
/**
* This location is only for caching, do not install directly from this location
* because if the file is on the External Storage, any other app could swap out
* the APK while the install was in process, allowing malware to install things.
* Using {@link Installer#installPackage(Uri localApkUri, Uri downloadUri, String packageName)}
* Using {@link Installer#installPackage(Uri, Uri, Apk)}
* is fine since that does the right thing.
*/
private static File getApkCacheDir(Context context) {
public static File getApkCacheDir(Context context) {
File apkCacheDir = new File(StorageUtils.getCacheDirectory(context, true), CACHE_DIR);
if (apkCacheDir.isFile()) {
apkCacheDir.delete();
@ -139,31 +133,4 @@ public class ApkCache {
}
return apkCacheDir;
}
/**
* Recursively delete files in {@code dir} that were last modified
* {@code secondsAgo} seconds ago, e.g. when it was downloaded.
*
* @param dir The directory to recurse in
* @param secondsAgo The number of seconds old that marks a file for deletion.
*/
public static void clearOldFiles(File dir, long secondsAgo) {
if (dir == null) {
return;
}
File[] files = dir.listFiles();
if (files == null) {
return;
}
long olderThan = System.currentTimeMillis() - (secondsAgo * 1000L);
for (File f : files) {
if (f.isDirectory()) {
clearOldFiles(f, olderThan);
f.delete();
}
if (FileUtils.isFileOlder(f, olderThan)) {
f.delete();
}
}
}
}

8
app/src/main/java/org/fdroid/fdroid/views/fragments/PreferencesFragment.java
View File

@ -15,6 +15,7 @@ import android.text.Html;
import android.text.TextUtils;
import org.fdroid.fdroid.AppDetails;
import org.fdroid.fdroid.CleanCacheService;
import org.fdroid.fdroid.FDroidApp;
import org.fdroid.fdroid.Preferences;
import org.fdroid.fdroid.PreferencesActivity;
@ -49,6 +50,7 @@ public class PreferencesFragment extends PreferenceFragment
private static final int REQUEST_INSTALL_ORBOT = 0x1234;
private CheckBoxPreference enableProxyCheckPref;
private CheckBoxPreference useTorCheckPref;
private long currentKeepCacheTime;
@Override
public void onCreate(Bundle savedInstanceState) {
@ -144,6 +146,10 @@ public class PreferencesFragment extends PreferenceFragment
case Preferences.PREF_KEEP_CACHE_TIME:
entrySummary(key);
if (changing
&& currentKeepCacheTime != Preferences.get().getKeepCacheTime()) {
CleanCacheService.schedule(getContext());
}
break;
case Preferences.PREF_EXPERT:
@ -283,6 +289,8 @@ public class PreferencesFragment extends PreferenceFragment
updateSummary(key, false);
}
currentKeepCacheTime = Preferences.get().getKeepCacheTime();
initPrivilegedInstallerPreference();
initManagePrivilegedAppPreference();
// this pref's default is dynamically set based on whether Orbot is installed

12
app/src/main/res/values/donottranslate.xml
View File

@ -26,12 +26,12 @@
</string-array>
<string-array name="keepCacheValues">
<item>3600</item>
<item>86400</item>
<item>604800</item>
<item>2592000</item>
<item>31449600</item>
<item>2147483647</item>
<item>3600000</item>
<item>86400000</item>
<item>604800000</item>
<item>2592000000</item>
<item>31536000000</item>
<item>9223372036854775807</item>
</string-array>
<string-array name="themeValues">

1
app/src/main/res/xml/preferences.xml
View File

@ -75,6 +75,7 @@
<PreferenceCategory android:title="@string/other">
<ListPreference android:title="@string/cache_downloaded"
android:key="keepCacheFor"
android:defaultValue="86400000"
android:entries="@array/keepCacheNames"
android:entryValues="@array/keepCacheValues" />
<CheckBoxPreference android:title="@string/expert"