From f9bc21907386a6e9c24c69ddb900debb358503d0 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 20 Dec 2018 23:35:00 +0100 Subject: [PATCH] SDCardScannerService for using repos from SD Cards Creates an IntentService subclass for scanning removable "external storage" for F-Droid package repos, e.g. SD Cards. This is intented to support sharable package repos, so it ignores non-removable storage, like the fake emulated sdcard from devices with only built-in storage. This method will only ever allow for reading repos, never writing. It also will not work for removeable storage devices plugged in via USB, since do not show up as "External Storage" * https://stackoverflow.com/a/40201333 * https://commonsware.com/blog/2017/11/14/storage-situation-external-storage.html closes #1377 --- .../localrepo/SDCardScannerService.java | 30 ++++ app/src/full/AndroidManifest.xml | 4 + .../localrepo/SDCardScannerService.java | 168 ++++++++++++++++++ .../fdroid/views/main/NearbyViewBinder.java | 4 +- .../java/org/fdroid/fdroid/FDroidApp.java | 3 + .../org/fdroid/fdroid/data/NewRepoConfig.java | 6 +- .../fdroid/fdroid/net/DownloaderFactory.java | 2 + .../fdroid/net/LocalFileDownloader.java | 86 +++++++++ .../fdroid/views/ManageReposActivity.java | 3 +- 9 files changed, 302 insertions(+), 4 deletions(-) create mode 100644 app/src/basic/java/org/fdroid/fdroid/localrepo/SDCardScannerService.java create mode 100644 app/src/full/java/org/fdroid/fdroid/localrepo/SDCardScannerService.java create mode 100644 app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java diff --git a/app/src/basic/java/org/fdroid/fdroid/localrepo/SDCardScannerService.java b/app/src/basic/java/org/fdroid/fdroid/localrepo/SDCardScannerService.java new file mode 100644 index 000000000..3fc00c4ce --- /dev/null +++ b/app/src/basic/java/org/fdroid/fdroid/localrepo/SDCardScannerService.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2018 Hans-Christoph Steiner + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 3 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +package org.fdroid.fdroid.localrepo; + +import android.content.Context; + +/** + * Dummy version for basic app flavor. + */ +public class SDCardScannerService { + public static void scan(Context context) { + } +} diff --git a/app/src/full/AndroidManifest.xml b/app/src/full/AndroidManifest.xml index 5bdd57aec..a98dd272d 100644 --- a/app/src/full/AndroidManifest.xml +++ b/app/src/full/AndroidManifest.xml @@ -38,6 +38,7 @@ + @@ -80,6 +81,9 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 3 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +package org.fdroid.fdroid.localrepo; + +import android.Manifest; +import android.app.IntentService; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.net.Uri; +import android.os.Build; +import android.os.Environment; +import android.os.Process; +import android.support.v4.content.ContextCompat; +import android.util.Log; +import org.fdroid.fdroid.IndexV1Updater; +import org.fdroid.fdroid.IndexUpdater; +import org.fdroid.fdroid.Utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * An {@link IntentService} subclass for scanning removable "external storage" + * for F-Droid package repos, e.g. SD Cards. This is intented to support + * sharable package repos, so it ignores non-removable storage, like the fake + * emulated sdcard from devices with only built-in storage. This method will + * only ever allow for reading repos, never writing. It also will not work + * for removeable storage devices plugged in via USB, since do not show up as + * "External Storage" + * + * @see TreeUriScannerIntentService TreeUri method for writing repos to be shared + * @see Universal way to write to external SD card on Android + * @see The Storage Situation: External Storage + */ +public class SDCardScannerService extends IntentService { + public static final String TAG = "SDCardScannerService"; + + private static final String ACTION_SCAN = "org.fdroid.fdroid.localrepo.SCAN"; + + private static final List SKIP_DIRS = Arrays.asList(".android_secure", "LOST.DIR"); + + public SDCardScannerService() { + super("SDCardScannerService"); + } + + public static void scan(Context context) { + Intent intent = new Intent(context, SDCardScannerService.class); + intent.setAction(ACTION_SCAN); + context.startService(intent); + } + + @Override + protected void onHandleIntent(Intent intent) { + if (intent == null || !ACTION_SCAN.equals(intent.getAction())) { + return; + } + Process.setThreadPriority(Process.THREAD_PRIORITY_LOWEST); + + HashSet files = new HashSet<>(); + if (Build.VERSION.SDK_INT < 21) { + if (Environment.isExternalStorageRemovable()) { + File sdcard = Environment.getExternalStorageDirectory(); + String state = Environment.getExternalStorageState(); + Collections.addAll(files, checkExternalStorage(sdcard, state)); + } + } else { + for (File f : getExternalFilesDirs(null)) { + Log.i(TAG, "getExternalFilesDirs " + f); + if (f == null || !f.isDirectory()) { + continue; + } + Log.i(TAG, "getExternalFilesDirs " + f); + if (Environment.isExternalStorageRemovable(f)) { + String state = Environment.getExternalStorageState(f); + if (ContextCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE) + == PackageManager.PERMISSION_GRANTED) { + // remove Android/data/org.fdroid.fdroid/files to get root + File sdcard = f.getParentFile().getParentFile().getParentFile().getParentFile(); + Collections.addAll(files, checkExternalStorage(sdcard, state)); + } else { + Collections.addAll(files, checkExternalStorage(f, state)); + } + } + } + } + + Log.i(TAG, "sdcard files " + files.toString()); + ArrayList filesList = new ArrayList<>(); + for (File dir : files) { + if (!dir.isDirectory()) { + continue; + } + searchDirectory(dir); + } + } + + private File[] checkExternalStorage(File sdcard, String state) { + File[] files = null; + if (sdcard != null && + (Environment.MEDIA_MOUNTED_READ_ONLY.equals(state) || Environment.MEDIA_MOUNTED.equals(state))) { + files = sdcard.listFiles(); + } + + if (files == null) { + Utils.debugLog(TAG, "checkExternalStorage returned blank, F-Droid probaby doesn't have Storage perm!"); + return new File[0]; + } else { + return files; + } + } + + private void searchDirectory(File dir) { + if (SKIP_DIRS.contains(dir.getName())) { + return; + } + File[] files = dir.listFiles(); + if (files == null) { + return; + } + for (File file : files) { + if (file.isDirectory()) { + searchDirectory(file); + } else { + if (IndexV1Updater.SIGNED_FILE_NAME.equals(file.getName())) { + registerRepo(file); + } + } + } + } + + private void registerRepo(File file) { + InputStream inputStream = null; + try { + inputStream = new FileInputStream(file); + TreeUriScannerIntentService.registerRepo(this, inputStream, Uri.fromFile(file.getParentFile())); + } catch (IOException | IndexUpdater.SigningException e) { + e.printStackTrace(); + } finally { + Utils.closeQuietly(inputStream); + } + } + +} \ No newline at end of file diff --git a/app/src/full/java/org/fdroid/fdroid/views/main/NearbyViewBinder.java b/app/src/full/java/org/fdroid/fdroid/views/main/NearbyViewBinder.java index 8840b915c..be8b0e7b6 100644 --- a/app/src/full/java/org/fdroid/fdroid/views/main/NearbyViewBinder.java +++ b/app/src/full/java/org/fdroid/fdroid/views/main/NearbyViewBinder.java @@ -17,6 +17,7 @@ import android.widget.ImageView; import android.widget.TextView; import android.widget.Toast; import org.fdroid.fdroid.R; +import org.fdroid.fdroid.localrepo.SDCardScannerService; import org.fdroid.fdroid.localrepo.TreeUriScannerIntentService; import org.fdroid.fdroid.views.swap.SwapWorkflowActivity; @@ -47,6 +48,7 @@ import java.io.File; * write access to the the removable storage. * * @see TreeUriScannerIntentService + * @see org.fdroid.fdroid.localrepo.SDCardScannerService */ class NearbyViewBinder { public static final String TAG = "NearbyViewBinder"; @@ -114,7 +116,7 @@ class NearbyViewBinder { ActivityCompat.requestPermissions(activity, new String[]{writeExternalStorage}, MainActivity.REQUEST_STORAGE_PERMISSIONS); } else { - // TODO do something + SDCardScannerService.scan(activity); } } }); diff --git a/app/src/main/java/org/fdroid/fdroid/FDroidApp.java b/app/src/main/java/org/fdroid/fdroid/FDroidApp.java index 3ea9f5990..9c8567ffa 100644 --- a/app/src/main/java/org/fdroid/fdroid/FDroidApp.java +++ b/app/src/main/java/org/fdroid/fdroid/FDroidApp.java @@ -69,6 +69,7 @@ import org.fdroid.fdroid.data.Repo; import org.fdroid.fdroid.data.RepoProvider; import org.fdroid.fdroid.installer.ApkFileProvider; import org.fdroid.fdroid.installer.InstallHistoryService; +import org.fdroid.fdroid.localrepo.SDCardScannerService; import org.fdroid.fdroid.net.ConnectivityMonitorService; import org.fdroid.fdroid.net.HttpDownloader; import org.fdroid.fdroid.net.ImageLoaderForUIL; @@ -502,6 +503,8 @@ public class FDroidApp extends Application { } else { atStartTime.edit().remove(queryStringKey).apply(); } + + SDCardScannerService.scan(this); } /** diff --git a/app/src/main/java/org/fdroid/fdroid/data/NewRepoConfig.java b/app/src/main/java/org/fdroid/fdroid/data/NewRepoConfig.java index f7efb42ba..c4411321b 100644 --- a/app/src/main/java/org/fdroid/fdroid/data/NewRepoConfig.java +++ b/app/src/main/java/org/fdroid/fdroid/data/NewRepoConfig.java @@ -4,6 +4,7 @@ import android.content.Context; import android.content.Intent; import android.net.Uri; import android.text.TextUtils; +import android.util.Log; import org.fdroid.fdroid.R; import org.fdroid.fdroid.Utils; import org.fdroid.fdroid.localrepo.peers.WifiPeer; @@ -53,8 +54,9 @@ public class NewRepoConfig { String scheme = uri.getScheme(); host = uri.getHost(); port = uri.getPort(); - if (TextUtils.isEmpty(scheme) || TextUtils.isEmpty(host)) { + if (TextUtils.isEmpty(scheme) || (TextUtils.isEmpty(host) && !"file".equals(scheme))) { errorMessage = String.format(context.getString(R.string.malformed_repo_uri), uri); + Log.i(TAG, errorMessage); isValidRepo = false; return; } @@ -82,7 +84,7 @@ public class NewRepoConfig { host = host.toLowerCase(Locale.ENGLISH); if (uri.getPath() == null - || !Arrays.asList("https", "http", "fdroidrepos", "fdroidrepo", "content").contains(scheme)) { + || !Arrays.asList("https", "http", "fdroidrepos", "fdroidrepo", "content", "file").contains(scheme)) { isValidRepo = false; return; } diff --git a/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java b/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java index 9fcd1a2ea..2a13e947a 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java +++ b/app/src/main/java/org/fdroid/fdroid/net/DownloaderFactory.java @@ -33,6 +33,8 @@ public class DownloaderFactory { downloader = new BluetoothDownloader(uri, destFile); } else if ("content".equals(scheme)) { downloader = new TreeUriDownloader(uri, destFile); + } else if ("file".equals(scheme)) { + downloader = new LocalFileDownloader(uri, destFile); } else { final String[] projection = {Schema.RepoTable.Cols.USERNAME, Schema.RepoTable.Cols.PASSWORD}; Repo repo = RepoProvider.Helper.findByUrl(context, uri, projection); diff --git a/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java b/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java new file mode 100644 index 000000000..9701eae39 --- /dev/null +++ b/app/src/main/java/org/fdroid/fdroid/net/LocalFileDownloader.java @@ -0,0 +1,86 @@ +package org.fdroid.fdroid.net; + +import android.net.Uri; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.ConnectException; +import java.net.ProtocolException; + +/** + * "Downloads" files from {@code file:///} {@link Uri}s. Even though it is + * obviously unnecessary to download a file that is locally available, this + * class is here so that the whole security-sensitive installation process is + * the same, no matter where the files are downloaded from. Also, for things + * like icons and graphics, it makes sense to have them copied to the cache so + * that they are available even after removable storage is no longer present. + */ +public class LocalFileDownloader extends Downloader { + + private InputStream inputStream; + private final File sourceFile; + + LocalFileDownloader(Uri uri, File destFile) { + super(uri, destFile); + sourceFile = new File(uri.getPath()); + } + + /** + * This needs to convert {@link FileNotFoundException} + * and {@link SecurityException} to {@link ProtocolException} since the + * mirror failover logic expects network errors, not filesystem or other + * errors. In the downloading logic, filesystem errors are related to the + * file as it is being downloaded and written to disk. Things can fail + * here if the SDCard is not longer mounted, the files were deleted by + * some other process, etc. + */ + @Override + protected InputStream getDownloadersInputStream() throws IOException { + try { + inputStream = new FileInputStream(sourceFile); + return inputStream; + } catch (FileNotFoundException | SecurityException e) { + throw new ProtocolException(e.getLocalizedMessage()); + } + } + + @Override + protected void close() { + IOUtils.closeQuietly(inputStream); + } + + @Override + public boolean hasChanged() { + return true; + } + + @Override + protected long totalDownloadSize() { + return sourceFile.length(); + } + + @Override + public void download() throws ConnectException, IOException, InterruptedException { + if (!sourceFile.exists()) { + notFound = true; + throw new ConnectException(sourceFile + " does not exist, try a mirror"); + } + + boolean resumable = false; + long contentLength = sourceFile.length(); + long fileLength = outputFile.length(); + if (fileLength > contentLength) { + FileUtils.deleteQuietly(outputFile); + } else if (fileLength == contentLength && outputFile.isFile()) { + return; // already have it! + } else if (fileLength > 0) { + resumable = true; + } + downloadFromStream(8192, resumable); + } +} diff --git a/app/src/main/java/org/fdroid/fdroid/views/ManageReposActivity.java b/app/src/main/java/org/fdroid/fdroid/views/ManageReposActivity.java index 6949f3f7c..012fffd16 100644 --- a/app/src/main/java/org/fdroid/fdroid/views/ManageReposActivity.java +++ b/app/src/main/java/org/fdroid/fdroid/views/ManageReposActivity.java @@ -556,7 +556,8 @@ public class ManageReposActivity extends AppCompatActivity return originalAddress; } - if (originalAddress.startsWith(ContentResolver.SCHEME_CONTENT)) { + if (originalAddress.startsWith(ContentResolver.SCHEME_CONTENT) + || originalAddress.startsWith(ContentResolver.SCHEME_FILE)) { // TODO check whether there is read access return originalAddress; }