From f064e33de93426d2016311c464f963831b26b468 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 7 Jan 2019 16:02:33 +0100 Subject: [PATCH 1/7] disable all compression when downloading on < android-19 Compression seems to just give stacktraces: HttpDownloaderTest I URL: https://en.wikipedia.org/wiki/Index.html TestRunner I failed: downloadUninterruptedTest(org.fdroid.fdroid.net.HttpDownloaderTest) I ----- begin exception ----- I java.io.EOFException I at java.util.zip.GZIPInputStream.readFully(GZIPInputStream.java:206) I at java.util.zip.GZIPInputStream.(GZIPInputStream.java:98) I at java.util.zip.GZIPInputStream.(GZIPInputStream.java:81) I at libcore.net.http.HttpEngine.initContentStream(HttpEngine.java:541) I at libcore.net.http.HttpEngine.readResponse(HttpEngine.java:844) I at libcore.net.http.HttpURLConnectionImpl.getResponse(HttpURLConnectionImpl.java:283) I at libcore.net.http.HttpURLConnectionImpl.getHeaderField(HttpURLConnectionImpl.java:139) I at libcore.net.http.HttpsURLConnectionImpl.getHeaderField(HttpsURLConnectionImpl.java:246) I at org.fdroid.fdroid.net.HttpDownloader.download(HttpDownloader.java:111) I at org.fdroid.fdroid.net.HttpDownloaderTest.downloadUninterruptedTest(HttpDownloaderTest.java:74) I at java.lang.reflect.Method.invokeNative(Native Method) I at java.lang.reflect.Method.invoke(Method.java:511) I at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) I at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) I at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) I at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) I at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) I at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) I at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) I at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) I at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) I at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) I at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) I at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) I at org.junit.runners.ParentRunner.run(ParentRunner.java:363) I at org.junit.runners.Suite.runChild(Suite.java:128) I at org.junit.runners.Suite.runChild(Suite.java:27) I at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) I at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) I at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) I at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) I at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) I at org.junit.runners.ParentRunner.run(ParentRunner.java:363) I at org.junit.runner.JUnitCore.run(JUnitCore.java:137) I at org.junit.runner.JUnitCore.run(JUnitCore.java:115) I at android.support.test.internal.runner.TestExecutor.execute(TestExecutor.java:56) I at android.support.test.runner.AndroidJUnitRunner.onStart(AndroidJUnitRunner.java:384) I at android.app.Instrumentation$InstrumentationThread.run(Instrumentation.java:1661) --- app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java b/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java index eab3097f1..a6fe63dc2 100644 --- a/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java +++ b/app/src/main/java/org/fdroid/fdroid/net/HttpDownloader.java @@ -177,6 +177,10 @@ public class HttpDownloader extends Downloader { connection.setRequestProperty("User-Agent", "F-Droid " + BuildConfig.VERSION_NAME); connection.setConnectTimeout(getTimeout()); + if (Build.VERSION.SDK_INT < 19) { // gzip encoding can be troublesome on old Androids + connection.setRequestProperty("Accept-Encoding", "identity"); + } + if (username != null && password != null) { // add authorization header from username / password if set String authString = username + ":" + password; From 97ad4ddc1fd9fefb1fad86d9da246a64c2208c94 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 7 Jan 2019 16:31:32 +0100 Subject: [PATCH 2/7] get HttpDownloaderTest working on at least android-17 and above The two excluded URLs seem to always resolve to IPv6 addresses first, then fail since there isn't IPv6 connectivity. Donno why, but only on old android versions, so just skip them there. --- .../fdroid/fdroid/net/HttpDownloaderTest.java | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java b/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java index 47400116c..57238613d 100644 --- a/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java +++ b/app/src/androidTest/java/org/fdroid/fdroid/net/HttpDownloaderTest.java @@ -2,11 +2,15 @@ package org.fdroid.fdroid.net; import android.net.Uri; +import android.os.Build; +import android.util.Log; import org.fdroid.fdroid.ProgressListener; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -15,22 +19,37 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class HttpDownloaderTest { + private static final String TAG = "HttpDownloaderTest"; - final String[] urls = { - "https://en.wikipedia.org/wiki/Index.html", - "https://mirrors.kernel.org/debian/dists/stable/Release", - "https://f-droid.org/repo/index.jar", - // sites that use SNI for HTTPS - "https://guardianproject.info/fdroid/repo/index.jar", - //"https://microg.org/fdroid/repo/index.jar", - //"https://grobox.de/fdroid/repo/index.jar", - }; + static final String[] URLS; + + // https://developer.android.com/reference/javax/net/ssl/SSLContext + static { + ArrayList tempUrls = new ArrayList<>(Arrays.asList( + "https://f-droid.org/repo/index-v1.jar", + // sites that use SNI for HTTPS + "https://mirrors.kernel.org/debian/dists/stable/Release", + "https://fdroid.tetaneutral.net/fdroid/repo/index-v1.jar", + "https://ftp.fau.de/fdroid/repo/index-v1.jar", + //"https://microg.org/fdroid/repo/index-v1.jar", + //"https://grobox.de/fdroid/repo/index.jar", + "https://guardianproject.info/fdroid/repo/index-v1.jar" + )); + if (Build.VERSION.SDK_INT >= 22) { + tempUrls.addAll(Arrays.asList( + "https://en.wikipedia.org/wiki/Index.html", // no SNI but weird ipv6 lookup issues + "https://mirror.cyberbits.eu/fdroid/repo/index-v1.jar" // TLSv1.2 only and SNI + )); + } + URLS = tempUrls.toArray(new String[tempUrls.size()]); + } private boolean receivedProgress; @Test public void downloadUninterruptedTest() throws IOException, InterruptedException { - for (String urlString : urls) { + for (String urlString : URLS) { + Log.i(TAG, "URL: " + urlString); Uri uri = Uri.parse(urlString); File destFile = File.createTempFile("dl-", ""); HttpDownloader httpDownloader = new HttpDownloader(uri, destFile); From 77b0bdf453236296b45b6e7dab39bd786ead5192 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 7 Jan 2019 11:42:03 +0100 Subject: [PATCH 3/7] gitlab-ci: force HTTP with sdkmanager to make for better caching Caching proxies are a lot easier with plain HTTP. HTTPS is not needed here since these are only CI builds. The worst an attacker could do it annoy us --- .gitlab-ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61c03de38..b2019d361 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: before_script: - export GRADLE_USER_HOME=$PWD/.gradle - export ANDROID_COMPILE_SDK=`sed -n 's,.*compileSdkVersion\s*\([0-9][0-9]*\).*,\1,p' app/build.gradle` + - alias sdkmanager="sdkmanager --no_https" - echo y | sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" > /dev/null .test-template: &test-template @@ -59,7 +60,13 @@ errorprone: .connected-template: &connected-template script: - ./gradlew assembleDebug - - echo y | sdkmanager "platforms;android-$AVD_SDK" > /dev/null + - export AVD_SDK=`echo $CI_JOB_NAME | awk '{print $2}'` + - export AVD_TAG=`echo $CI_JOB_NAME | awk '{print $3}'` + - export AVD_ARCH=`echo $CI_JOB_NAME | awk '{print $4}'` + - export AVD_PACKAGE="system-images;android-${AVD_SDK};${AVD_TAG};${AVD_ARCH}" + - echo $AVD_PACKAGE + - alias sdkmanager + - echo y | sdkmanager "emulator" "platforms;android-$AVD_SDK" > /dev/null - if ! avdmanager list avd | grep "Name. avd$AVD_SDK$"; then rm -rf ~/.android/avd $ANDROID_HOME/system-images; echo y | sdkmanager "$AVD_PACKAGE" > /dev/null; From 11ff830d3d23bbc4bfcaee6e808e89f0026a7b7e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 7 Jan 2019 23:44:28 +0100 Subject: [PATCH 4/7] gitlab-ci: first jobs for KVM x86 emulators * seems that android-24-x86 emulator don't work without google_apis * -wipe-data is required to use google_apis_playstore, otherwise adb fails: https://github.com/butomo1989/docker-android/issues/87 --- .gitlab-ci.yml | 84 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2019d361..0dcdfd9bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,15 +59,17 @@ errorprone: # connectedCheck to test all the build flavors .connected-template: &connected-template script: - - ./gradlew assembleDebug + - ./gradlew assembleFullDebug - export AVD_SDK=`echo $CI_JOB_NAME | awk '{print $2}'` - export AVD_TAG=`echo $CI_JOB_NAME | awk '{print $3}'` - export AVD_ARCH=`echo $CI_JOB_NAME | awk '{print $4}'` - export AVD_PACKAGE="system-images;android-${AVD_SDK};${AVD_TAG};${AVD_ARCH}" - echo $AVD_PACKAGE + - emulator -accel-check || true - alias sdkmanager - echo y | sdkmanager "emulator" "platforms;android-$AVD_SDK" > /dev/null - if ! avdmanager list avd | grep "Name. avd$AVD_SDK$"; then + set -x; rm -rf ~/.android/avd $ANDROID_HOME/system-images; echo y | sdkmanager "$AVD_PACKAGE" > /dev/null; echo no | avdmanager create avd --name avd$AVD_SDK --tag "$AVD_TAG" --package "$AVD_PACKAGE" --sdcard 64M --device "Nexus 5"; @@ -75,21 +77,89 @@ errorprone: sed -i '/^hw\.ramSize\s*=.*/d' ~/.android/avd/*.avd/config.ini; echo "hw.ramSize=$RAMSIZE" >> ~/.android/avd/*.avd/config.ini; avdmanager list avd; + set +x; fi - - emulator64-arm -avd avd$AVD_SDK -no-audio -no-window -no-snapstorage & + - adb start-server + - ls -l ~/.android + - emulator -avd avd$AVD_SDK + -no-audio + -no-jni + -no-snapstorage + -no-window + -skip-adb-auth + -verbose + -wipe-data + & - wait-for-emulator + - adb devices - adb shell input keyevent 82 & - ./gradlew connectedFullDebugAndroidTest || (adb -e logcat -d > logcat.txt; exit 1) -connected24: +connected 24 default armeabi-v7a: retry: 1 + only: + - merge_requests <<: *test-template - variables: - AVD_SDK: "24" - AVD_TAG: "default" - AVD_PACKAGE: "system-images;android-${AVD_SDK};${AVD_TAG};armeabi-v7a" <<: *connected-template +.kvm-template: &kvm-template + tags: + - fdroid + - kvm + only: + - merge_requests + - branches@fdroid/fdroidclient + <<: *test-template + <<: *connected-template + +connected 17 default x86: + <<: *kvm-template + +connected 18 default x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 19 default x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 21 default x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 22 default x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 23 default x86: + <<: *kvm-template + +connected 24 google_apis x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 25 default x86: + <<: *kvm-template + +connected 26 google_apis x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 27 google_apis_playstore x86: + <<: *kvm-template + only: + - master@fdroid/fdroidclient + +connected 28 default x86_64: + <<: *kvm-template + + deploy_nightly: stage: deploy only: From 22773f3de3fafed0f9273dc300f2fad974b69cf7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 8 Jan 2019 13:29:32 +0100 Subject: [PATCH 5/7] gitlab-ci: force skip Espresso tests on < android-25 It seems that ARM emulators timeout even when just trying to run the assumeTrue() tests via Espresso. There needs to be one test still enabled in the file, otherwise, the run fails with: org.fdroid.fdroid.MainActivityEspressoTest > initializationError[Nexus_One_API_19(AVD) - 4.4.2] FAILED java.lang.Exception: No runnable methods at org.junit.runners.BlockJUnit4ClassRunner.validateInstanceMethods(BlockJUnit4ClassRunner.java:191) --- .gitlab-ci.yml | 3 ++- .../org/fdroid/fdroid/MainActivityEspressoTest.java | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0dcdfd9bf..423974155 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,8 @@ errorprone: - wait-for-emulator - adb devices - adb shell input keyevent 82 & - - ./gradlew connectedFullDebugAndroidTest || (adb -e logcat -d > logcat.txt; exit 1) + - test $AVD_SDK -ge 25 || export FLAG=-Pandroid.testInstrumentationRunnerArguments.notAnnotation=android.test.suitebuilder.annotation.LargeTest + - ./gradlew connectedFullDebugAndroidTest $FLAG || (adb -e logcat -d > logcat.txt; exit 1) connected 24 default armeabi-v7a: retry: 1 diff --git a/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java b/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java index 984e2869a..25d20d1af 100644 --- a/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java +++ b/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java @@ -48,7 +48,6 @@ import static org.junit.Assert.assertTrue; @Ignore @RunWith(AndroidJUnit4.class) -@LargeTest public class MainActivityEspressoTest { public static final String TAG = "MainActivityEspressoTest"; @@ -151,7 +150,7 @@ public class MainActivityEspressoTest { } } - @Test + @LargeTest public void showSettings() { ViewInteraction settingsBottonNavButton = onView( allOf(withText(R.string.menu_settings), isDisplayed())); @@ -168,14 +167,14 @@ public class MainActivityEspressoTest { onView(withText(R.string.installed_apps__activity_title)).check(matches(isDisplayed())); } - @Test + @LargeTest public void showUpdates() { ViewInteraction updatesBottonNavButton = onView(allOf(withText(R.string.main_menu__updates), isDisplayed())); updatesBottonNavButton.perform(click()); onView(withText(R.string.main_menu__updates)).check(matches(isDisplayed())); } - @Test + @LargeTest public void startSwap() { if (!BuildConfig.FLAVOR.startsWith("full")) { return; @@ -189,7 +188,7 @@ public class MainActivityEspressoTest { onView(withText(R.string.swap_send_fdroid)).check(matches(isDisplayed())); } - @Test + @LargeTest public void showCategories() { if (!BuildConfig.FLAVOR.startsWith("full")) { return; @@ -214,7 +213,7 @@ public class MainActivityEspressoTest { .perform(click()); } - @Test + @LargeTest public void showLatest() { if (!BuildConfig.FLAVOR.startsWith("full")) { return; @@ -236,7 +235,7 @@ public class MainActivityEspressoTest { .perform(click()); } - @Test + @LargeTest public void showSearch() { onView(allOf(withText(R.string.menu_settings), isDisplayed())).perform(click()); onView(withId(R.id.fab_search)).check(doesNotExist()); From 3c185d6d64f82a59679f91fba20712c3c24f22e3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 7 Jan 2019 21:12:53 +0100 Subject: [PATCH 6/7] enable Espresso tests to auto run, now there are x86 emulators --- .../fdroid/MainActivityEspressoTest.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java b/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java index 25d20d1af..307297e8e 100644 --- a/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java +++ b/app/src/androidTest/java/org/fdroid/fdroid/MainActivityEspressoTest.java @@ -21,8 +21,8 @@ import org.fdroid.fdroid.views.main.MainActivity; import org.hamcrest.Matchers; import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,29 +45,36 @@ import static android.support.test.espresso.matcher.ViewMatchers.withText; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; -@Ignore @RunWith(AndroidJUnit4.class) public class MainActivityEspressoTest { public static final String TAG = "MainActivityEspressoTest"; /** + * Emulators older than {@code android-25} seem to fail at running Espresso tests. + *

* ARM emulators are too slow to run these tests in a useful way. The sad * thing is that it would probably work if Android didn't put up the ANR * "Process system isn't responding" on boot each time. There seems to be no * way to increase the ANR timeout. */ + private static boolean canRunEspresso() { + if (Build.VERSION.SDK_INT < 25 + || (Build.SUPPORTED_ABIS[0].startsWith("arm") && isEmulator())) { + Log.e(TAG, "SKIPPING TEST: ARM emulators are too slow to run these tests in a useful way"); + return false; + } + return true; + } + @BeforeClass public static void classSetUp() { - Log.i(TAG, "setUp " + isEmulator() + " " + Build.SUPPORTED_ABIS[0]); - if (Build.SUPPORTED_ABIS[0].startsWith("arm") && isEmulator()) { - Log.e(TAG, "SKIPPING TEST: ARM emulators are too slow to run these tests in a useful way"); - org.junit.Assume.assumeTrue(false); - return; - } - IdlingPolicies.setIdlingResourceTimeout(10, TimeUnit.MINUTES); IdlingPolicies.setMasterPolicyTimeout(10, TimeUnit.MINUTES); + if (!canRunEspresso()) { + return; + } Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); try { UiDevice.getInstance(instrumentation) @@ -105,6 +112,11 @@ public class MainActivityEspressoTest { || "google_sdk".equals(Build.PRODUCT); } + @Before + public void setUp() { + assumeTrue(canRunEspresso()); + } + /** * Placate {@link android.os.StrictMode} * From e690fbb061d24cbbb1ba062ee46a6e8d1d8cb9d4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 8 Jan 2019 20:39:56 +0100 Subject: [PATCH 7/7] gitlab-ci: switch ARM emulator to android-22, it seems more reliable https://medium.com/zendesk-engineering/speeding-up-android-builds-on-travis-ci-1bb4cdbd9c62 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 423974155..ea070f8da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,7 +96,7 @@ errorprone: - test $AVD_SDK -ge 25 || export FLAG=-Pandroid.testInstrumentationRunnerArguments.notAnnotation=android.test.suitebuilder.annotation.LargeTest - ./gradlew connectedFullDebugAndroidTest $FLAG || (adb -e logcat -d > logcat.txt; exit 1) -connected 24 default armeabi-v7a: +connected 22 default armeabi-v7a: retry: 1 only: - merge_requests