From 41b2e175c99d7d530ff4bef1fac3ed252a78cecb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 5 Apr 2016 00:16:00 +0200 Subject: [PATCH 1/7] improve build server performance by allowing disabling of pre-dexing It seems that Google is finally paying some attention to CI builds with the emulator, they issued a recommendation: http://tools.android.com/tech-docs/new-build-system/tips#TOC-Improving-Build-Server-performance --- .gitlab-ci.yml | 8 ++++---- build.gradle | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b3e35ba4..5ec3742ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ gradle: # always report on lint errors to the build log - sed -i -e 's,textReport .*,textReport true,' app/build.gradle # 'build' means assemble and check - - ./gradlew build || { + - ./gradlew build -PdisablePreDex || { for log in app/build/reports/*ests/*/*ml; do echo "read $log here:" cat "$log" | curl --silent -F 'clbin=<-' https://clbin.com; @@ -40,7 +40,7 @@ gradle: - ./tools/wait-for-emulator - adb shell input keyevent 82 - export EXITVALUE=0 - - ADB_INSTALL_TIMEOUT=8 ./gradlew connectedCheck || { + - ADB_INSTALL_TIMEOUT=8 ./gradlew connectedCheck -PdisablePreDex || { adb -e logcat -d '*:E'; echo "get the full logcat here:"; adb -e logcat -d | curl --silent -F 'clbin=<-' https://clbin.com; @@ -57,12 +57,12 @@ gradle: pmd: script: - export GRADLE_USER_HOME=$PWD/.gradle - - ./gradlew pmd + - ./gradlew pmd -PdisablePreDex checkstyle: script: - export GRADLE_USER_HOME=$PWD/.gradle - - ./gradlew checkstyle + - ./gradlew checkstyle -PdisablePreDex tools: script: diff --git a/build.gradle b/build.gradle index e02ad215c..2b3560d32 100644 --- a/build.gradle +++ b/build.gradle @@ -7,3 +7,19 @@ buildscript { classpath files('libs/gradle-witness.jar') } } + +/** + * Improve build server performance by allowing disabling of pre-dexing + * (see http://tools.android.com/tech-docs/new-build-system/tips#TOC-Improving-Build-Server-performance.) + */ +project.ext.preDexLibs = !project.hasProperty('disablePreDex') + +subprojects { + project.plugins.whenPluginAdded { plugin -> + if ("com.android.build.gradle.AppPlugin".equals(plugin.class.name)) { + project.android.dexOptions.preDexLibraries = rootProject.ext.preDexLibs + } else if ("com.android.build.gradle.LibraryPlugin".equals(plugin.class.name)) { + project.android.dexOptions.preDexLibraries = rootProject.ext.preDexLibs + } + } +} From a4161aeb73e4f1564ff040828cc542e5397ab594 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 5 Apr 2016 00:21:45 +0200 Subject: [PATCH 2/7] gitlab-ci: run `adb shell input keyevent 82` in the background This is the common pattern I've seen in travis-ci builds. It should speed things up a little bit since the adb connection process will happen in parallel with waiting for the screen lock to be dismissed. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ec3742ae..97a8f8514 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ gradle: --target android-$AVD_SDK - emulator64-arm -avd fcl-test -no-skin -no-audio -no-window & - ./tools/wait-for-emulator - - adb shell input keyevent 82 + - adb shell input keyevent 82 & - export EXITVALUE=0 - ADB_INSTALL_TIMEOUT=8 ./gradlew connectedCheck -PdisablePreDex || { adb -e logcat -d '*:E'; From d8c87c3d4b827aec1153aee09f29eb805f84031a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 5 Apr 2016 00:24:24 +0200 Subject: [PATCH 3/7] gitlab-ci: switch glibc to a memory conserving mode --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97a8f8514..a1f3f7956 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,8 @@ cache: variables: AVD_SDK: "17" SKIN: "QVGA" + # switch glibc to a memory conserving mode + MALLOC_ARENA_MAX: "2" gradle: script: From 3112ba75c99950eae457d002e0f0d0fe94502ed5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 11 Apr 2016 16:06:01 -0400 Subject: [PATCH 4/7] gitlab-ci: improve caching of gradle stuff --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1f3f7956..7c8817838 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,8 @@ gradle: cat "$log" | curl --silent -F 'clbin=<-' https://clbin.com; done - sed -n 's/.*"ctr2">\([0-9]*\)%<.*/Coverage - \1.0% covered\n/p' app/build/reports/coverage/debug/index.html + # this file changes every time but should not be cached + - rm -f $GRADLE_USER_HOME/caches/modules-2/modules-2.lock - exit $EXITVALUE pmd: From 00c6db81a7dc73d0742033592b48574ca415f9dc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 May 2016 11:59:55 +0200 Subject: [PATCH 5/7] gitlab-ci: add missing semi-colon in failure script It was echoing 'cat "$log" instead of cat'ing the log. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c8817838..cbc6dc61c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ gradle: # 'build' means assemble and check - ./gradlew build -PdisablePreDex || { for log in app/build/reports/*ests/*/*ml; do - echo "read $log here:" + echo "read $log here:"; cat "$log" | curl --silent -F 'clbin=<-' https://clbin.com; done; exit 1; From d6ed2a5e8ae37644f0c7517009ad5559b31ab28b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 4 May 2016 14:34:57 +0200 Subject: [PATCH 6/7] simplify downloadUninterruptedTests to improve reliability Test downloads with actual files, not dynamically generated things. Testing with the progress reports is really hard with multiple URLs, so just test progress with a single URL for now, and multiple URLs can still be tested without the progress check. fixes #650 https://gitlab.com/fdroid/fdroidclient/issues/650 --- .../fdroid/fdroid/net/HttpDownloaderTest.java | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/app/src/test/java/org/fdroid/fdroid/net/HttpDownloaderTest.java b/app/src/test/java/org/fdroid/fdroid/net/HttpDownloaderTest.java index 9d92fa74e..7b56bee6b 100644 --- a/app/src/test/java/org/fdroid/fdroid/net/HttpDownloaderTest.java +++ b/app/src/test/java/org/fdroid/fdroid/net/HttpDownloaderTest.java @@ -16,13 +16,11 @@ import static org.junit.Assert.fail; public class HttpDownloaderTest { String[] urls = { - "https://www.google.com", "https://en.wikipedia.org/wiki/Index.html", "https://mirrors.kernel.org/debian/dists/stable/Release", - "https://f-droid.org/archive/de.we.acaldav_5.apk", + "https://f-droid.org/repo/index.jar", // sites that use SNI for HTTPS "https://guardianproject.info/fdroid/repo/index.jar", - "https://firstlook.org", }; private boolean receivedProgress; @@ -42,23 +40,38 @@ public class HttpDownloaderTest { @Test public void downloadUninterruptedTestWithProgress() throws IOException, InterruptedException { - for (String urlString : urls) { - receivedProgress = false; - URL url = new URL(urlString); - File destFile = File.createTempFile("dl-", ""); - HttpDownloader httpDownloader = new HttpDownloader(url, destFile); - httpDownloader.setListener(new Downloader.DownloaderProgressListener() { - @Override - public void sendProgress(URL sourceUrl, int bytesRead, int totalBytes) { - receivedProgress = true; + final CountDownLatch latch = new CountDownLatch(1); + String urlString = "https://f-droid.org/repo/index.jar"; + receivedProgress = false; + System.out.println("downloadUninterruptedTestWithProgress: " + urlString); + receivedProgress = false; + URL url = new URL(urlString); + File destFile = File.createTempFile("dl-", ""); + final HttpDownloader httpDownloader = new HttpDownloader(url, destFile); + httpDownloader.setListener(new Downloader.DownloaderProgressListener() { + @Override + public void sendProgress(URL sourceUrl, int bytesRead, int totalBytes) { + System.out.println("DownloaderProgressListener.sendProgress " + sourceUrl + " " + bytesRead + " / " + totalBytes); + receivedProgress = true; + } + }); + new Thread() { + @Override + public void run() { + try { + httpDownloader.download(); + latch.countDown(); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + fail(); } - }); - httpDownloader.download(); - assertTrue(destFile.exists()); - assertTrue(destFile.canRead()); - assertTrue(receivedProgress); - destFile.deleteOnExit(); - } + } + }.start(); + latch.await(100, TimeUnit.SECONDS); // either 2 progress reports or 100 seconds + assertTrue(destFile.exists()); + assertTrue(destFile.canRead()); + assertTrue(receivedProgress); + destFile.deleteOnExit(); } @Test From cff807e1915e4bcd92b3cb7d272ec524cd66f974 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 10 May 2016 14:34:15 +0200 Subject: [PATCH 7/7] include useful output logs for failing tests This helps when tests fail on the gitlab-ci builds. --- app/build.gradle | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 5103af08b..e5967b14b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -157,8 +157,17 @@ android { } testOptions { - // prevent tests from dying on android.util.Log calls - unitTests.returnDefaultValues = true + unitTests { + // prevent tests from dying on android.util.Log calls + returnDefaultValues = true + all { + // All the usual Gradle options. + testLogging { + events "skipped", "failed", "standardOut", "standardError" + showStandardStreams = true + } + } + } } lintOptions {