Merge branch 'repoupdater-simplify-and-streams' into 'master'

simplify RepoUpdater and use more streams

This is an overhaul of `RepoUpdater` to make its code match the architecture that is in use now: only download and use a signed index.jar.  It also streams index.xml directly out of the index.jar and directly into the XML parser.  That makes the update process quicker and more reliable because it no longer has to write out an index.xml to the filesystem, then read it in.  Ultimately I hope to stream the index.jar download directly to the XML parser, so not even the index.jar needs to be written to disk.  You can see that work in my git repo under the branches SKETCH-JarURLConnection and SKETCH-verify-with-JarInputStream for two different approaches.

This also changes the index parsing process to be based on bytes for now.  The progress is based on the stream now, and this will still work once the full streaming mode is implemented.  It also simplifies `RepoXMPHandler`.

This includes tests for index.jar signature verification. The tests all pass on my machine and our Jenkins.

See merge request !101
This commit is contained in:
Peter Serwylo 2015-07-13 20:50:50 +00:00
commit a653e6392c
29 changed files with 248 additions and 361 deletions

View File

@ -98,9 +98,6 @@ La voleu actualitzar?</string>
<string name="category_recentlyupdated">S\'ha actualitzat fa poc</string>
<string name="status_download">S\'està baixant
%2$s / %3$s (%4$d%%) des de
%1$s</string>
<string name="status_processing_xml">S\'està processant l\'aplicació
%2$d de %3$d des de
%1$s</string>
<string name="status_connecting_to_repo">S\'està connectant a
%1$s</string>

View File

@ -171,9 +171,6 @@ Sollen diese aktualisiert werden?</string>
<string name="proxy_port_summary">Konfigurieren Sie Ihren Proxy-Port (z.B. 8818)</string>
<string name="status_download">Herunterladen
%2$s / %3$s (%4$d%%) von
%1$s</string>
<string name="status_processing_xml">Anwendung wird vorbereitet
%2$d / %3$d von
%1$s</string>
<string name="status_connecting_to_repo">Mit %1$s
wird verbunden</string>

View File

@ -123,9 +123,6 @@
<string name="local_repos_scanning">Αναζήτηση για τοπικά αποθετήρια FDroid…</string>
<string name="status_download">Λήψη
%2$s / %3$s (%4$d%%) από
%1$s</string>
<string name="status_processing_xml">Επεξεργασία εφαρμογής
%2$d από %3$d από
%1$s</string>
<string name="status_connecting_to_repo">Σύνδεση με
%1$s</string>

View File

@ -170,9 +170,6 @@ La dirección de un repositorio es algo similar a esto: https://f-droid.org/repo
<string name="proxy_port_summary">Configurar el número del puerto del proxy (p.ej. 8118)</string>
<string name="status_download">Descargando
%2$s / %3$s (%4$d%%) de
%1$s</string>
<string name="status_processing_xml">Procesando la aplicación
%2$d de %3$d desde
%1$s</string>
<string name="status_connecting_to_repo">Conectando a
%1$s</string>

View File

@ -82,9 +82,6 @@
<string name="category_all">همه</string>
<string name="status_download">دریافت
%2$s / %3$s (%4$d%%) از
%1$s</string>
<string name="status_processing_xml">پردازش برنامه
%2$d از %3$d از
%1$s</string>
<string name="status_connecting_to_repo">اتصال به
%1$s</string>

View File

@ -167,9 +167,6 @@ Voulez-vous les mettre à jour?</string>
<string name="proxy_port_summary">Configurer votre numéro de port de proxy (ex. 8118)</string>
<string name="status_download">Téléchargement
%2$s / %3$s (%4$d%%) de
%1$s</string>
<string name="status_processing_xml">Prise en compte de l\'application
%2$d de %3$d depuis
%1$s</string>
<string name="status_connecting_to_repo">Connexion à
%1$s</string>

View File

@ -95,9 +95,6 @@ Quere actualizalos?</string>
<string name="category_recentlyupdated">Actualizado recentemente</string>
<string name="status_download">Descargando
%2$s / %3$s (%4$d%%) desde
%1$s</string>
<string name="status_processing_xml">Procesando o aplicativo
%2$d de %3$d desde
%1$s</string>
<string name="status_connecting_to_repo">Conectándose con
%1$s</string>

View File

@ -113,9 +113,6 @@ Szeretné ezeket frissíteni?\"</string>
<string name="category_recentlyupdated">Legutóbb frissítve</string>
<string name="status_download">\"Letöltés
%2$s / %3$s (%4$d%%) innen
%1$s\"</string>
<string name="status_processing_xml">\"Feldolgozási kérelem
%2$d of %3$d innen
%1$s\"</string>
<string name="status_connecting_to_repo">\"Kapcsolódás
%1$s\"</string>

View File

@ -170,9 +170,6 @@ Vuoi aggiornarlo?</string>
<string name="proxy_port_summary">Configura la porta del tuo proxy (es. 8118)</string>
<string name="status_download">Scaricamento
%2$s / %3$s (%4$d%%) da
%1$s</string>
<string name="status_processing_xml">Elaborazione applicazione
%2$d di %3$d da
%1$s</string>
<string name="status_connecting_to_repo">Connessione a
%1$s</string>

View File

@ -172,9 +172,6 @@ GNU GPLv3 ライセンスに基づいてリリースされました.</string>
<string name="proxy_port_summary">プロキシーのポート番号を設定(例:8118)</string>
<string name="status_download">ダウンロード中
%2$s / %3$s (%4$d%%) ダウンロード元
%1$s</string>
<string name="status_processing_xml">アプリケーションの処理中
%2$d / %3$d
%1$s</string>
<string name="status_connecting_to_repo">接続中
%1$s</string>

View File

@ -78,9 +78,6 @@
<string name="category_recentlyupdated">최근 업데이트</string>
<string name="status_download">%1$s 에서 다운로드 중입니다.
%2$s / %3$s (%4$d%%)</string>
<string name="status_processing_xml">응용 프로그램 처리중
%1$s
%2$d / %3$d</string>
<string name="status_connecting_to_repo">%1$s에 접속중</string>
<string name="status_checking_compatibility">장치와 응용프로그램의 호환성 확인중…</string>
<string name="no_permissions">사용된 권한이 없습니다.</string>

View File

@ -165,14 +165,8 @@ Lisensiert GNU GPLv3.</string>
<string name="proxy_host_summary">Sett opp tjenernavn for din mellomtjener (f.eks. 127.0.0.1)</string>
<string name="proxy_port">Mellomtjener-port</string>
<string name="proxy_port_summary">Sett opp portnummer for din mellomtjener (f.eks. 8118)</string>
<string name="status_download">Laster ned
%2$s / %3$s (%4$d%%) fra
%1$s</string>
<string name="status_processing_xml">Prosesserer applikasjon
%2$d of %3$d fra
%1$s</string>
<string name="status_connecting_to_repo">Kobler til
%1$s</string>
<string name="status_download">Laster ned\n%2$s / %3$s (%4$d%%) fra\n%1$s</string>
<string name="status_connecting_to_repo">Kobler til\n%1$s</string>
<string name="status_checking_compatibility">Sjekker programstøtte for ditt utstyr…</string>
<string name="status_inserting">Lagrer programdata (%1$d%%)</string>
<string name="repos_unchanged">Ingen av pakkebrønnene hadde noen oppdateringer</string>

View File

@ -124,9 +124,6 @@ Wilt u ze vernieuwen?</string>
<string name="local_repos_title">Lokale FDroid opslagplaatsen</string>
<string name="status_download">Downloaden
%2$s / %3$s (%4$d%%) van
%1$s</string>
<string name="status_processing_xml">Verwerken applicatie
%2$d van %3$d van
%1$s</string>
<string name="status_connecting_to_repo">Verbinden met %1$s</string>
<string name="status_checking_compatibility">Controleer app compatibiliteit met uw apparaat…</string>

View File

@ -129,7 +129,6 @@ Czy chcesz je zaktualizować?</string>
<string name="proxy_host_summary">Skonfiguruj host proxy</string>
<string name="proxy_port">Port proxy</string>
<string name="proxy_port_summary">Skonfiguruj port proxy</string>
<string name="status_processing_xml">Przetwarzanie aplikacji %2$d / %3$d z %1$s</string>
<string name="status_connecting_to_repo">Trwa łączenie z
%1$s</string>
<string name="status_checking_compatibility">Sprawdzanie kompatybilności aplikacji z urządzeniem…</string>

View File

@ -138,9 +138,6 @@ Você deseja atualizá-los?</string>
<string name="proxy_port_summary">Configurar o número da porta do seu proxy (ex. 8118)</string>
<string name="status_download">Baixando
%2$s / %3$s (%4$d%%) de
%1$s</string>
<string name="status_processing_xml">Processando aplicativo
%2$d de %3$d, de
%1$s</string>
<string name="status_connecting_to_repo">Conectando-se a
%1$s</string>

View File

@ -171,9 +171,6 @@
<string name="proxy_port_summary">Настройка номера порта вашего прокси (напр. 8118)</string>
<string name="status_download">Загрузка
%2$s / %3$s (%4$d%%) из
%1$s</string>
<string name="status_processing_xml">Обработка приложения
%2$d из %3$d от
%1$s</string>
<string name="status_connecting_to_repo">Соединение с
%1$s</string>

View File

@ -169,9 +169,6 @@
<string name="proxy_port_summary">Подесите порт вашег проксија (нпр. 8118)</string>
<string name="status_download">Преузимам
%2$s / %3$s (%4$d%%) са
%1$s</string>
<string name="status_processing_xml">Обрађујем апликацију
%2$d од %3$d са
%1$s</string>
<string name="status_connecting_to_repo">Повезујем се са
%1$s</string>

View File

@ -170,9 +170,6 @@ Vill du uppdatera dem?</string>
<string name="proxy_port_summary">Konfigurera din proxys portnummer (t.ex. 8118)</string>
<string name="status_download">Hämtar
%2$s / %3$s (%4$d%%) från
%1$s</string>
<string name="status_processing_xml">Bearbetar program
%2$d av %3$d från
%1$s</string>
<string name="status_connecting_to_repo">Ansluter till
%1$s</string>

View File

@ -153,9 +153,6 @@ Güncellemek ister misiniz?</string>
<string name="proxy_port_summary">Proxy port numarasını yapılandır (örn. 8118)</string>
<string name="status_download">İndiriliyor
%2$s / %3$s (%4$d%%) şuradan
%1$s</string>
<string name="status_processing_xml">Uygulama ele alınıyor
%2$d toplam %3$d şuradan
%1$s</string>
<string name="status_connecting_to_repo">%1$s konumuna
bağlanılıyor</string>

View File

@ -91,9 +91,6 @@
<string name="category_recentlyupdated">يېقىنقى يېڭىلانغانلار</string>
<string name="status_download">چۈشۈرۈۋاتىدۇ
%2$s / %3$s (%4$d%%)
%1$s</string>
<string name="status_processing_xml">ئەپنى بىر تەرەپ قىلىۋاتىدۇ
%2$d of %3$d
%1$s</string>
<string name="status_connecting_to_repo">%1$s غا
باغلىنىۋاتىدۇ</string>

View File

@ -140,9 +140,6 @@ https://f-droid.org/repo</string>
<string name="status_download">正在从以下位置下载:
%1$s
进度:%2$s / %3$s (%4$d%%)</string>
<string name="status_processing_xml">正在处理以下位置的应用程序:
%1$s
进度:%2$d / %3$d</string>
<string name="status_connecting_to_repo">正在连接到
%1$s</string>
<string name="status_checking_compatibility">正在检查应用程序与您的设备的兼容性…</string>

View File

@ -211,7 +211,7 @@
- Percentage complete (int between 0-100)
-->
<string name="status_download">Downloading\n%2$s / %3$s (%4$d%%) from\n%1$s</string>
<string name="status_processing_xml">Processing application\n%2$d of %3$d from\n%1$s</string>
<string name="status_processing_xml_percent">Processing %2$s / %3$s (%4$d%%) from %1$s</string>
<string name="status_connecting_to_repo">Connecting to\n%1$s</string>
<string name="status_checking_compatibility">Checking apps compatibility with your device…</string>
<string name="status_inserting">Saving application details (%1$d%%)</string>

View File

@ -0,0 +1,51 @@
package org.fdroid.fdroid;
import android.os.Bundle;
import org.fdroid.fdroid.data.Repo;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
public class ProgressBufferedInputStream extends BufferedInputStream {
private static final String TAG = "ProgressBufferedInputSt";
final Repo repo;
final ProgressListener progressListener;
final Bundle data;
final int totalBytes;
int currentBytes = 0;
/**
* Reports progress to the specified {@link ProgressListener}, with the
* progress based on the {@code totalBytes}.
*/
public ProgressBufferedInputStream(InputStream in, ProgressListener progressListener, Repo repo, int totalBytes)
throws IOException {
super(in);
this.progressListener = progressListener;
this.repo = repo;
this.data = new Bundle(1);
this.data.putString(RepoUpdater.PROGRESS_DATA_REPO_ADDRESS, repo.address);
this.totalBytes = totalBytes;
}
@Override
public int read(byte[] buffer, int byteOffset, int byteCount) throws IOException {
if (progressListener != null) {
currentBytes += byteCount;
/* don't send every change to keep things efficient. 333333 bytes to keep all
* the digits changing because it looks pretty, < 9000 since the reads won't
* line up exactly */
if (currentBytes % 333333 < 9000) {
progressListener.onProgress(
new ProgressListener.Event(
RepoUpdater.PROGRESS_TYPE_PROCESS_XML,
currentBytes, totalBytes, data));
}
}
return super.read(buffer, byteOffset, byteCount);
}
}

View File

@ -19,14 +19,12 @@ import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.CodeSigner;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@ -49,13 +47,18 @@ public class RepoUpdater {
private List<App> apps = new ArrayList<>();
private List<Apk> apks = new ArrayList<>();
private RepoUpdateRememberer rememberer = null;
protected boolean usePubkeyInJar = false;
protected boolean hasChanged = false;
@Nullable protected ProgressListener progressListener;
public RepoUpdater(@NonNull Context ctx, @NonNull Repo repo) {
this.context = ctx;
this.repo = repo;
/**
* Updates an app repo as read out of the database into a {@link Repo} instance.
*
* @param context
* @param repo a {@link Repo} read out of the local database
*/
public RepoUpdater(@NonNull Context context, @NonNull Repo repo) {
this.context = context;
this.repo = repo;
}
public void setProgressListener(ProgressListener progressListener) {
@ -68,39 +71,6 @@ public class RepoUpdater {
public List<Apk> getApks() { return apks; }
/**
* All repos are represented by a signed jar file, {@code index.jar}, which contains
* a single file, {@code index.xml}. This takes the {@code index.jar}, verifies the
* signature, then returns the unzipped {@code index.xml}.
*
* @throws UpdateException All error states will come from here.
*/
protected File getIndexFromFile(File downloadedFile) throws UpdateException {
final Date updateTime = new Date(System.currentTimeMillis());
Log.d(TAG, "Getting signed index from " + repo.address + " at " +
Utils.formatLogDate(updateTime));
final File indexJar = downloadedFile;
File indexXml = null;
// Don't worry about checking the status code for 200. If it was a
// successful download, then we will have a file ready to use:
if (indexJar != null && indexJar.exists()) {
// Due to a bug in android 5.0 lollipop, the inclusion of BouncyCastle causes
// breakage when verifying the signature of the downloaded .jar. For more
// details, check out https://gitlab.com/fdroid/fdroidclient/issues/111.
try {
FDroidApp.disableSpongyCastleOnLollipop();
indexXml = extractIndexFromJar(indexJar);
} finally {
FDroidApp.enableSpongyCastleOnLollipop();
}
}
return indexXml;
}
protected String getIndexAddress() {
try {
String versionName = context.getPackageManager().getPackageInfo(context.getPackageName(), 0).versionName;
@ -111,7 +81,7 @@ public class RepoUpdater {
return repo.address + "/index.jar";
}
protected Downloader downloadIndex() throws UpdateException {
Downloader downloadIndex() throws UpdateException {
Downloader downloader = null;
try {
downloader = DownloaderFactory.create(
@ -141,77 +111,83 @@ public class RepoUpdater {
return downloader;
}
private int estimateAppCount(File indexFile) {
int count = -1;
try {
// A bit of a hack, this might return false positives if an apps description
// or some other part of the XML file contains this, but it is a pretty good
// estimate and makes the progress counter more informative.
// As with asking the server about the size of the index before downloading,
// this also has a time tradeoff. It takes about three seconds to iterate
// through the file and count 600 apps on a slow emulator (v17), but if it is
// taking two minutes to update, the three second wait may be worth it.
final String APPLICATION = "<application";
count = Utils.countSubstringOccurrence(indexFile, APPLICATION);
} catch (IOException e) {
// Do nothing. Leave count at default -1 value.
}
return count;
}
/**
* All repos are represented by a signed jar file, {@code index.jar}, which contains
* a single file, {@code index.xml}. This takes the {@code index.jar}, verifies the
* signature, then returns the unzipped {@code index.xml}.
*
* @throws UpdateException All error states will come from here.
*/
public void update() throws UpdateException {
File downloadedFile = null;
File indexFile = null;
final Downloader downloader = downloadIndex();
hasChanged = downloader.hasChanged();
if (hasChanged) {
// Don't worry about checking the status code for 200. If it was a
// successful download, then we will have a file ready to use:
processDownloadedFile(downloader.getFile(), downloader.getCacheTag());
}
}
void processDownloadedFile(File downloadedFile, String cacheTag) throws UpdateException {
InputStream indexInputStream = null;
try {
if (downloadedFile == null || !downloadedFile.exists())
throw new UpdateException(repo, downloadedFile + " does not exist!");
final Downloader downloader = downloadIndex();
hasChanged = downloader.hasChanged();
// Due to a bug in Android 5.0 Lollipop, the inclusion of spongycastle causes
// breakage when verifying the signature of the downloaded .jar. For more
// details, check out https://gitlab.com/fdroid/fdroidclient/issues/111.
FDroidApp.disableSpongyCastleOnLollipop();
if (hasChanged) {
JarFile jarFile = new JarFile(downloadedFile, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry("index.xml");
indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry),
progressListener, repo, (int)indexEntry.getSize());
downloadedFile = downloader.getFile();
indexFile = getIndexFromFile(downloadedFile);
// Process the index...
final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
final XMLReader reader = parser.getXMLReader();
final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo);
reader.setContentHandler(repoXMLHandler);
reader.parse(new InputSource(indexInputStream));
// Process the index...
final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
final XMLReader reader = parser.getXMLReader();
final RepoXMLHandler handler = new RepoXMLHandler(repo, progressListener);
/* JarEntry can only read certificates after the file represented by that JarEntry
* has been read completely, so verification cannot run until now... */
X509Certificate certFromJar = getSigningCertFromJar(indexEntry);
if (progressListener != null) {
// Only bother spending the time to count the expected apps
// if we can show that to the user...
handler.setTotalAppCount(estimateAppCount(indexFile));
}
String certFromIndexXml = repoXMLHandler.getSigningCertFromIndexXml();
reader.setContentHandler(handler);
InputSource is = new InputSource(
new BufferedReader(new FileReader(indexFile)));
reader.parse(is);
apps = handler.getApps();
apks = handler.getApks();
rememberer = new RepoUpdateRememberer();
rememberer.context = context;
rememberer.repo = repo;
rememberer.values = prepareRepoDetailsForSaving(handler, downloader.getCacheTag());
// no signing cert read from database, this is the first use
if (repo.pubkey == null) {
verifyAndStoreTOFUCerts(certFromIndexXml, certFromJar);
}
verifyCerts(certFromIndexXml, certFromJar);
apps = repoXMLHandler.getApps();
apks = repoXMLHandler.getApks();
rememberer = new RepoUpdateRememberer();
rememberer.context = context;
rememberer.repo = repo;
rememberer.values = prepareRepoDetailsForSaving(repoXMLHandler, cacheTag);
} catch (SAXException | ParserConfigurationException | IOException e) {
throw new UpdateException(repo, "Error parsing index for repo " + repo.address, e);
} finally {
if (downloadedFile != null && downloadedFile != indexFile && downloadedFile.exists()) {
FDroidApp.enableSpongyCastleOnLollipop();
Utils.closeQuietly(indexInputStream);
if (downloadedFile != null) {
downloadedFile.delete();
}
if (indexFile != null && indexFile.exists()) {
indexFile.delete();
}
}
}
/**
* Update tracking data for the repo represented by this instance (index version, etag,
* description, human-readable name, etc.
*/
private ContentValues prepareRepoDetailsForSaving(RepoXMLHandler handler, String etag) {
ContentValues values = new ContentValues();
values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatDate(new Date(), ""));
@ -220,16 +196,6 @@ public class RepoUpdater {
values.put(RepoProvider.DataColumns.LAST_ETAG, etag);
}
/*
* We received a repo config that included the fingerprint, so we need to save
* the pubkey now.
*/
if (handler.getPubKey() != null && (repo.pubkey == null || usePubkeyInJar)) {
Log.d(TAG, "Public key found - saving in the database.");
values.put(RepoProvider.DataColumns.PUBLIC_KEY, handler.getPubKey());
usePubkeyInJar = false;
}
if (handler.getVersion() != -1 && handler.getVersion() != repo.version) {
Log.d(TAG, "Repo specified a new version: from "
+ repo.version + " to " + handler.getVersion());
@ -237,8 +203,7 @@ public class RepoUpdater {
}
if (handler.getMaxAge() != -1 && handler.getMaxAge() != repo.maxage) {
Log.d(TAG,
"Repo specified a new maximum age - updated");
Log.d(TAG, "Repo specified a new maximum age - updated");
values.put(RepoProvider.DataColumns.MAX_AGE, handler.getMaxAge());
}
@ -283,98 +248,96 @@ public class RepoUpdater {
}
}
private boolean verifyCerts(JarEntry item) throws UpdateException {
final Certificate[] certs = item.getCertificates();
if (certs == null || certs.length == 0) {
/**
* FDroid's index.jar is signed using a particular format and does not allow lots of
* signing setups that would be valid for a regular jar. This validates those
* restrictions.
*/
private X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws UpdateException {
final CodeSigner[] codeSigners = jarEntry.getCodeSigners();
if (codeSigners == null || codeSigners.length == 0) {
throw new UpdateException(repo, "No signature found in index");
}
Log.d(TAG, "Index has " + certs.length + " signature(s)");
boolean match = false;
for (final Certificate cert : certs) {
String certdata = Hasher.hex(cert);
if (repo.pubkey == null && repo.fingerprint != null) {
String certFingerprint = Utils.calcFingerprint(cert);
Log.d(TAG, "No public key for repo " + repo.address + " yet, but it does have a fingerprint, so comparing them.");
Log.d(TAG, "Repo fingerprint: " + repo.fingerprint);
Log.d(TAG, "Cert fingerprint: " + certFingerprint);
if (repo.fingerprint.equalsIgnoreCase(certFingerprint)) {
repo.pubkey = certdata;
usePubkeyInJar = true;
}
}
if (repo.pubkey != null && repo.pubkey.equals(certdata)) {
Log.d(TAG, "Checking repo public key against cert found in jar.");
match = true;
break;
}
/* we could in theory support more than 1, but as of now we do not */
if (codeSigners.length > 1) {
throw new UpdateException(repo, "index.jar must be signed by a single code signer!");
}
return match;
}
protected File extractIndexFromJar(File indexJar) throws UpdateException {
File indexFile = null;
JarFile jarFile = null;
try {
jarFile = new JarFile(indexJar, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry("index.xml");
indexFile = File.createTempFile("index-", "-extracted.xml", context.getCacheDir());
InputStream input = null;
OutputStream output = null;
try {
/*
* JarFile.getInputStream() provides the signature check, even
* though the Android docs do not mention this, the Java docs do
* and Android seems to implement it the same:
* http://docs.oracle.com/javase/6/docs/api/java/util/jar/JarFile.html#getInputStream(java.util.zip.ZipEntry)
* https://developer.android.com/reference/java/util/jar/JarFile.html#getInputStream(java.util.zip.ZipEntry)
*/
input = jarFile.getInputStream(indexEntry);
output = new FileOutputStream(indexFile);
Utils.copy(input, output);
} finally {
Utils.closeQuietly(output);
Utils.closeQuietly(input);
}
// Can only read certificates from jar after it has been read
// completely, so we put it after the copy above...
if (isTofuRequest()) {
Log.i(TAG, "Implicitly trusting the signature of index.jar, because this is a TOFU request");
// Note that later on in the process we will save the pubkey against they repo, so
// that future requests verify against the signature we got this time.
} else if (!verifyCerts(indexEntry)) {
indexFile.delete();
throw new UpdateException(repo, "Index signature mismatch");
}
} catch (IOException e) {
if (indexFile != null) {
indexFile.delete();
}
throw new UpdateException(
repo, "Error opening signed index", e);
} finally {
if (jarFile != null) {
try {
jarFile.close();
} catch (IOException ioe) {
// ignore
}
}
List<? extends Certificate> certs = codeSigners[0].getSignerCertPath().getCertificates();
if (certs.size() != 1) {
throw new UpdateException(repo, "index.jar code signers must only have a single certificate!");
}
return indexFile;
return (X509Certificate) certs.get(0);
}
/**
* If the repo doesn't have a fingerprint, then this is a "Trust On First Use" (TOFU)
* request. In that case, we will not verify the certificate, but rather implicitly trust
* the file we downloaded. We'll extract the certificate from the jar, and then use that
* to verify future requests to the same repository.
* A new repo can be added with or without the fingerprint of the signing
* certificate. If no fingerprint is supplied, then do a pure TOFU and just
* store the certificate as valid. If there is a fingerprint, then first
* check that the signing certificate in the jar matches that fingerprint.
*/
private boolean isTofuRequest() {
return TextUtils.isEmpty(repo.fingerprint);
private void verifyAndStoreTOFUCerts(String certFromIndexXml, X509Certificate rawCertFromJar)
throws UpdateException {
if (repo.pubkey != null)
return; // there is a repo.pubkey already, nothing to TOFU
/* The first time a repo is added, it can be added with the signing certificate's
* fingerprint. In that case, check that fingerprint against what is
* actually in the index.jar itself. If no fingerprint, just store the
* signing certificate */
boolean trustNewSigningCertificate = false;
if (repo.fingerprint == null) {
// no info to check things are valid, so just Trust On First Use
trustNewSigningCertificate = true;
} else {
String fingerprintFromIndexXml = Utils.calcFingerprint(certFromIndexXml);
String fingerprintFromJar = Utils.calcFingerprint(rawCertFromJar);
if (repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml)
&& repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) {
trustNewSigningCertificate = true;
} else {
throw new UpdateException(repo, "Supplied certificate fingerprint does not match!");
}
}
if (trustNewSigningCertificate) {
Log.d(TAG, "Saving new signing certificate in the database for " + repo.address);
ContentValues values = new ContentValues(2);
values.put(RepoProvider.DataColumns.LAST_UPDATED, Utils.formatDate(new Date(), ""));
values.put(RepoProvider.DataColumns.PUBLIC_KEY, Hasher.hex(rawCertFromJar));
RepoProvider.Helper.update(context, repo, values);
}
}
/**
* FDroid works with three copies of the signing certificate:
* <li>in the downloaded jar</li>
* <li>in the index XML</li>
* <li>stored in the local database</li>
* It would work better removing the copy from the index XML, but it needs to stay
* there for backwards compatibility since the old TOFU process requires it. Therefore,
* since all three have to be present, all three are compared.
*
* @param certFromIndexXml the cert written into the header of the index XML
* @param rawCertFromJar the {@link X509Certificate} embedded in the downloaded jar
*/
private void verifyCerts(String certFromIndexXml, X509Certificate rawCertFromJar) throws UpdateException {
// convert binary data to string version that is used in FDroid's database
String certFromJar = Hasher.hex(rawCertFromJar);
// repo and repo.pubkey must be pre-loaded from the database
if (repo == null
|| TextUtils.isEmpty(repo.pubkey)
|| TextUtils.isEmpty(certFromJar)
|| TextUtils.isEmpty(certFromIndexXml))
throw new UpdateException(repo, "A empty repo or signing certificate is invalid!");
// though its called repo.pubkey, its actually a X509 certificate
if (repo.pubkey.equals(certFromJar)
&& repo.pubkey.equals(certFromIndexXml)
&& certFromIndexXml.equals(certFromJar)) {
return; // we have a match!
}
throw new UpdateException(repo, "Signing certificate does not match!");
}
}

View File

@ -32,6 +32,9 @@ import org.xml.sax.helpers.DefaultHandler;
import java.util.ArrayList;
import java.util.List;
/**
* Parses the index.xml into Java data structures.
*/
public class RepoXMLHandler extends DefaultHandler {
// The repo we're processing.
@ -49,27 +52,18 @@ public class RepoXMLHandler extends DefaultHandler {
private int version = -1;
private int maxage = -1;
// After processing the XML, this will be null if the index specified a
// public key - otherwise a public key. This is used for TOFU where an
// index.xml is read on the first connection, and a signed index.jar is
// expected on all subsequent connections.
private String pubkey;
/** the X.509 signing certificate stored in the header of index.xml */
private String signingCertFromIndexXml;
private String name;
private String description;
private String hashType;
private int progressCounter = 0;
private final ProgressListener progressListener;
private int totalAppCount;
public RepoXMLHandler(Repo repo, ProgressListener listener) {
public RepoXMLHandler(Repo repo) {
this.repo = repo;
pubkey = null;
signingCertFromIndexXml = null;
name = null;
description = null;
progressListener = listener;
}
public List<App> getApps() { return apps; }
@ -84,7 +78,7 @@ public class RepoXMLHandler extends DefaultHandler {
public String getName() { return name; }
public String getPubKey() { return pubkey; }
public String getSigningCertFromIndexXml() { return signingCertFromIndexXml; }
@Override
public void characters(char[] ch, int start, int length) {
@ -248,10 +242,7 @@ public class RepoXMLHandler extends DefaultHandler {
super.startElement(uri, localName, qName, attributes);
if (localName.equals("repo")) {
final String pk = attributes.getValue("", "pubkey");
if (pk != null)
pubkey = pk;
signingCertFromIndexXml = attributes.getValue("", "pubkey");
maxage = Utils.parseInt(attributes.getValue("", "maxage"), -1);
version = Utils.parseInt(attributes.getValue("", "version"), -1);
@ -265,16 +256,6 @@ public class RepoXMLHandler extends DefaultHandler {
} else if (localName.equals("application") && curapp == null) {
curapp = new App();
curapp.id = attributes.getValue("", "id");
/* show progress for the first 25, then start skipping every 25 */
if (totalAppCount < 25 || progressCounter % (totalAppCount / 25) == 0) {
Bundle data = new Bundle(1);
data.putString(RepoUpdater.PROGRESS_DATA_REPO_ADDRESS, repo.address);
progressListener.onProgress(
new ProgressListener.Event(
RepoUpdater.PROGRESS_TYPE_PROCESS_XML,
progressCounter, totalAppCount, data));
}
progressCounter++;
} else if (localName.equals("package") && curapp != null && curapk == null) {
curapk = new Apk();
curapk.id = curapp.id;
@ -287,10 +268,6 @@ public class RepoXMLHandler extends DefaultHandler {
curchars.setLength(0);
}
public void setTotalAppCount(int totalAppCount) {
this.totalAppCount = totalAppCount;
}
private String cleanWhiteSpace(String str) {
return str.replaceAll("\n", " ").replaceAll(" ", " ");
}

View File

@ -769,7 +769,6 @@ public class UpdateService extends IntentService implements ProgressListener {
Log.d(TAG, "Removing " + numDeleted + " apks that don't have any apks");
}
/**
* Received progress event from the RepoXMLHandler. It could be progress
* downloading from the repo, or perhaps processing the info from the repo.
@ -780,16 +779,16 @@ public class UpdateService extends IntentService implements ProgressListener {
// TODO: Switch to passing through Bundles of data with the event, rather than a repo address. They are
// now much more general purpose then just repo downloading.
String repoAddress = event.getData().getString(RepoUpdater.PROGRESS_DATA_REPO_ADDRESS);
String downloadedSize = Utils.getFriendlySize(event.progress);
String totalSize = Utils.getFriendlySize(event.total);
int percent = (int) ((double) event.progress / event.total * 100);
switch (event.type) {
case Downloader.EVENT_PROGRESS:
String downloadedSize = Utils.getFriendlySize(event.progress);
String totalSize = Utils.getFriendlySize(event.total);
int percent = (int)((double)event.progress/event.total * 100);
message = getString(R.string.status_download, repoAddress, downloadedSize, totalSize, percent);
break;
case RepoUpdater.PROGRESS_TYPE_PROCESS_XML:
message = getString(R.string.status_processing_xml, repoAddress, event.progress, event.total);
break;
case Downloader.EVENT_PROGRESS:
message = getString(R.string.status_download, repoAddress, downloadedSize, totalSize, percent);
break;
case RepoUpdater.PROGRESS_TYPE_PROCESS_XML:
message = getString(R.string.status_processing_xml_percent, repoAddress, downloadedSize, totalSize, percent);
break;
}
sendStatus(STATUS_INFO, message);
}

View File

@ -45,7 +45,6 @@ import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -260,36 +259,6 @@ public final class Utils {
return getMinMaxSdkVersion(context, packageName, "maxSdkVersion");
}
public static int countSubstringOccurrence(File file, String substring) throws IOException {
int count = 0;
FileReader input = null;
try {
int currentSubstringIndex = 0;
char[] buffer = new char[4096];
input = new FileReader(file);
int numRead = input.read(buffer);
while(numRead != -1) {
for (char c : buffer) {
if (c == substring.charAt(currentSubstringIndex)) {
currentSubstringIndex++;
if (currentSubstringIndex == substring.length()) {
count++;
currentSubstringIndex = 0;
}
} else {
currentSubstringIndex = 0;
}
}
numRead = input.read(buffer);
}
} finally {
closeQuietly(input);
}
return count;
}
// return a fingerprint formatted for display
public static String formatFingerprint(Context context, String fingerprint) {
if (TextUtils.isEmpty(fingerprint)

View File

@ -8,10 +8,11 @@ import org.fdroid.fdroid.compat.FileCompatForTest;
import org.fdroid.fdroid.data.SanitizedFile;
import java.io.File;
import java.util.UUID;
public class FileCompatTest extends InstrumentationTestCase {
private static final String TAG = "org.fdroid.fdroid.FileCompatTest";
private static final String TAG = "FileCompatTest";
private File dir;
private SanitizedFile sourceFile;
@ -20,25 +21,22 @@ public class FileCompatTest extends InstrumentationTestCase {
public void setUp() {
dir = TestUtils.getWriteableDir(getInstrumentation());
sourceFile = SanitizedFile.knownSanitized(TestUtils.copyAssetToDir(getInstrumentation().getContext(), "simpleIndex.jar", dir));
destFile = new SanitizedFile(dir, "dest.txt");
assertTrue(!destFile.exists());
destFile = new SanitizedFile(dir, "dest-" + UUID.randomUUID() + ".testproduct");
assertFalse(destFile.exists());
assertTrue(sourceFile.getAbsolutePath() + " should exist.", sourceFile.exists());
}
public void tearDown() {
if (sourceFile.exists()) {
assertTrue("Can't delete " + sourceFile.getAbsolutePath() + ".", sourceFile.delete());
if (!sourceFile.delete()) {
System.out.println("Can't delete " + sourceFile.getAbsolutePath() + ".");
}
if (destFile.exists()) {
assertTrue("Can't delete " + destFile.getAbsolutePath() + ".", destFile.delete());
if (!destFile.delete()) {
System.out.println("Can't delete " + destFile.getAbsolutePath() + ".");
}
}
public void testSymlinkRuntime() {
SanitizedFile destFile = new SanitizedFile(dir, "dest.txt");
assertFalse(destFile.exists());
FileCompatForTest.symlinkRuntimeTest(sourceFile, destFile);
assertTrue(destFile.getAbsolutePath() + " should exist after symlinking", destFile.exists());
}
@ -46,9 +44,6 @@ public class FileCompatTest extends InstrumentationTestCase {
public void testSymlinkLibcore() {
if (Build.VERSION.SDK_INT >= 19) {
SanitizedFile destFile = new SanitizedFile(dir, "dest.txt");
assertFalse(destFile.exists());
FileCompatForTest.symlinkLibcoreTest(sourceFile, destFile);
assertTrue(destFile.getAbsolutePath() + " should exist after symlinking", destFile.exists());
} else {
@ -59,9 +54,6 @@ public class FileCompatTest extends InstrumentationTestCase {
public void testSymlinkOs() {
if (Build.VERSION.SDK_INT >= 21 ) {
SanitizedFile destFile = new SanitizedFile(dir, "dest.txt");
assertFalse(destFile.exists());
FileCompatForTest.symlinkOsTest(sourceFile, destFile);
assertTrue(destFile.getAbsolutePath() + " should exist after symlinking", destFile.exists());
} else {

View File

@ -5,12 +5,11 @@ import android.annotation.TargetApi;
import android.content.Context;
import android.test.InstrumentationTestCase;
import org.apache.commons.io.FileUtils;
import org.fdroid.fdroid.RepoUpdater.UpdateException;
import org.fdroid.fdroid.data.Repo;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
@TargetApi(8)
public class RepoUpdaterTest extends InstrumentationTestCase {
@ -34,17 +33,12 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
public void testExtractIndexFromJar() {
if (!testFilesDir.canWrite())
return;
File simpleIndexXml = TestUtils.copyAssetToDir(context, "simpleIndex.xml", testFilesDir);
File simpleIndexJar = TestUtils.copyAssetToDir(context, "simpleIndex.jar", testFilesDir);
File testFile = null;
// these are supposed to succeed
try {
testFile = repoUpdater.getIndexFromFile(simpleIndexJar);
assertTrue(testFile.length() == simpleIndexXml.length());
assertEquals(FileUtils.readFileToString(testFile),
FileUtils.readFileToString(simpleIndexXml));
} catch (IOException | UpdateException e) {
repoUpdater.processDownloadedFile(simpleIndexJar, UUID.randomUUID().toString());
} catch (UpdateException e) {
e.printStackTrace();
fail();
}
@ -55,7 +49,8 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
return;
// this is supposed to fail
try {
repoUpdater.getIndexFromFile(TestUtils.copyAssetToDir(context, "simpleIndexWithoutSignature.jar", testFilesDir));
File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithoutSignature.jar", testFilesDir);
repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString());
fail();
} catch (UpdateException e) {
// success!
@ -67,7 +62,8 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
return;
// this is supposed to fail
try {
repoUpdater.getIndexFromFile(TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedManifest.jar", testFilesDir));
File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedManifest.jar", testFilesDir);
repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString());
fail();
} catch (UpdateException e) {
e.printStackTrace();
@ -82,7 +78,8 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
return;
// this is supposed to fail
try {
repoUpdater.getIndexFromFile(TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedSignature.jar", testFilesDir));
File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedSignature.jar", testFilesDir);
repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString());
fail();
} catch (UpdateException e) {
e.printStackTrace();
@ -97,7 +94,8 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
return;
// this is supposed to fail
try {
repoUpdater.getIndexFromFile(TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedCertificate.jar", testFilesDir));
File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedCertificate.jar", testFilesDir);
repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString());
fail();
} catch (UpdateException e) {
e.printStackTrace();
@ -112,7 +110,8 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
return;
// this is supposed to fail
try {
repoUpdater.getIndexFromFile(TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedEverything.jar", testFilesDir));
File jarFile = TestUtils.copyAssetToDir(context, "simpleIndexWithCorruptedEverything.jar", testFilesDir);
repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString());
fail();
} catch (UpdateException e) {
e.printStackTrace();
@ -127,7 +126,8 @@ public class RepoUpdaterTest extends InstrumentationTestCase {
return;
// this is supposed to fail
try {
repoUpdater.getIndexFromFile(TestUtils.copyAssetToDir(context, "masterKeyIndex.jar", testFilesDir));
File jarFile = TestUtils.copyAssetToDir(context, "masterKeyIndex.jar", testFilesDir);
repoUpdater.processDownloadedFile(jarFile, UUID.randomUUID().toString());
fail();
} catch (UpdateException | SecurityException e) {
// success!