Refactored updating code to make future modifications easier.

Rebased several months of work, and attempted to resolve any conflicts.
The conflicts were a tad more difficult than usual to resolve because they
were in files where large blocks of code were refactored into different
files, and git didn't realise.

Conflicts:
	src/org/fdroid/fdroid/RepoXMLHandler.java
	src/org/fdroid/fdroid/UpdateService.java

src/org/fdroid/fdroid/DB.java

@@ -65,7 +65,7 @@ public class DB {
     // Get access to the database. Must be called before any database activity,
     // and releaseDB must be called subsequently. Returns null in the event of
     // failure.
-    static DB getDB() {
+    public static DB getDB() {
         try {
             dbSync.acquire();
             return dbInstance;
@@ -75,7 +75,7 @@ public class DB {
     }
 
     // Release database access lock acquired via getDB().
-    static void releaseDB() {
+    public static void releaseDB() {
         dbSync.release();
     }
 

src/org/fdroid/fdroid/Hasher.java

@@ -26,6 +26,8 @@ import java.io.FileInputStream;
 import java.io.InputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
 
 public class Hasher {
 
@@ -94,6 +96,16 @@ public class Hasher {
         digest.reset();
     }
 
+    public static String hex(Certificate cert) {
+        byte[] encoded = null;
+        try {
+            encoded = cert.getEncoded();
+        } catch(CertificateEncodingException e) {
+            encoded = new byte[0];
+        }
+        return hex(encoded);
+    }
+
     public static String hex(byte[] sig) {
         byte[] csig = new byte[sig.length * 2];
         for (int j = 0; j < sig.length; j++) {

src/org/fdroid/fdroid/RepoXMLHandler.java

@@ -19,36 +19,15 @@
 
 package org.fdroid.fdroid;
 
-import java.io.BufferedReader;
+import android.os.Bundle;
-import java.io.File;
+import org.fdroid.fdroid.updater.RepoUpdater;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.security.cert.Certificate;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-
-import javax.net.ssl.SSLHandshakeException;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
 import org.xml.sax.Attributes;
-import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
-import org.xml.sax.XMLReader;
 import org.xml.sax.helpers.DefaultHandler;
 
-import android.os.Bundle;
+import java.text.ParseException;
-import android.content.Context;
+import java.text.SimpleDateFormat;
-import android.util.Log;
+import java.util.List;
 
 public class RepoXMLHandler extends DefaultHandler {
 
@@ -78,14 +57,9 @@ public class RepoXMLHandler extends DefaultHandler {
     private int progressCounter = 0;
     private ProgressListener progressListener;
 
-    public static final int PROGRESS_TYPE_DOWNLOAD     = 1;
-    public static final int PROGRESS_TYPE_PROCESS_XML  = 2;
-
-    public static final String PROGRESS_DATA_REPO = "repo";
 
     // The date format used in the repo XML file.
     private SimpleDateFormat mXMLDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-    private static final SimpleDateFormat logDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
     private int totalAppCount;
 
@@ -98,6 +72,22 @@ public class RepoXMLHandler extends DefaultHandler {
         progressListener = listener;
     }
 
+    public int getMaxAge() {
+        int age = 0;
+        if (maxage != null) {
+            try {
+                age = Integer.parseInt(maxage);
+            } catch (NumberFormatException e) {
+                // Do nothing...
+            }
+        }
+        return age;
+    }
+
+    public String getPubKey() {
+        return pubkey;
+    }
+
     @Override
     public void characters(char[] ch, int start, int length) {
         curchars.append(ch, start, length);
@@ -250,12 +240,6 @@ public class RepoXMLHandler extends DefaultHandler {
         }
     }
 
-    private static Bundle createProgressData(String repoAddress) {
-        Bundle data = new Bundle();
-        data.putString(PROGRESS_DATA_REPO, repoAddress);
-        return data;
-    }
-
     @Override
     public void startElement(String uri, String localName, String qName,
             Attributes attributes) throws SAXException {
@@ -275,11 +259,11 @@ public class RepoXMLHandler extends DefaultHandler {
             curapp = new DB.App();
             curapp.detail_Populated = true;
             curapp.id = attributes.getValue("", "id");
-            Bundle progressData = createProgressData(repo.address);
+            Bundle progressData = RepoUpdater.createProgressData(repo.address);
             progressCounter ++;
             progressListener.onProgress(
                 new ProgressListener.Event(
-                    RepoXMLHandler.PROGRESS_TYPE_PROCESS_XML, progressCounter,
+                    RepoUpdater.PROGRESS_TYPE_PROCESS_XML, progressCounter,
                     totalAppCount, progressData));
         } else if (localName.equals("package") && curapp != null && curapk == null) {
             curapk = new DB.Apk();
@@ -292,228 +276,6 @@ public class RepoXMLHandler extends DefaultHandler {
         curchars.setLength(0);
     }
 
-    // Get a remote file. Returns the HTTP response code.
-    // If 'etag' is not null, it's passed to the server as an If-None-Match
-    // header, in which case expect a 304 response if nothing changed.
-    // In the event of a 200 response ONLY, 'retag' (which should be passed
-    // empty) may contain an etag value for the response, or it may be left
-    // empty if none was available.
-    private static int getRemoteFile(Context ctx, String url, String dest,
-            String etag, StringBuilder retag,
-            ProgressListener progressListener,
-            ProgressListener.Event progressEvent) throws MalformedURLException,
-            IOException {
-
-        long startTime = System.currentTimeMillis();
-        URL u = new URL(url);
-        HttpURLConnection connection = (HttpURLConnection) u.openConnection();
-        if (etag != null)
-            connection.setRequestProperty("If-None-Match", etag);
-        int code = connection.getResponseCode();
-        if (code == 200) {
-            // Testing in the emulator for me, showed that figuring out the filesize took about 1 to 1.5 seconds.
-            // To put this in context, downloading a repo of:
-            //  - 400k takes ~6 seconds
-            //  - 5k   takes ~3 seconds
-            // on my connection. I think the 1/1.5 seconds is worth it, because as the repo grows, the tradeoff will
-            // become more worth it.
-            progressEvent.total = connection.getContentLength();
-            Log.d("FDroid", "Downloading " + progressEvent.total + " bytes from " + url);
-            InputStream input = null;
-            OutputStream output = null;
-            try {
-                input = connection.getInputStream();
-                output = ctx.openFileOutput(dest, Context.MODE_PRIVATE);
-                Utils.copy(input, output, progressListener, progressEvent);
-            } finally {
-                Utils.closeQuietly(output);
-                Utils.closeQuietly(input);
-            }
-
-            String et = connection.getHeaderField("ETag");
-            if (et != null)
-                retag.append(et);
-        }
-        Log.d("FDroid", "Fetched " + url + " (" + progressEvent.total +
-                " bytes) in " + (System.currentTimeMillis() - startTime) +
-                "ms");
-        return code;
-
-    }
-
-    // Do an update from the given repo. All applications found, and their
-    // APKs, are added to 'apps'. (If 'apps' already contains an app, its
-    // APKs are merged into the existing one).
-    // Returns null if successful, otherwise an error message to be displayed
-    // to the user (if there is an interactive user!)
-    // 'newetag' should be passed empty. On success, it may contain an etag
-    // value for the index that was successfully processed, or it may contain
-    // null if none was available.
-    public static String doUpdate(Context ctx, DB.Repo repo,
-            List<DB.App> apps, StringBuilder newetag, List<Integer> keeprepos,
-            ProgressListener progressListener) {
-        try {
-
-            int code = 0;
-            if (repo.pubkey != null) {
-
-                // This is a signed repo - we download the jar file,
-                // check the signature, and extract the index...
-                Log.d("FDroid", "Getting signed index from " + repo.address + " at " +
-                    logDateFormat.format(new Date(System.currentTimeMillis())));
-                String address = repo.address + "/index.jar?"
-                    + ctx.getString(R.string.version_name);
-                Bundle progressData = createProgressData(repo.address);
-                ProgressListener.Event event = new ProgressListener.Event(
-                        RepoXMLHandler.PROGRESS_TYPE_DOWNLOAD, progressData);
-                code = getRemoteFile(ctx, address, "tempindex.jar",
-                        repo.lastetag, newetag, progressListener, event );
-                if (code == 200) {
-                    String jarpath = ctx.getFilesDir() + "/tempindex.jar";
-                    JarFile jar = null;
-                    JarEntry je;
-                    Certificate[] certs;
-                    try {
-                        jar = new JarFile(jarpath, true);
-                        je = (JarEntry) jar.getEntry("index.xml");
-                        File efile = new File(ctx.getFilesDir(),
-                                "/tempindex.xml");
-                        InputStream input = null;
-                        OutputStream output = null;
-                        try {
-                            input = jar.getInputStream(je);
-                            output = new FileOutputStream(efile);
-                            Utils.copy(input, output);
-                        } finally {
-                            Utils.closeQuietly(output);
-                            Utils.closeQuietly(input);
-                        }
-                        certs = je.getCertificates();
-                    } catch (SecurityException e) {
-                        Log.e("FDroid", "Invalid hash for index file");
-                        return "Invalid hash for index file";
-                    } finally {
-                        if (jar != null) {
-                            jar.close();
-                        }
-                    }
-                    if (certs == null) {
-                        Log.d("FDroid", "No signature found in index");
-                        return "No signature found in index";
-                    }
-                    Log.d("FDroid", "Index has " + certs.length + " signature"
-                            + (certs.length > 1 ? "s." : "."));
-
-                    boolean match = false;
-                    for (Certificate cert : certs) {
-                        String certdata = Hasher.hex(cert.getEncoded());
-                        if (repo.pubkey.equals(certdata)) {
-                            match = true;
-                            break;
-                        }
-                    }
-                    if (!match) {
-                        Log.d("FDroid", "Index signature mismatch");
-                        return "Index signature mismatch";
-                    }
-                }
-
-            } else {
-
-                // It's an old-fashioned unsigned repo...
-                Log.d("FDroid", "Getting unsigned index from " + repo.address);
-                Bundle eventData = createProgressData(repo.address);
-                ProgressListener.Event event = new ProgressListener.Event(
-                        RepoXMLHandler.PROGRESS_TYPE_DOWNLOAD, eventData);
-                code = getRemoteFile(ctx, repo.address + "/index.xml",
-                        "tempindex.xml", repo.lastetag, newetag,
-                        progressListener, event);
-            }
-
-            if (code == 200) {
-                // Process the index...
-                SAXParserFactory spf = SAXParserFactory.newInstance();
-                SAXParser sp = spf.newSAXParser();
-                XMLReader xr = sp.getXMLReader();
-                RepoXMLHandler handler = new RepoXMLHandler(repo, apps, progressListener);
-                xr.setContentHandler(handler);
-
-                File tempIndex = new File(ctx.getFilesDir() + "/tempindex.xml");
-                BufferedReader r = new BufferedReader(new FileReader(tempIndex));
-
-                // A bit of a hack, this might return false positives if an apps description
-                // or some other part of the XML file contains this, but it is a pretty good
-                // estimate and makes the progress counter more informative.
-                // As with asking the server about the size of the index before downloading,
-                // this also has a time tradeoff. It takes about three seconds to iterate
-                // through the file and count 600 apps on a slow emulator (v17), but if it is
-                // taking two minutes to update, the three second wait may be worth it.
-                final String APPLICATION = "<application";
-                handler.setTotalAppCount(Utils.countSubstringOccurrence(tempIndex, APPLICATION));
-
-                InputSource is = new InputSource(r);
-                xr.parse(is);
-
-                if (handler.pubkey != null && repo.pubkey == null) {
-                    // We read an unsigned index, but that indicates that
-                    // a signed version is now available...
-                    Log.d("FDroid",
-                            "Public key found - switching to signed repo for future updates");
-                    repo.pubkey = handler.pubkey;
-                    try {
-                        DB db = DB.getDB();
-                        db.updateRepoByAddress(repo);
-                    } finally {
-                        DB.releaseDB();
-                    }
-                }
-
-                if (handler.maxage != null) {
-                    int maxage = Integer.parseInt(handler.maxage);
-                    if (maxage != repo.maxage) {
-                        Log.d("FDroid",
-                                "Repo specified a new maximum age - updated");
-                        repo.maxage = maxage;
-                        try {
-                            DB db = DB.getDB();
-                            db.updateRepoByAddress(repo);
-                        } finally {
-                            DB.releaseDB();
-                        }
-                    }
-                }
-
-            } else if (code == 304) {
-                // The index is unchanged since we last read it. We just mark
-                // everything that came from this repo as being updated.
-                Log.d("FDroid", "Repo index for " + repo.address
-                        + " is up to date (by etag)");
-                keeprepos.add(repo.id);
-                // Make sure we give back the same etag. (The 200 route will
-                // have supplied a new one.
-                newetag.append(repo.lastetag);
-
-            } else {
-                return "Failed to read index - HTTP response "
-                        + Integer.toString(code);
-            }
-
-        } catch (SSLHandshakeException sslex) {
-            Log.e("FDroid", "SSLHandShakeException updating from "
-                    + repo.address + ":\n" + Log.getStackTraceString(sslex));
-            return "A problem occurred while establishing an SSL connection. If this problem persists, AND you have a very old device, you could try using http instead of https for the repo URL.";
-        } catch (Exception e) {
-            Log.e("FDroid", "Exception updating from " + repo.address + ":\n"
-                    + Log.getStackTraceString(e));
-            return "Failed to update - " + e.getMessage();
-        } finally {
-            ctx.deleteFile("tempindex.xml");
-            ctx.deleteFile("tempindex.jar");
-        }
-
-        return null;
-    }
-
     public void setTotalAppCount(int totalAppCount) {
         this.totalAppCount = totalAppCount;
     }

src/org/fdroid/fdroid/UpdateService.java

@@ -29,7 +29,6 @@ import android.content.Context;
 import android.content.Intent;
 import android.content.SharedPreferences;
 import android.content.SharedPreferences.Editor;
-import android.graphics.BitmapFactory;
 import android.net.ConnectivityManager;
 import android.net.NetworkInfo;
 import android.os.Build;
@@ -38,6 +37,7 @@ import android.os.ResultReceiver;
 import android.os.SystemClock;
 import android.preference.PreferenceManager;
 import android.util.Log;
+import org.fdroid.fdroid.updater.RepoUpdater;
 
 import android.support.v4.app.NotificationCompat;
 import android.support.v4.app.TaskStackBuilder;
@@ -164,45 +164,39 @@ public class UpdateService extends IntentService implements ProgressListener {
             // database while we do all the downloading, etc...
             int updates = 0;
             List<DB.Repo> repos;
+            List<DB.App> apps;
             try {
                 DB db = DB.getDB();
                 repos = db.getRepos();
+                apps = db.getApps(false);
             } finally {
                 DB.releaseDB();
             }
 
             // Process each repo...
-            List<DB.App> apps;
             List<DB.App> updatingApps = new ArrayList<DB.App>();
             List<Integer> keeprepos = new ArrayList<Integer>();
             boolean success = true;
             boolean changes = false;
             for (DB.Repo repo : repos) {
-                if (repo.inuse) {
+                if (!repo.inuse) {
-
+                    continue;
-                    sendStatus(
+                }
-                            STATUS_INFO,
+                sendStatus(STATUS_INFO, getString(R.string.status_connecting_to_repo, repo.address));
-                            getString(R.string.status_connecting_to_repo,
+                RepoUpdater updater = RepoUpdater.createUpdaterFor(getBaseContext(), repo);
-                                    repo.address));
+                updater.setProgressListener(this);
-
+                try {
-                    StringBuilder newetag = new StringBuilder();
+                    updater.update();
-                    String err = RepoXMLHandler.doUpdate(getBaseContext(),
+                    if (updater.hasChanged()) {
-                            repo, updatingApps, newetag, keeprepos, this);
+                        updatingApps.addAll(updater.getApps());
-                    if (err == null) {
+                        changes = true;
-                        String nt = newetag.toString();
-                        if (!nt.equals(repo.lastetag)) {
-                            repo.lastetag = newetag.toString();
-                            changes = true;
-                        }
                     } else {
-                        success = false;
+                        keeprepos.add(repo.id);
-                        err = "Update failed for " + repo.address + " - " + err;
-                        Log.d("FDroid", err);
-                        if (errmsg.length() == 0)
-                            errmsg = err;
-                        else
-                            errmsg += "\n" + err;
                     }
+                } catch (RepoUpdater.UpdateException e) {
+                    errmsg += (errmsg.length() == 0 ? "" : "\n") + e.getMessage();
+                    Log.e("FDroid", "Error updating repository " + repo.address + ": " + e.getMessage());
+                    Log.e("FDroid", Log.getStackTraceString(e));
                 }
             }
 
@@ -348,23 +342,17 @@ public class UpdateService extends IntentService implements ProgressListener {
      */
     @Override
     public void onProgress(ProgressListener.Event event) {
-
         String message = "";
-        if (event.type == RepoXMLHandler.PROGRESS_TYPE_DOWNLOAD) {
+        if (event.type == RepoUpdater.PROGRESS_TYPE_DOWNLOAD) {
-            String repoAddress = event.data
+            String repoAddress    = event.data.getString(RepoUpdater.PROGRESS_DATA_REPO);
-                    .getString(RepoXMLHandler.PROGRESS_DATA_REPO);
+            String downloadedSize = Utils.getFriendlySize( event.progress );
-            String downloadedSize = Utils.getFriendlySize(event.progress);
+            String totalSize      = Utils.getFriendlySize( event.total );
-            String totalSize = Utils.getFriendlySize(event.total);
+            int percent           = (int)((double)event.progress/event.total * 100);
-            int percent = (int) ((double) event.progress / event.total * 100);
+            message = getString(R.string.status_download, repoAddress, downloadedSize, totalSize, percent);
-            message = getString(R.string.status_download, repoAddress,
+        } else if (event.type == RepoUpdater.PROGRESS_TYPE_PROCESS_XML) {
-                    downloadedSize, totalSize, percent);
+            String repoAddress    = event.data.getString(RepoUpdater.PROGRESS_DATA_REPO);
-        } else if (event.type == RepoXMLHandler.PROGRESS_TYPE_PROCESS_XML) {
+            message = getString(R.string.status_processing_xml, repoAddress, event.progress, event.total);
-            String repoAddress = event.data
-                    .getString(RepoXMLHandler.PROGRESS_DATA_REPO);
-            message = getString(R.string.status_processing_xml, repoAddress,
-                    event.progress, event.total);
         }
-
         sendStatus(STATUS_INFO, message);
     }
 }

src/org/fdroid/fdroid/Utils.java

@@ -25,6 +25,7 @@ import java.io.FileReader;
 import java.io.InputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.text.SimpleDateFormat;
 
 public final class Utils {
 
@@ -33,6 +34,10 @@ public final class Utils {
     private static final String[] FRIENDLY_SIZE_FORMAT = {
             "%.0f B", "%.0f KiB", "%.1f MiB", "%.2f GiB" };
 
+    public static final SimpleDateFormat LOG_DATE_FORMAT =
+            new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+
 
     public static void copy(InputStream input, OutputStream output)
             throws IOException {

src/org/fdroid/fdroid/net/Downloader.java

@@ -0,0 +1,138 @@
+package org.fdroid.fdroid.net;
+
+import java.io.*;
+import java.net.*;
+import android.content.*;
+import org.fdroid.fdroid.*;
+
+public class Downloader {
+
+    private static final String HEADER_IF_NONE_MATCH = "If-None-Match";
+    private static final String HEADER_FIELD_ETAG = "ETag";
+
+    private URL sourceUrl;
+    private OutputStream outputStream;
+    private ProgressListener progressListener = null;
+    private ProgressListener.Event progressEvent = null;
+    private String eTag = null;
+    private final File outputFile;
+    private HttpURLConnection connection;
+    private int statusCode = -1;
+
+    // The context is required for opening the file to write to.
+    public Downloader(String source, String destFile, Context ctx)
+            throws FileNotFoundException, MalformedURLException {
+        sourceUrl    = new URL(source);
+        outputStream = ctx.openFileOutput(destFile, Context.MODE_PRIVATE);
+        outputFile   = new File(ctx.getFilesDir() + File.separator + destFile);
+    }
+
+    /**
+     * Downloads to a temporary file, which *you must delete yourself when
+     * you are done*.
+     * @see org.fdroid.fdroid.net.Downloader#getFile()
+     */
+    public Downloader(String source, Context ctx) throws IOException {
+        // http://developer.android.com/guide/topics/data/data-storage.html#InternalCache
+        outputFile = File.createTempFile("dl-", "", ctx.getCacheDir());
+        outputStream = new FileOutputStream(outputFile);
+        sourceUrl = new URL(source);
+    }
+
+    public Downloader(String source, OutputStream output)
+            throws MalformedURLException {
+        sourceUrl    = new URL(source);
+        outputStream = output;
+        outputFile   = null;
+    }
+
+    public void setProgressListener(ProgressListener progressListener,
+                                    ProgressListener.Event progressEvent) {
+        this.progressListener = progressListener;
+        this.progressEvent = progressEvent;
+    }
+
+    /**
+     * Only available if you passed a context object into the constructor
+     * (rather than an outputStream, which may or  may not be associated with
+     * a file).
+     */
+    public File getFile() {
+        return outputFile;
+    }
+
+    /**
+     * Only available after downloading a file.
+     */
+    public int getStatusCode() {
+        return statusCode;
+    }
+
+    /**
+     * If you ask for the eTag before calling download(), you will get the
+     * same one you passed in (if any). If you call it after download(), you
+     * will get the new eTag from the server, or null if there was none.
+     */
+    public String getETag() {
+        return eTag;
+    }
+
+    /**
+     * If this eTag matches that returned by the server, then no download will
+     * take place, and a status code of 304 will be returned by download().
+     */
+    public void setETag(String eTag) {
+        this.eTag = eTag;
+    }
+
+    // Get a remote file. Returns the HTTP response code.
+    // If 'etag' is not null, it's passed to the server as an If-None-Match
+    // header, in which case expect a 304 response if nothing changed.
+    // In the event of a 200 response ONLY, 'retag' (which should be passed
+    // empty) may contain an etag value for the response, or it may be left
+    // empty if none was available.
+    public int download() throws IOException {
+        connection = (HttpURLConnection)sourceUrl.openConnection();
+        setupCacheCheck();
+        statusCode = connection.getResponseCode();
+        if (statusCode == 200) {
+            setupProgressListener();
+            InputStream input = null;
+            try {
+                input = connection.getInputStream();
+                Utils.copy(input, outputStream,
+                        progressListener, progressEvent);
+            } finally {
+                Utils.closeQuietly(outputStream);
+                Utils.closeQuietly(input);
+            }
+            updateCacheCheck();
+        }
+        return statusCode;
+    }
+
+    protected void setupCacheCheck() {
+        if (eTag != null) {
+            connection.setRequestProperty(HEADER_IF_NONE_MATCH, eTag);
+        }
+    }
+
+    protected void updateCacheCheck() {
+        eTag = connection.getHeaderField(HEADER_FIELD_ETAG);
+    }
+
+    protected void setupProgressListener() {
+        if (progressListener != null && progressEvent != null) {
+            // Testing in the emulator for me, showed that figuring out the
+            // filesize took about 1 to 1.5 seconds.
+            // To put this in context, downloading a repo of:
+            //  - 400k takes ~6 seconds
+            //  - 5k   takes ~3 seconds
+            // on my connection. I think the 1/1.5 seconds is worth it,
+            // because as the repo grows, the tradeoff will
+            // become more worth it.
+            progressEvent.total = connection.getContentLength();
+        }
+    }
+
+}

src/org/fdroid/fdroid/updater/RepoUpdater.java

@@ -0,0 +1,243 @@
+package org.fdroid.fdroid.updater;
+
+import android.content.Context;
+import android.os.Bundle;
+import android.util.Log;
+import org.fdroid.fdroid.DB;
+import org.fdroid.fdroid.ProgressListener;
+import org.fdroid.fdroid.RepoXMLHandler;
+import org.fdroid.fdroid.Utils;
+import org.fdroid.fdroid.net.Downloader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+import javax.net.ssl.SSLHandshakeException;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+abstract public class RepoUpdater {
+
+    public static final int PROGRESS_TYPE_DOWNLOAD     = 1;
+    public static final int PROGRESS_TYPE_PROCESS_XML  = 2;
+    public static final String PROGRESS_DATA_REPO      = "repo";
+
+    public static RepoUpdater createUpdaterFor(Context ctx, DB.Repo repo) {
+        if (repo.pubkey != null) {
+            return new SignedRepoUpdater(ctx, repo);
+        } else {
+            return new UnsignedRepoUpdater(ctx, repo);
+        }
+    }
+
+    protected final Context context;
+    protected final DB.Repo repo;
+    protected final List<DB.App> apps = new ArrayList<DB.App>();
+    protected boolean hasChanged = false;
+    protected ProgressListener progressListener;
+
+    public RepoUpdater(Context ctx, DB.Repo repo) {
+        this.context = ctx;
+        this.repo    = repo;
+    }
+
+    public void setProgressListener(ProgressListener progressListener) {
+        this.progressListener = progressListener;
+    }
+
+    public boolean hasChanged() {
+        return hasChanged;
+    }
+
+    public List<DB.App> getApps() {
+        return apps;
+    }
+
+    public boolean isInteractive() {
+        return progressListener != null;
+    }
+
+    /**
+     * Return the index file if it is different than last time,
+     * otherwise returns null to indicate that the file has not changed.
+     * All error states will come via an UpdateException.
+     */
+    protected abstract File getIndexFile() throws UpdateException;
+
+    protected abstract String getIndexAddress();
+
+    protected Downloader downloadIndex() throws UpdateException {
+        Bundle progressData = createProgressData(repo.address);
+        Downloader downloader = null;
+        try {
+            downloader = new Downloader(getIndexAddress(), context);
+            downloader.setETag(repo.lastetag);
+
+            if (isInteractive()) {
+                ProgressListener.Event event =
+                    new ProgressListener.Event(
+                        RepoUpdater.PROGRESS_TYPE_DOWNLOAD, progressData);
+                downloader.setProgressListener(progressListener, event);
+            }
+
+            int status = downloader.download();
+
+            repo.lastetag = downloader.getETag();
+            if (status == 304) {
+                // The index is unchanged since we last read it. We just mark
+                // everything that came from this repo as being updated.
+                Log.d("FDroid", "Repo index for " + repo.address
+                        + " is up to date (by etag)");
+            } else if (status == 200) {
+                hasChanged = true;
+            } else {
+                // Is there any code other than 200 which still returns
+                // content? Just in case, lets try to clean up.
+                if (downloader.getFile() != null) {
+                    downloader.getFile().delete();
+                }
+                throw new UpdateException(
+                        repo,
+                        "Failed to update repo " + repo.address +
+                        " - HTTP response " + status);
+            }
+        } catch (SSLHandshakeException e) {
+            throw new UpdateException(
+                    repo,
+                    "A problem occurred while establishing an SSL " +
+                    "connection. If this problem persists, AND you have a " +
+                    "very old device, you could try using http instead of " +
+                    "https for the repo URL.",
+                    e );
+        } catch (IOException e) {
+            if (downloader != null && downloader.getFile() != null) {
+                downloader.getFile().delete();
+            }
+            throw new UpdateException(
+                    repo,
+                    "Error getting index file from " + repo.address,
+                    e);
+        }
+        return downloader;
+    }
+
+    public static Bundle createProgressData(String repoAddress) {
+        Bundle data = new Bundle();
+        data.putString(PROGRESS_DATA_REPO, repoAddress);
+        return data;
+    }
+
+    private int estimateAppCount(File indexFile) {
+        int count = -1;
+        try {
+            // A bit of a hack, this might return false positives if an apps description
+            // or some other part of the XML file contains this, but it is a pretty good
+            // estimate and makes the progress counter more informative.
+            // As with asking the server about the size of the index before downloading,
+            // this also has a time tradeoff. It takes about three seconds to iterate
+            // through the file and count 600 apps on a slow emulator (v17), but if it is
+            // taking two minutes to update, the three second wait may be worth it.
+            final String APPLICATION = "<application";
+            count = Utils.countSubstringOccurrence(indexFile, APPLICATION);
+        } catch (IOException e) {
+            // Do nothing. Leave count at default -1 value.
+        }
+        return count;
+    }
+
+
+    public void update() throws UpdateException {
+
+        File indexFile = null;
+        try {
+            indexFile = getIndexFile();
+
+            // Process the index...
+            SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+            XMLReader reader = parser.getXMLReader();
+            RepoXMLHandler handler = new RepoXMLHandler(repo, apps, progressListener);
+
+            if (isInteractive()) {
+                // Only bother spending the time to count the expected apps
+                // if we can show that to the user...
+                handler.setTotalAppCount(estimateAppCount(indexFile));
+            }
+
+            reader.setContentHandler(handler);
+            InputSource is = new InputSource(
+                    new BufferedReader(new FileReader(indexFile)));
+
+            reader.parse(is);
+            updateRepo(handler.getPubKey(), handler.getMaxAge());
+
+        } catch (SAXException e) {
+            throw new UpdateException(
+                    repo, "Error parsing index for repo " + repo.address, e);
+        } catch (FileNotFoundException e) {
+            throw new UpdateException(
+                    repo, "Error parsing index for repo " + repo.address, e);
+        } catch (ParserConfigurationException e) {
+            throw new UpdateException(
+                    repo, "Error parsing index for repo " + repo.address, e);
+        } catch (IOException e) {
+            throw new UpdateException(
+                    repo, "Error parsing index for repo " + repo.address, e);
+        } finally {
+            if (indexFile != null && indexFile.exists()) {
+                indexFile.delete();
+            }
+        }
+    }
+
+    private void updateRepo(String publicKey, int maxAge) {
+        boolean changed = false;
+
+        // We read an unsigned index, but that indicates that
+        // a signed version is now available...
+        if (publicKey != null && repo.pubkey == null) {
+            changed = true;
+            // TODO: Spend the time *now* going to get the etag of the signed
+            // repo, so that we can prevent downloading it next time. Otherwise
+            // next time we update, we have to download the signed index
+            // in its entirety, regardless of if it contains the same
+            // information as the unsigned one does not...
+            Log.d("FDroid", "Public key found - switching to signed repo " +
+                    "for future updates");
+            repo.pubkey = publicKey;
+        }
+
+        if (repo.maxage != maxAge) {
+            changed = true;
+            repo.maxage = maxAge;
+        }
+
+        if (changed) {
+            try {
+                DB db = DB.getDB();
+                db.updateRepoByAddress(repo);
+            } finally {
+                DB.releaseDB();
+            }
+        }
+    }
+
+    public static class UpdateException extends Exception {
+
+        public final DB.Repo repo;
+
+        public UpdateException(DB.Repo repo, String message) {
+            super(message);
+            this.repo = repo;
+        }
+
+        public UpdateException(DB.Repo repo, String message, Exception cause) {
+            super(message, cause);
+            this.repo = repo;
+        }
+    }
+
+}

src/org/fdroid/fdroid/updater/SignedRepoUpdater.java

@@ -0,0 +1,114 @@
+package org.fdroid.fdroid.updater;
+
+import android.content.Context;
+import android.util.Log;
+import org.fdroid.fdroid.DB;
+import org.fdroid.fdroid.Hasher;
+import org.fdroid.fdroid.R;
+import org.fdroid.fdroid.Utils;
+import org.fdroid.fdroid.net.Downloader;
+
+import java.io.*;
+import java.security.cert.Certificate;
+import java.util.Date;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+public class SignedRepoUpdater extends RepoUpdater {
+
+    public SignedRepoUpdater(Context ctx, DB.Repo repo) {
+        super(ctx, repo);
+    }
+
+    private boolean verifyCerts(JarEntry item) throws UpdateException {
+        Certificate[] certs = item.getCertificates();
+        if (certs == null || certs.length == 0) {
+            throw new UpdateException(repo, "No signature found in index");
+        }
+
+        Log.d("FDroid", "Index has " + certs.length + " signature(s)");
+        boolean match = false;
+        for (Certificate cert : certs) {
+            String certdata = Hasher.hex(cert);
+            if (repo.pubkey.equals(certdata)) {
+                match = true;
+                break;
+            }
+        }
+        return match;
+    }
+
+    protected File extractIndexFromJar(File indexJar) throws UpdateException {
+        File indexFile  = null;
+        JarFile jarFile = null;
+        try {
+            jarFile = new JarFile(indexJar, true);
+            JarEntry indexEntry = (JarEntry)jarFile.getEntry("index.xml");
+
+            indexFile  = File.createTempFile("index-", ".xml", context.getFilesDir());
+            InputStream input = null;
+            OutputStream output = null;
+            try {
+                input = jarFile.getInputStream(indexEntry);
+                output = new FileOutputStream(indexFile);
+                Utils.copy(input, output);
+            } finally {
+                Utils.closeQuietly(output);
+                Utils.closeQuietly(input);
+            }
+
+            // Can only read certificates from jar after it has been read
+            // completely, so we put it after the copy above...
+            if (!verifyCerts(indexEntry)) {
+                indexFile.delete();
+                throw new UpdateException(repo, "Index signature mismatch");
+            }
+        } catch (IOException e) {
+            if (indexFile != null) {
+                indexFile.delete();
+            }
+            throw new UpdateException(
+                    repo, "Error opening signed index", e);
+        } finally {
+            if (jarFile != null) {
+                try {
+                    jarFile.close();
+                } catch (IOException ioe) {
+                    // ignore
+                }
+            }
+        }
+
+        return indexFile;
+    }
+
+    protected String getIndexAddress() {
+        return repo.address + "/index.jar?" + context.getString(R.string.version_name);
+    }
+
+    /**
+     * As this is a signed repo - we download the jar file,
+     * check the signature, and extract the index file
+     */
+    @Override
+    protected File getIndexFile() throws UpdateException {
+        Date updateTime = new Date(System.currentTimeMillis());
+        Log.d("FDroid", "Getting signed index from " + repo.address + " at " +
+                Utils.LOG_DATE_FORMAT.format(updateTime));
+
+        Downloader downloader = downloadIndex();
+        File indexJar  = downloader.getFile();
+        File indexXml  = null;
+
+        // Don't worry about checking the status code for 200. If it was a
+        // successful download, then we will have a file ready to use:
+        if (indexJar != null && indexJar.exists()) {
+            try {
+                indexXml = extractIndexFromJar(indexJar);
+            } finally {
+                indexJar.delete();
+            }
+        }
+        return indexXml;
+    }
+}

src/org/fdroid/fdroid/updater/UnsignedRepoUpdater.java

@@ -0,0 +1,27 @@
+package org.fdroid.fdroid.updater;
+
+import android.content.Context;
+import android.util.Log;
+import org.fdroid.fdroid.DB;
+import org.fdroid.fdroid.net.Downloader;
+
+import java.io.File;
+
+public class UnsignedRepoUpdater extends RepoUpdater {
+
+    public UnsignedRepoUpdater(Context ctx, DB.Repo repo) {
+        super(ctx, repo);
+    }
+
+    @Override
+    protected File getIndexFile() throws UpdateException {
+        Log.d("FDroid", "Getting unsigned index from " + getIndexAddress());
+        Downloader downloader = downloadIndex();
+        return downloader.getFile();
+    }
+
+    @Override
+    protected String getIndexAddress() {
+        return repo.address + "/index.xml";
+    }
+}