Karl/BobStore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use etags - highly experimental, especially where multiple repos are concerned

Browse Source
This commit is contained in:
Ciaran Gultnieks 2012-09-22 22:33:06 +01:00
parent 9a7d0b9f10
commit 407c903010
5 changed files with 226 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/org/fdroid/fdroid/AppDetails.java
View File

@ -307,7 +307,7 @@ public class AppDetails extends ListActivity {
// Make sure the app is populated. // Make sure the app is populated.
try { try {
DB db = DB.getDB(); DB db = DB.getDB();
db.populateDetails(app); db.populateDetails(app, null);
} catch (Exception ex) { } catch (Exception ex) {
Log.d("FDroid", "Failed to populate app - " + ex.getMessage()); Log.d("FDroid", "Failed to populate app - " + ex.getMessage());
} finally { } finally {

70
src/org/fdroid/fdroid/DB.java
View File

@ -239,7 +239,7 @@ public class DB {
detail_size = 0; detail_size = 0;
apkSource = null; apkSource = null;
added = null; added = null;
detail_server = null; server = null;
detail_hash = null; detail_hash = null;
detail_hashType = null; detail_hashType = null;
detail_permissions = null; detail_permissions = null;
@ -250,7 +250,7 @@ public class DB {
public String version; public String version;
public int vercode; public int vercode;
public int detail_size; // Size in bytes - 0 means we don't know! public int detail_size; // Size in bytes - 0 means we don't know!
public String detail_server; public String server;
public String detail_hash; public String detail_hash;
public String detail_hashType; public String detail_hashType;
public int minSdkVersion; // 0 if unknown public int minSdkVersion; // 0 if unknown
@ -282,7 +282,7 @@ public class DB {
public String getURL() { public String getURL() {
String path = apkName.replace(" ", "%20"); String path = apkName.replace(" ", "%20");
return detail_server + "/" + path; return server + "/" + path;
} }
// Call isCompatible(apk) on an instance of this class to // Call isCompatible(apk) on an instance of this class to
@ -364,16 +364,17 @@ public class DB {
private static final String CREATE_TABLE_REPO = "create table " private static final String CREATE_TABLE_REPO = "create table "
+ TABLE_REPO + " (" + "address text primary key, " + TABLE_REPO + " (" + "address text primary key, "
+ "inuse integer not null, " + "priority integer not null," + "inuse integer not null, " + "priority integer not null,"
+ "pubkey text);"; + "pubkey text, lastetag text);";
public static class Repo { public static class Repo {
public String address; public String address;
public boolean inuse; public boolean inuse;
public int priority; public int priority;
public String pubkey; // null for an unsigned repo public String pubkey; // null for an unsigned repo
public String lastetag; // last etag we updated from, null forces update
} }
private final int DBVersion = 18; private final int DBVersion = 19;
private static void createAppApk(SQLiteDatabase db) { private static void createAppApk(SQLiteDatabase db) {
db.execSQL(CREATE_TABLE_APP); db.execSQL(CREATE_TABLE_APP);
@ -409,6 +410,7 @@ public class DB {
mContext.getString(R.string.default_repo_pubkey)); mContext.getString(R.string.default_repo_pubkey));
values.put("inuse", 1); values.put("inuse", 1);
values.put("priority", 10); values.put("priority", 10);
values.put("lastetag", (String) null);
db.insert(TABLE_REPO, null, values); db.insert(TABLE_REPO, null, values);
} }
@ -417,6 +419,8 @@ public class DB {
resetTransient(db); resetTransient(db);
if (oldVersion < 7) if (oldVersion < 7)
db.execSQL("alter table " + TABLE_REPO + " add pubkey string"); db.execSQL("alter table " + TABLE_REPO + " add pubkey string");
if (oldVersion < 19)
db.execSQL("alter table " + TABLE_REPO + " add lastetag string");
} }
} }
@ -520,7 +524,9 @@ public class DB {
} }
// Populate the details for the given app, if necessary. // Populate the details for the given app, if necessary.
public void populateDetails(App app) { // If 'apkrepo' is not null, only apks from that repo address are
// populated (this is used during the update process)
public void populateDetails(App app, String apkrepo) {
if (app.detail_Populated) if (app.detail_Populated)
return; return;
Cursor c = null; Cursor c = null;
@ -538,24 +544,22 @@ public class DB {
c.close(); c.close();
c = null; c = null;
cols = new String[] { "server", "hash", "hashType", "size", cols = new String[] { "hash", "hashType", "size", "permissions" };
"permissions" };
for (Apk apk : app.apks) { for (Apk apk : app.apks) {
c = db.query( if (apkrepo == null || apkrepo.equals(apk.server)) {
TABLE_APK, c = db.query(TABLE_APK, cols, "id = ? and vercode = "
cols, + Integer.toString(apk.vercode),
"id = ? and vercode = " + Integer.toString(apk.vercode), new String[] { apk.id }, null, null, null, null);
new String[] { apk.id }, null, null, null, null); c.moveToFirst();
c.moveToFirst(); apk.detail_hash = c.getString(0);
apk.detail_server = c.getString(0); apk.detail_hashType = c.getString(1);
apk.detail_hash = c.getString(1); apk.detail_size = c.getInt(2);
apk.detail_hashType = c.getString(2); apk.detail_permissions = CommaSeparatedList.make(c
apk.detail_size = c.getInt(3); .getString(3));
apk.detail_permissions = CommaSeparatedList c.close();
.make(c.getString(4)); c = null;
c.close(); }
c = null;
} }
app.detail_Populated = true; app.detail_Populated = true;
@ -636,7 +640,7 @@ public class DB {
cols = new String[] { "id", "version", "vercode", "sig", "srcname", cols = new String[] { "id", "version", "vercode", "sig", "srcname",
"apkName", "apkSource", "minSdkVersion", "added", "apkName", "apkSource", "minSdkVersion", "added",
"features", "compatible" }; "features", "compatible", "server" };
c = db.query(TABLE_APK, cols, null, null, null, null, c = db.query(TABLE_APK, cols, null, null, null, null,
"vercode desc"); "vercode desc");
c.moveToFirst(); c.moveToFirst();
@ -655,6 +659,7 @@ public class DB {
: mDateFormat.parse(sApkAdded); : mDateFormat.parse(sApkAdded);
apk.features = CommaSeparatedList.make(c.getString(9)); apk.features = CommaSeparatedList.make(c.getString(9));
apk.compatible = c.getInt(10) == 1; apk.compatible = c.getInt(10) == 1;
apk.server = c.getString(11);
apps.get(apk.id).apks.add(apk); apps.get(apk.id).apks.add(apk);
c.moveToNext(); c.moveToNext();
} }
@ -945,7 +950,7 @@ public class DB {
values.put("id", upapk.id); values.put("id", upapk.id);
values.put("version", upapk.version); values.put("version", upapk.version);
values.put("vercode", upapk.vercode); values.put("vercode", upapk.vercode);
values.put("server", upapk.detail_server); values.put("server", upapk.server);
values.put("hash", upapk.detail_hash); values.put("hash", upapk.detail_hash);
values.put("hashType", upapk.detail_hashType); values.put("hashType", upapk.detail_hashType);
values.put("sig", upapk.sig); values.put("sig", upapk.sig);
@ -974,8 +979,9 @@ public class DB {
Vector<Repo> repos = new Vector<Repo>(); Vector<Repo> repos = new Vector<Repo>();
Cursor c = null; Cursor c = null;
try { try {
c = db.rawQuery("select address, inuse, priority, pubkey from " c = db.rawQuery(
+ TABLE_REPO + " order by priority", null); "select address, inuse, priority, pubkey, lastetag from "
+ TABLE_REPO + " order by priority", null);
c.moveToFirst(); c.moveToFirst();
while (!c.isAfterLast()) { while (!c.isAfterLast()) {
Repo repo = new Repo(); Repo repo = new Repo();
@ -983,6 +989,7 @@ public class DB {
repo.inuse = (c.getInt(1) == 1); repo.inuse = (c.getInt(1) == 1);
repo.priority = c.getInt(2); repo.priority = c.getInt(2);
repo.pubkey = c.getString(3); repo.pubkey = c.getString(3);
repo.lastetag = c.getString(4);
repos.add(repo); repos.add(repo);
c.moveToNext(); c.moveToNext();
} }
@ -997,7 +1004,7 @@ public class DB {
public void changeServerStatus(String address) { public void changeServerStatus(String address) {
db.execSQL("update " + TABLE_REPO db.execSQL("update " + TABLE_REPO
+ " set inuse=1-inuse where address = ?", + " set inuse=1-inuse, lastetag=null where address = ?",
new String[] { address }); new String[] { address });
} }
@ -1006,6 +1013,14 @@ public class DB {
values.put("inuse", repo.inuse); values.put("inuse", repo.inuse);
values.put("priority", repo.priority); values.put("priority", repo.priority);
values.put("pubkey", repo.pubkey); values.put("pubkey", repo.pubkey);
values.put("lastetag", (String) null);
db.update(TABLE_REPO, values, "address = ?",
new String[] { repo.address });
}
public void writeLastEtag(Repo repo) {
ContentValues values = new ContentValues();
values.put("lastetag", repo.lastetag);
db.update(TABLE_REPO, values, "address = ?", db.update(TABLE_REPO, values, "address = ?",
new String[] { repo.address }); new String[] { repo.address });
} }
@ -1016,6 +1031,7 @@ public class DB {
values.put("inuse", 1); values.put("inuse", 1);
values.put("priority", priority); values.put("priority", priority);
values.put("pubkey", pubkey); values.put("pubkey", pubkey);
values.put("lastetag", (String) null);
db.insert(TABLE_REPO, null, values); db.insert(TABLE_REPO, null, values);
} }

2
src/org/fdroid/fdroid/Downloader.java
View File

@ -117,7 +117,7 @@ public class Downloader extends Thread {
// If we haven't got the apk locally, we'll have to download it... // If we haven't got the apk locally, we'll have to download it...
String remotefile; String remotefile;
if (curapk.apkSource == null) { if (curapk.apkSource == null) {
remotefile = curapk.detail_server + "/" + apkname.replace(" ", "%20"); remotefile = curapk.server + "/" + apkname.replace(" ", "%20");
} else { } else {
remotefile = curapk.apkSource; remotefile = curapk.apkSource;
} }

209
src/org/fdroid/fdroid/RepoXMLHandler.java
View File

@ -28,6 +28,7 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.OutputStream; import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.security.cert.Certificate; import java.security.cert.Certificate;
@ -237,7 +238,7 @@ public class RepoXMLHandler extends DefaultHandler {
} else if (localName == "package" && curapp != null && curapk == null) { } else if (localName == "package" && curapp != null && curapk == null) {
curapk = new DB.Apk(); curapk = new DB.Apk();
curapk.id = curapp.id; curapk.id = curapp.id;
curapk.detail_server = server; curapk.server = server;
hashType = null; hashType = null;
} else if (localName == "hash" && curapk != null) { } else if (localName == "hash" && curapk != null) {
hashType = attributes.getValue("", "type"); hashType = attributes.getValue("", "type");
@ -245,24 +246,44 @@ public class RepoXMLHandler extends DefaultHandler {
curchars.setLength(0); curchars.setLength(0);
} }
// Get a remote file. Returns the HTTP response code.
// If 'etag' is not null, it's passed to the server as an If-None-Match
// header, in which case expect a 304 response if nothing changed.
// In the event of a 200 response ONLY, 'retag' (which should be passed
// empty) may contain an etag value for the response, or it may be left
// empty if none was available.
private static int getRemoteFile(Context ctx, String url, String dest,
String etag, StringBuilder retag) throws MalformedURLException,
IOException {
private static void getRemoteFile(Context ctx, String url, String dest) URL u = new URL(url);
throws MalformedURLException, IOException { HttpURLConnection uc = (HttpURLConnection) u.openConnection();
FileOutputStream f = ctx.openFileOutput(dest, Context.MODE_PRIVATE); if (etag != null)
uc.setRequestProperty("If-None-Match", etag);
int code = uc.getResponseCode();
if (code == 200) {
BufferedInputStream getit = new BufferedInputStream( FileOutputStream f = ctx.openFileOutput(dest, Context.MODE_PRIVATE);
new URL(url).openStream());
BufferedOutputStream bout = new BufferedOutputStream(f, 1024);
byte data[] = new byte[1024];
int readed = getit.read(data, 0, 1024); BufferedInputStream getit = new BufferedInputStream(
while (readed != -1) { new URL(url).openStream());
bout.write(data, 0, readed); BufferedOutputStream bout = new BufferedOutputStream(f, 1024);
readed = getit.read(data, 0, 1024); byte data[] = new byte[1024];
int readed = getit.read(data, 0, 1024);
while (readed != -1) {
bout.write(data, 0, readed);
readed = getit.read(data, 0, 1024);
}
bout.close();
getit.close();
f.close();
String et = uc.getHeaderField("ETag");
if (et != null)
retag.append(et);
} }
bout.close(); return code;
getit.close();
f.close();
} }
@ -271,9 +292,14 @@ public class RepoXMLHandler extends DefaultHandler {
// APKs are merged into the existing one). // APKs are merged into the existing one).
// Returns null if successful, otherwise an error message to be displayed // Returns null if successful, otherwise an error message to be displayed
// to the user (if there is an interactive user!) // to the user (if there is an interactive user!)
public static String doUpdate(Context ctx, DB.Repo repo, Vector<DB.App> apps) { // 'newetag' should be passed empty. On success, it may contain an etag
// value for the index that was successfully processed, or it may contain
// null if none was available.
public static String doUpdate(Context ctx, DB.Repo repo,
Vector<DB.App> apps, StringBuilder newetag, Vector<String> keeprepos) {
try { try {
int code = 0;
if (repo.pubkey != null) { if (repo.pubkey != null) {
// This is a signed repo - we download the jar file, // This is a signed repo - we download the jar file,
@ -286,85 +312,106 @@ public class RepoXMLHandler extends DefaultHandler {
address += "?" + pi.versionName; address += "?" + pi.versionName;
} catch (Exception e) { } catch (Exception e) {
} }
getRemoteFile(ctx, address, "tempindex.jar"); code = getRemoteFile(ctx, address, "tempindex.jar",
String jarpath = ctx.getFilesDir() + "/tempindex.jar"; repo.lastetag, newetag);
JarFile jar; if (code == 200) {
JarEntry je; String jarpath = ctx.getFilesDir() + "/tempindex.jar";
try { JarFile jar;
jar = new JarFile(jarpath, true); JarEntry je;
je = (JarEntry) jar.getEntry("index.xml"); try {
File efile = new File(ctx.getFilesDir(), "/tempindex.xml"); jar = new JarFile(jarpath, true);
InputStream in = new BufferedInputStream( je = (JarEntry) jar.getEntry("index.xml");
jar.getInputStream(je), 8192); File efile = new File(ctx.getFilesDir(),
OutputStream out = new BufferedOutputStream( "/tempindex.xml");
new FileOutputStream(efile), 8192); InputStream in = new BufferedInputStream(
byte[] buffer = new byte[8192]; jar.getInputStream(je), 8192);
while (true) { OutputStream out = new BufferedOutputStream(
int nBytes = in.read(buffer); new FileOutputStream(efile), 8192);
if (nBytes <= 0) byte[] buffer = new byte[8192];
break; while (true) {
out.write(buffer, 0, nBytes); int nBytes = in.read(buffer);
if (nBytes <= 0)
break;
out.write(buffer, 0, nBytes);
}
out.flush();
out.close();
in.close();
} catch (SecurityException e) {
Log.e("FDroid", "Invalid hash for index file");
return "Invalid hash for index file";
} }
out.flush(); Certificate[] certs = je.getCertificates();
out.close(); jar.close();
in.close(); if (certs == null) {
} catch (SecurityException e) { Log.d("FDroid", "No signature found in index");
Log.e("FDroid", "Invalid hash for index file"); return "No signature found in index";
return "Invalid hash for index file"; }
} Log.d("FDroid", "Index has " + certs.length + " signature"
Certificate[] certs = je.getCertificates(); + (certs.length > 1 ? "s." : "."));
jar.close();
if (certs == null) {
Log.d("FDroid", "No signature found in index");
return "No signature found in index";
}
Log.d("FDroid", "Index has " + certs.length + " signature"
+ (certs.length > 1 ? "s." : "."));
boolean match = false; boolean match = false;
for (Certificate cert : certs) { for (Certificate cert : certs) {
String certdata = Hasher.hex(cert.getEncoded()); String certdata = Hasher.hex(cert.getEncoded());
if (repo.pubkey.equals(certdata)) { if (repo.pubkey.equals(certdata)) {
match = true; match = true;
break; break;
}
}
if (!match) {
Log.d("FDroid", "Index signature mismatch");
return "Index signature mismatch";
} }
}
if (!match) {
Log.d("FDroid", "Index signature mismatch");
return "Index signature mismatch";
} }
} else { } else {
// It's an old-fashioned unsigned repo... // It's an old-fashioned unsigned repo...
Log.d("FDroid", "Getting unsigned index from " + repo.address); Log.d("FDroid", "Getting unsigned index from " + repo.address);
getRemoteFile(ctx, repo.address + "/index.xml", "tempindex.xml"); code = getRemoteFile(ctx, repo.address + "/index.xml",
"tempindex.xml", repo.lastetag, newetag);
} }
// Process the index... if (code == 200) {
SAXParserFactory spf = SAXParserFactory.newInstance(); // Process the index...
SAXParser sp = spf.newSAXParser(); SAXParserFactory spf = SAXParserFactory.newInstance();
XMLReader xr = sp.getXMLReader(); SAXParser sp = spf.newSAXParser();
RepoXMLHandler handler = new RepoXMLHandler(repo.address, apps); XMLReader xr = sp.getXMLReader();
xr.setContentHandler(handler); RepoXMLHandler handler = new RepoXMLHandler(repo.address, apps);
xr.setContentHandler(handler);
InputStreamReader isr = new FileReader(new File(ctx.getFilesDir() InputStreamReader isr = new FileReader(new File(
+ "/tempindex.xml")); ctx.getFilesDir() + "/tempindex.xml"));
InputSource is = new InputSource(isr); InputSource is = new InputSource(isr);
xr.parse(is); xr.parse(is);
if (handler.pubkey != null && repo.pubkey == null) { if (handler.pubkey != null && repo.pubkey == null) {
// We read an unsigned index, but that indicates that // We read an unsigned index, but that indicates that
// a signed version is now available... // a signed version is now available...
Log.d("FDroid", Log.d("FDroid",
"Public key found - switching to signed repo for future updates"); "Public key found - switching to signed repo for future updates");
repo.pubkey = handler.pubkey; repo.pubkey = handler.pubkey;
DB db = DB.getDB(); try {
try { DB db = DB.getDB();
db.updateRepoByAddress(repo); db.updateRepoByAddress(repo);
} finally { } finally {
DB.releaseDB(); DB.releaseDB();
}
} }
} else if (code == 304) {
// The index is unchanged since we last read it. We just mark
// everything that came from this repo as being updated.
Log.d("FDroid", "Repo index for " + repo.address
+ " is up to date (by etag)");
keeprepos.add(repo.address);
// Make sure we give back the same etag. (The 200 route will
// have supplied a new one.
newetag.append(repo.lastetag);
} else {
return "Failed to read index - HTTP response "
+ Integer.toString(code);
} }
} catch (SSLHandshakeException sslex) { } catch (SSLHandshakeException sslex) {

66
src/org/fdroid/fdroid/UpdateService.java
View File

@ -102,13 +102,12 @@ public class UpdateService extends IntentService {
boolean notify = prefs.getBoolean("updateNotify", false); boolean notify = prefs.getBoolean("updateNotify", false);
// Grab some preliminary information, then we can release the // Grab some preliminary information, then we can release the
// database // database while we do all the downloading, etc...
// while we do all the downloading, etc...
DB db = DB.getDB();
int prevUpdates = 0; int prevUpdates = 0;
int newUpdates = 0; int newUpdates = 0;
Vector<DB.Repo> repos; Vector<DB.Repo> repos;
try { try {
DB db = DB.getDB();
repos = db.getRepos(); repos = db.getRepos();
} finally { } finally {
DB.releaseDB(); DB.releaseDB();
@ -116,12 +115,16 @@ public class UpdateService extends IntentService {
// Process each repo... // Process each repo...
Vector<DB.App> apps = new Vector<DB.App>(); Vector<DB.App> apps = new Vector<DB.App>();
Vector<String> keeprepos = new Vector<String>();
boolean success = true; boolean success = true;
for (DB.Repo repo : repos) { for (DB.Repo repo : repos) {
if (repo.inuse) { if (repo.inuse) {
StringBuilder newetag = new StringBuilder();
String err = RepoXMLHandler.doUpdate(getBaseContext(), String err = RepoXMLHandler.doUpdate(getBaseContext(),
repo, apps); repo, apps, newetag, keeprepos);
if (err != null) { if (err == null) {
repo.lastetag = newetag.toString();
} else {
success = false; success = false;
err = "Update failed for " + repo.address + " - " + err; err = "Update failed for " + repo.address + " - " + err;
Log.d("FDroid", err); Log.d("FDroid", err);
@ -137,16 +140,55 @@ public class UpdateService extends IntentService {
Vector<DB.App> acceptedapps = new Vector<DB.App>(); Vector<DB.App> acceptedapps = new Vector<DB.App>();
Vector<DB.App> prevapps = ((FDroidApp) getApplication()) Vector<DB.App> prevapps = ((FDroidApp) getApplication())
.getApps(); .getApps();
db = DB.getDB();
DB db = DB.getDB();
try { try {
// Need to flag things we're keeping despite having received
// no data about during the update. (i.e. stuff from a repo
// that we know is unchanged due to the etag)
for (String keep : keeprepos) {
for (DB.App app : prevapps) {
boolean keepapp = false;
for (DB.Apk apk : app.apks) {
if (apk.server.equals(keep)) {
keepapp = true;
break;
}
}
if (keepapp) {
DB.App app_k = null;
for (DB.App app2 : apps) {
if (app2.id.equals(app.id)) {
app_k = app2;
break;
}
}
if (app_k == null) {
apps.add(app);
app_k = app;
}
app_k.updated = true;
if (!app_k.detail_Populated) {
db.populateDetails(app_k, keep);
}
for (DB.Apk apk : app.apks)
if (apk.server.equals(keep))
apk.updated = true;
}
}
}
prevUpdates = db.beginUpdate(prevapps); prevUpdates = db.beginUpdate(prevapps);
for (DB.App app : apps) { for (DB.App app : apps) {
if(db.updateApplication(app)) if (db.updateApplication(app))
acceptedapps.add(app); acceptedapps.add(app);
} }
db.endUpdate(); db.endUpdate();
if (notify) if (notify)
newUpdates = db.getNumUpdates(); newUpdates = db.getNumUpdates();
for (DB.Repo repo : repos)
db.writeLastEtag(repo);
} catch (Exception ex) { } catch (Exception ex) {
db.cancelUpdate(); db.cancelUpdate();
Log.e("FDroid", "Exception during update processing:\n" Log.e("FDroid", "Exception during update processing:\n"
@ -217,7 +259,7 @@ public class UpdateService extends IntentService {
} }
} }
private void getIcon(DB.App app) { private void getIcon(DB.App app) {
try { try {
@ -225,9 +267,9 @@ public class UpdateService extends IntentService {
if (f.exists()) if (f.exists())
return; return;
if(app.apks.size() == 0) if (app.apks.size() == 0)
return; return;
String server = app.apks.get(0).detail_server; String server = app.apks.get(0).server;
URL u = new URL(server + "/icons/" + app.icon); URL u = new URL(server + "/icons/" + app.icon);
HttpURLConnection uc = (HttpURLConnection) u.openConnection(); HttpURLConnection uc = (HttpURLConnection) u.openConnection();
if (uc.getResponseCode() == 200) { if (uc.getResponseCode() == 200) {
@ -251,7 +293,5 @@ public class UpdateService extends IntentService {
} }
} }
} }