From c60d2ec3e3ea09b344e42f9671fefb46eb8ec334 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Tue, 24 Apr 2018 10:37:31 -0700 Subject: use libdivecomputer 'fingerprint' to avoid downloading extra data This opportunistically uses a cache of 'fingerprints' for already downloaded dives. As we download data from a dive computer, we save the fingerprint and dive ID of the most recent dive in a per-divecopmputer fingerprint cache file. The next time we download from that dive computer, we will load the cache file for that dive computer if it exists, verify that we still have the dive that is referenced in that cachefile, and if so use the fingerprint to let libdivecomputer potentially stop downloading dives early. This doesn't much matter for most dive computers, but some (like the Scubapro G2) are not able to download one dive at a time, and need the fingerprint to avoid doing a full dump. That is particularly noticeable over bluetooth, where a full dump can be very slow. NOTE! The fingerprint cache is a separate entity from the dive log itself. Unlike the dive log, it doesn't synchronize over the cloud, so if you download using different clients (say, your phone and your laptop), the fingerprint cache entries are per device. So you may still end up downloading dives you already have, because the fingerprint code basically only works to avoid duplicate downloads on the same installation. Also, note that we only have a cache of one single entry per dive computer and downloader, so if you download dives and then don't save the end result, the fingerprint will now point to a dive that you don't actually have in your dive list. As a result, next time you download, the fingerprint won't match any existing dive, and we'll resort to the old non-optimized behavior. Signed-off-by: Linus Torvalds --- core/libdivecomputer.c | 146 +++++++++++++++++++++++++++++++++++++++++++++++++ core/libdivecomputer.h | 2 + 2 files changed, 148 insertions(+) (limited to 'core') diff --git a/core/libdivecomputer.c b/core/libdivecomputer.c index c547a85b2..990904878 100644 --- a/core/libdivecomputer.c +++ b/core/libdivecomputer.c @@ -8,6 +8,9 @@ #include #include #include +#include +#include +#include #include "gettext.h" #include "dive.h" #include "device.h" @@ -21,6 +24,9 @@ #include "libdivecomputer.h" #include "core/version.h" +#include "core/qthelper.h" +#include "core/membuffer.h" +#include "core/file.h" // // If we have an old libdivecomputer, it doesn't @@ -785,6 +791,16 @@ static int dive_cb(const unsigned char *data, unsigned int size, dive->dc.model = strdup(devdata->model); dive->dc.diveid = calculate_diveid(fingerprint, fsize); + /* Should we add it to the cached fingerprint file? */ + if (fingerprint && fsize && !devdata->fingerprint) { + devdata->fingerprint = calloc(fsize, 1); + if (devdata->fingerprint) { + devdata->fsize = fsize; + devdata->fdiveid = dive->dc.diveid; + memcpy(devdata->fingerprint, fingerprint, fsize); + } + } + // Parse the dive's header data rc = libdc_header_parser (parser, devdata, dive); if (rc != DC_STATUS_SUCCESS) { @@ -924,6 +940,121 @@ static unsigned int fixup_suunto_versions(device_data_t *devdata, const dc_event return serial; } +#ifndef O_BINARY + #define O_BINARY 0 +#endif +static void do_save_fingerprint(device_data_t *devdata, const char *tmp, const char *final) +{ + int fd, written = -1; + + fd = subsurface_open(tmp, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); + if (fd < 0) + return; + + /* The fingerprint itself.. */ + written = write(fd, devdata->fingerprint, devdata->fsize); + + /* ..followed by the dive ID of the fingerprinted dive */ + if (write(fd, &devdata->fdiveid, 4) != 4) + written = -1; + + /* I'd like to do fsync() here too, but does Windows support it? */ + if (close(fd) < 0) + written = -1; + + if (written == devdata->fsize) { + if (!subsurface_rename(tmp, final)) + return; + } + unlink(tmp); +} + +/* + * Save the fingerprint after a successful download + */ +static void save_fingerprint(device_data_t *devdata) +{ + char *dir, *tmp, *final; + + if (!devdata->fingerprint) + return; + + dir = format_string("%s/fingerprints", system_default_directory()); + subsurface_mkdir(dir); + tmp = format_string("%s/%04x.tmp", dir, devdata->deviceid); + final = format_string("%s/%04x", dir, devdata->deviceid); + free(dir); + + do_save_fingerprint(devdata, tmp, final); + free(tmp); + free(final); + free(devdata->fingerprint); + devdata->fingerprint = NULL; +} + +static int has_dive(unsigned int deviceid, unsigned int diveid) +{ + int i; + struct dive *dive; + + for_each_dive (i, dive) { + struct divecomputer *dc; + + for_each_dc (dive, dc) { + if (dc->deviceid != deviceid) + continue; + if (dc->diveid != diveid) + continue; + return 1; + } + } + return 0; +} + +/* + * The fingerprint cache files contain the actual libdivecomputer + * fingerprint, followed by 4 bytes of diveid data. Before we use + * the fingerprint data, verify that we actually do have that + * fingerprinted dive. + */ +static void verify_fingerprint(dc_device_t *device, device_data_t *devdata, const unsigned char *buffer, size_t size) +{ + unsigned int diveid, deviceid; + + if (size <= 4) + return; + size -= 4; + + /* Get the dive ID from the end of the fingerprint cache file.. */ + memcpy(&diveid, buffer + size, 4); + /* .. and the device ID from the device data */ + deviceid = devdata->deviceid; + + /* Only use it if we *have* that dive! */ + if (has_dive(deviceid, diveid)) + dc_device_set_fingerprint(device, buffer, size); +} + +/* + * Look up the fingerprint from the fingerprint caches, and + * give it to libdivecomputer to avoid downloading already + * downloaded dives. + */ +static void lookup_fingerprint(dc_device_t *device, device_data_t *devdata) +{ + char *cachename; + struct memblock mem; + + if (devdata->force_download) + return; + cachename = format_string("%s/fingerprints/%04x", + system_default_directory(), devdata->deviceid); + if (readfile(cachename, &mem) > 0) { + verify_fingerprint(device, devdata, mem.buffer, mem.size); + free(mem.buffer); + } + free(cachename); +} static void event_cb(dc_device_t *device, dc_event_type_t event, const void *data, void *userdata) { @@ -964,6 +1095,7 @@ static void event_cb(dc_device_t *device, dc_event_type_t event, const void *dat devinfo->firmware, devinfo->firmware, devinfo->serial, devinfo->serial); } + /* * libdivecomputer doesn't give serial numbers in the proper string form, * so we have to see if we can do some vendor-specific munging. @@ -977,6 +1109,9 @@ static void event_cb(dc_device_t *device, dc_event_type_t event, const void *dat * DC_FIELD_STRING interface instead */ devdata->libdc_serial = devinfo->serial; devdata->libdc_firmware = devinfo->firmware; + + lookup_fingerprint(device, devdata); + break; case DC_EVENT_CLOCK: dev_info(devdata, translate("gettextFromC", "Event: systime=%" PRId64 ", devtime=%u\n"), @@ -1159,6 +1294,8 @@ const char *do_libdivecomputer_import(device_data_t *data) data->device = NULL; data->context = NULL; data->iostream = NULL; + data->fingerprint = NULL; + data->fsize = 0; if (data->libdc_log && logfile_name) fp = subsurface_fopen(logfile_name, "w"); @@ -1207,6 +1344,15 @@ const char *do_libdivecomputer_import(device_data_t *data) fclose(fp); } + /* + * Note that we save the fingerprint unconditionally. + * This is ok because we only have fingerprint data if + * we got a dive header, and because we will use the + * dive id to verify that we actually have the dive + * it refers to before we use the fingerprint data. + */ + save_fingerprint(data); + return err; } diff --git a/core/libdivecomputer.h b/core/libdivecomputer.h index 1ff9f0fa0..e3822aa7b 100644 --- a/core/libdivecomputer.h +++ b/core/libdivecomputer.h @@ -25,6 +25,8 @@ typedef struct dc_user_device_t dc_descriptor_t *descriptor; const char *vendor, *product, *devname; const char *model; + unsigned char *fingerprint; + unsigned int fsize, fdiveid; uint32_t libdc_firmware, libdc_serial; uint32_t deviceid, diveid; dc_device_t *device; -- cgit v1.2.3-70-g09d2