From bfdae39eaa3b463c9ff07c32098739e725546aa7 Mon Sep 17 00:00:00 2001 From: Jan-Piet Mens Date: Sun, 30 Oct 2016 19:48:31 +0100 Subject: [PATCH] REC files now use tst from JSON payload as timestamp note that messages which do not contain a tst (e.g CARD) will be stored with now (time(0)) the tst in the location message payload is also used for the REC filename closes #133 --- doc/STORE.md | 2 +- recorder.c | 23 ++++++++++++++--------- util.c | 6 ++---- util.h | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/doc/STORE.md b/doc/STORE.md index fe42a20..dc4e9a0 100644 --- a/doc/STORE.md +++ b/doc/STORE.md @@ -9,7 +9,7 @@ As mentioned earlier, data is stored in files, and these files are relative to ` * `monitor` a file which contains a timestamp and the last received topic (see Monitoring below). * `msg/` contains messages received by the Messaging system. * `photos/` optional; contains the binary photos from a card. -* `rec/` the Recorder data proper. One subdirectory per user, one subdirectory therein per device. Data files are named `YYYY-MM.rec` (e.g. `2015-08.rec` for the data accumulated during the month of August 2015. +* `rec/` the Recorder data proper. One subdirectory per user, one subdirectory therein per device. Data files are named `YYYY-MM.rec` (e.g. `2015-08.rec` for the data accumulated during the month of August 2015. The content is a time stamp obtained from `tst` (or _now_, i.e. `time(0)` if there is no `tst` in the payload) followed by record type and message payload. * `waypoints/` contains a directory per user and device. Therein are individual files named by a timestamp with the JSON payload of published (i.e. shared) waypoints. The file names are timestamps because the `tst` of a waypoint is its key. If a user publishes all waypoints from a device (Publish Waypoints), the payload is stored in this directory as `username-device.otrw`. (Note, that this is the JSON [waypoints import format](http://owntracks.org/booklet/tech/json/#_typewaypoints).) You can use this `.otrw` file to restore the waypoints on your device by copying to the device and opening it in OwnTracks. You should definitely **not** modify or touch these files: they remain under the control of the Recorder. You can of course, remove old `.rec` files if they consume too much space. diff --git a/recorder.c b/recorder.c index 59ed26f..8fe9c31 100644 --- a/recorder.c +++ b/recorder.c @@ -111,7 +111,7 @@ int do_info(void *userdata, UT_string *username, UT_string *device, JsonNode *js /* I know the payload is valid JSON: write card */ - if ((fp = pathn("wb", "cards", username, NULL, "json")) != NULL) { + if ((fp = pathn("wb", "cards", username, NULL, "json", time(0))) != NULL) { char *js = json_stringify(json, NULL); if (js) { fprintf(fp, "%s\n", js); @@ -143,7 +143,7 @@ int do_info(void *userdata, UT_string *username, UT_string *device, JsonNode *js /* We have a base64-encoded "face". Decode it and store binary image */ if ((img = base64_decode(UB(face), &imglen)) != NULL) { - if ((fp = pathn("wb", "photos", username, NULL, "png")) != NULL) { + if ((fp = pathn("wb", "photos", username, NULL, "png", time(0))) != NULL) { fwrite(img, sizeof(char), imglen, fp); fclose(fp); } @@ -278,10 +278,11 @@ JsonNode *csv_to_json(char *payload) /* * Store payload in REC file unless our Lua putrec() function says - * we shouldn't for this particular user/device combo. + * we shouldn't for this particular user/device combo. Use the epoch + * time to construct path name and "key" */ -static void putrec(struct udata *ud, time_t now, UT_string *reltopic, UT_string *username, UT_string *device, char *string) +static void putrec(struct udata *ud, time_t epoch, UT_string *reltopic, UT_string *username, UT_string *device, char *string) { FILE *fp; int rc = 0; @@ -294,13 +295,13 @@ static void putrec(struct udata *ud, time_t now, UT_string *reltopic, UT_string #endif if (rc == 0) { - if ((fp = pathn("a", "rec", username, device, "rec")) == NULL) { + if ((fp = pathn("a", "rec", username, device, "rec", epoch)) == NULL) { olog(LOG_ERR, "Cannot write REC for %s/%s: %m", UB(username), UB(device)); return; } - fprintf(fp, RECFORMAT, isotime(now), + fprintf(fp, RECFORMAT, isotime(epoch), UB(reltopic), string); fclose(fp); } @@ -541,7 +542,7 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle static UT_string *basetopic = NULL, *username = NULL, *device = NULL, *addr = NULL, *cc = NULL, *ghash = NULL, *ts = NULL; static UT_string *reltopic = NULL, *filename = NULL; char *jsonstring, *_typestr = NULL; - time_t now; + time_t now, epoch; int pingping = FALSE, skipslash = 0, geoprec = geohash_prec(); int r_ok = TRUE; /* True if recording enabled for a publish */ payload_type _type; @@ -657,7 +658,8 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle if ((json = json_decode(payload)) == NULL) { if ((json = csv_to_json(payload)) == NULL) { - /* It's not JSON or it's not a location CSV; store it */ + /* It's not JSON or it's not a location CSV; store it using + * now as time -- we have no other */ putrec(ud, now, reltopic, username, device, bindump(payload, payloadlen)); return; } @@ -931,7 +933,10 @@ void handle_message(void *userdata, char *topic, char *payload, size_t payloadle if (!pingping) { if ((jsonstring = json_stringify(json, NULL)) != NULL) { - putrec(ud, now, reltopic, username, device, jsonstring); + double d_epoch = number(json, "tst"); + + epoch = (isnan(d_epoch)) ? now : d_epoch; + putrec(ud, epoch, reltopic, username, device, jsonstring); free(jsonstring); } } diff --git a/util.c b/util.c index a0392ce..949fd6e 100644 --- a/util.c +++ b/util.c @@ -468,10 +468,9 @@ static void ut_clean(UT_string *us) creating directories on the fly. If device is NULL, omit it. */ -FILE *pathn(char *mode, char *prefix, UT_string *user, UT_string *device, char *suffix) +FILE *pathn(char *mode, char *prefix, UT_string *user, UT_string *device, char *suffix, time_t epoch) { static UT_string *path = NULL; - time_t now; utstring_renew(path); @@ -503,8 +502,7 @@ FILE *pathn(char *mode, char *prefix, UT_string *user, UT_string *device, char * #endif if (strcmp(prefix, "rec") == 0) { - time(&now); - utstring_printf(path, "/%s.%s", yyyymm(now), suffix); + utstring_printf(path, "/%s.%s", yyyymm(epoch), suffix); } else { utstring_printf(path, "/%s.%s", UB(user), suffix); diff --git a/util.h b/util.h index e4b786e..9245759 100644 --- a/util.h +++ b/util.h @@ -29,7 +29,7 @@ int syslog_facility_code(char *facility); const char *yyyymm(time_t t); int tac(char *filename, long lines, int (*func)(char *, void *), void *param); int cat(char *filename, int (*func)(char *, void *), void *param); -FILE *pathn(char *mode, char *prefix, UT_string *user, UT_string *device, char *suffix); +FILE *pathn(char *mode, char *prefix, UT_string *user, UT_string *device, char *suffix, time_t epoch); int safewrite(char *filename, char *buf); void olog(int level, char *fmt, ...); void geohash_setprec(int precision);