1st pass at ocat; unfinished by far!

Makefile

@@ -9,7 +9,7 @@ ifneq ($(HAVE_REDIS),no)
 endif
 
 
-all: ot-recorder #### ot-reader
+all: ot-recorder ocat
 
 ot-reader: ot-reader.c json.o utstring.h ghash.o mkpath.o jget.o
 	$(CC) $(CFLAGS) ot-reader.c -o ot-reader json.o ghash.o mkpath.o jget.o $(LIBS)
@@ -26,6 +26,12 @@ safewrite.o: safewrite.h safewrite.c
 jget.o: jget.c jget.h json.h Makefile config.mk
 misc.o: misc.c misc.h udata.h Makefile config.mk
 
+ocat: ocat.o storage.o json.o geohash.o ghash.o mkpath.o
+	$(CC) $(CFLAGS) -o ocat ocat.o storage.o json.o geohash.o ghash.o mkpath.o
+
+ocat.o: ocat.c storage.h
+storage.o: storage.c storage.h config.h
+
 clean:
 	rm -f *.o
 clobber: clean

ocat.c

@@ -0,0 +1,143 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include <time.h>
+#include "json.h"
+#include "storage.h"
+
+static char *tstampyyyymm(time_t t) {
+        static char buf[] = "YYYY-MM";
+
+        strftime(buf, sizeof(buf), "%Y-%m", gmtime(&t));
+        return(buf);
+}
+
+void usage(char *prog)
+{
+	printf("Usage: %s [options..] [file ...]\n", prog);
+	printf("  --help		-h	this message\n");
+	printf("  --list		-l	list users (or a user's (-u) devices\n");
+	printf("  --user username	-u	specify username\n");
+	printf("  --device devicename   -d	specify device name\n");
+	printf("  --yyyymm YYYY-MM	-D	specify year-month (shell pat, eg: '2015-0[572]')\n");
+	exit(1);
+}
+
+int main(int argc, char **argv)
+{
+	char *progname = *argv;
+	int c;
+	int list = 0;
+	char *username = NULL, *device = NULL, *yyyymm = NULL;
+	JsonNode *json, *obj, *locs;
+	time_t now;
+
+	time(&now);
+	yyyymm = tstampyyyymm(now);
+
+	while (1) {
+		static struct option long_options[] = {
+			{ "help",	no_argument,	0, 	'h'},
+			{ "list",	no_argument,	0, 	'l'},
+			{ "user",	required_argument, 0, 	'u'},
+			{ "device",	required_argument, 0, 	'd'},
+			{ "yyyymm",	required_argument, 0, 	'D'},
+		  	{0, 0, 0, 0}
+		  };
+		int optindex = 0;
+
+		c = getopt_long(argc, argv, "hlu:d:D:", long_options, &optindex);
+		if (c == -1)
+			break;
+
+		switch (c) {
+			case 'l':
+				list = 1;
+				break;
+			case 'u':
+				username = strdup(optarg);
+				break;
+			case 'd':
+				device = strdup(optarg);
+				break;
+			case 'D':
+				yyyymm = strdup(optarg);
+				break;
+			case 'h':
+			case '?':
+				/* getopt_long already printed message */
+				usage(progname);
+				break;
+			default:
+				abort();
+		}
+
+	}
+
+	argc -= (optind);
+	argv += (optind);
+
+	if (!username && device) {
+		fprintf(stderr, "%s: device name without username doesn't make sense\n", progname);
+		return (-2);
+	}
+
+	if (list) {
+		char *js;
+
+		json = lister(username, device, yyyymm);
+		if (json == NULL) {
+			fprintf(stderr, "%s: cannot list\n", progname);
+			exit(2);
+		}
+		js = json_stringify(json, " ");
+		printf("%s\n", js);
+		json_delete(json);
+		free(js);
+	}
+
+	if (argc == 0 && !username && !device) {
+		fprintf(stderr, "%s: nothing to do. Specify filename or --user and --device\n", progname);
+		return (-1);
+	} else if (username && device && (argc > 0)) {
+		fprintf(stderr, "%s: filename with --user and --device is not supported\n", progname);
+		return (-1);
+	}
+	
+
+	/*
+	 * If any files were passed on the command line, we assume these are *.rec
+	 * and process those. Otherwise, we expect a `user' and `device' and process
+	 * "today"
+	 */
+
+	obj = json_mkobject();
+	locs = json_mkarray();
+
+
+	if (argc) {
+		int n;
+
+		for (n = 0; n < argc; n++) {
+			locations(argv[n], obj, locs);
+		}
+	} else {
+		JsonNode *arr, *f;
+
+		if ((json = lister(username, device, yyyymm)) != NULL) {
+			if ((arr = json_find_member(json, "results")) != NULL) { // get array
+				json_foreach(f, arr) {
+					printf("%s\n", f->string_);
+					locations(f->string_, obj, locs);
+				}
+			}
+			json_delete(json);
+		}
+	}
+
+	json_append_member(obj, "locations", locs);
+	printf("%s\n", json_stringify(obj, " "));
+
+	return (0);
+}

storage.c

@@ -0,0 +1,221 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dirent.h>
+#include <glob.h>
+#include "utstring.h"
+#include "config.h"
+#include "storage.h"
+#include "geohash.h"
+
+#include "udata.h"
+
+int ghash_readcache(struct udata *ud, char *ghash, UT_string *addr, UT_string *cc);
+
+void get_geo(JsonNode *o, char *ghash)
+{
+	static UT_string *addr = NULL, *cc = NULL;
+	static struct udata udata;
+
+	udata.usefiles = 1;
+
+	utstring_renew(addr);
+	utstring_renew(cc);
+
+	if (ghash_readcache(&udata, ghash, addr, cc) == 1) {
+		json_append_member(o, "addr", json_mkstring(utstring_body(addr)));
+		json_append_member(o, "cc", json_mkstring(utstring_body(cc)));
+	}
+}
+
+/*
+ * List the directories in the directory at `path' and put
+ * the names into a JSON array which is in obj.
+ */
+
+static void ls(char *path, JsonNode *obj)
+{
+        DIR *dirp;
+        struct dirent *dp;
+	JsonNode *jarr = json_mkarray();
+
+	printf("opendir %s\n", path);
+        if ((dirp = opendir(path)) == NULL) {
+		json_append_member(obj, "error", json_mkstring("Cannot open requested directory"));
+                return;
+        }
+
+        while ((dp = readdir(dirp)) != NULL) {
+                if ((*dp->d_name != '.') && (dp->d_type == DT_DIR)) {
+                        // char *s = strdup(dp->d_name);
+			json_append_element(jarr, json_mkstring(dp->d_name));
+                }
+        }
+
+	json_append_member(obj, "results", jarr);
+        closedir(dirp);
+}
+
+/*
+ * List the files (glob pattern) in the directory at `pathpat' and
+ * put the names into a JSON array in obj.
+ */
+
+static void lsglob(char *pathpat, JsonNode *obj)
+{
+        glob_t paths;
+        int flags = GLOB_NOCHECK;
+        char **p;
+	JsonNode *jarr = json_mkarray();
+
+        if (glob(pathpat, flags, NULL, &paths) != 0) {
+		json_append_member(obj, "error", json_mkstring("Cannot glob requested pattern"));
+                return;
+	}
+
+	/* Ensure we add paths only if the glob actuall expanded */
+	if (paths.gl_matchc > 0) {
+		for (p = paths.gl_pathv; *p != NULL; p++) {
+			// char *s = strdup(*p);
+			// utarray_push_back(dirs, &s);
+			json_append_element(jarr, json_mkstring(*p));
+		}
+        }
+        globfree(&paths);
+
+	json_append_member(obj, "results", jarr);
+        return;
+}
+
+/*
+ * If `user' and `device' are both NULL, return list of users.
+ * If `user` is specified, and device is NULL, return user's devices
+ * If both user and device are specified, return list of .rec files;
+ * in that case, limit with `yyyymm' which is YYYY-MM
+ */
+
+JsonNode *lister(char *user, char *device, char *yyyymm)
+{
+	JsonNode *json = json_mkobject();
+	UT_string *path = NULL;
+
+	utstring_renew(path);
+
+	/* FIXME: lowercase user/device */
+
+	if (!user && !device) {
+		utstring_printf(path, "%s/rec", STORAGEDIR);
+		ls(utstring_body(path), json);
+	} else if (!device) {
+		utstring_printf(path, "%s/rec/%s", STORAGEDIR, user);
+		ls(utstring_body(path), json);
+	} else {
+		utstring_printf(path, "%s/rec/%s/%s/%s.rec",
+			STORAGEDIR, user, device,
+			(yyyymm) ? yyyymm : "*");
+		lsglob(utstring_body(path), json);
+	}
+
+	return (json);
+}
+
+/*
+ * Read the file at `filename' (- is stdin) and store location
+ * objects at the JSON array `arr`. `obj' is a JSON object which
+ * contains `arr'.
+ */
+
+void locations(char *filename, JsonNode *obj, JsonNode *arr)
+{
+	JsonNode *o, *json, *j;
+	FILE *fp;
+	int doclose;
+	char buf[BUFSIZ], **element;
+	long counter = 0L;
+	static char *numbers[] = { "lat", "lon", "batt", "vel", "cog", "tst", "alt", "dist", "trip", NULL };
+	static char *strings[] = { "tid", "t", NULL };
+
+
+	if (!strcmp(filename, "-")) {
+		fp = stdin;
+		doclose = 0;
+	} else {
+		if ((fp = fopen(filename, "r")) == NULL) {
+			perror(filename);
+			return;
+		}
+		doclose = 1;
+	}
+
+	/* Initialize our counter to what the JSON obj currently has */
+
+	if ((j = json_find_member(obj, "count")) != NULL) {
+		counter = j->number_;
+		json_delete(j);
+	}
+
+
+	while (fgets(buf, sizeof(buf)-1, fp) != NULL) {
+		char *bp, *ghash;
+		double lat, lon;
+
+		if ((bp = strstr(buf, "Z\t* ")) != NULL) {
+			if ((bp = strrchr(bp, '\t')) == NULL) {
+				continue;
+			}
+			++counter;
+			json = json_decode(bp + 1);
+			if (json == NULL) {
+				puts("Cannot decode JSON");
+				continue;
+			}
+
+			o = json_mkobject();
+
+			/* Start adding stuff to the object, then copy over the elements
+			 * from the decoded original locations. */
+
+			for (element = numbers; element && *element; element++) {
+				if ((j = json_find_member(json, *element)) != NULL) {
+					if (j->tag == JSON_NUMBER) {
+						json_append_member(o, *element, json_mknumber(j->number_));
+					} else {
+						double d = atof(j->string_);
+						json_append_member(o, *element, json_mknumber(d));
+					}
+				}
+			}
+
+			for (element = strings; element && *element; element++) {
+				if ((j = json_find_member(json, *element)) != NULL) {
+					json_append_member(o, *element, json_mkstring(j->string_));
+				}
+			}
+
+			lat = lon = 0.0;
+			if ((j = json_find_member(o, "lat")) != NULL) {
+				lat = j->number_;
+			}
+			if ((j = json_find_member(o, "lon")) != NULL) {
+				lon = j->number_;
+			}
+
+			ghash = geohash_encode(lat, lon, GEOHASH_PREC);
+			json_append_member(o, "ghash", json_mkstring(ghash));
+
+			get_geo(o, ghash);
+
+
+			json_append_element(arr, o);
+		}
+	}
+
+	/* Add the counter back into `obj' */
+
+	json_append_member(obj, "count", json_mknumber(counter));
+
+
+	if (doclose)
+		fclose(fp);
+
+}

storage.h

@@ -0,0 +1,10 @@
+#ifndef _STORAGE_H_INCL_
+# define _STORAGE_H_INCL_
+
+#include "json.h"
+
+
+JsonNode *lister(char *username, char *device, char *yyyymm);
+void locations(char *filename, JsonNode *obj, JsonNode *arr);
+
+#endif