From 8d36de816a82e7380409d6e7be9f59d73adbc638 Mon Sep 17 00:00:00 2001 From: Jan-Piet Mens Date: Fri, 4 May 2018 09:34:02 +0200 Subject: [PATCH] Fully support OpenCage as a reverse geocoder closes #234 addresses #202 addresses #233 --- README.md | 14 +++-- doc/OPENCAGE.md | 10 +++- geo.c | 144 ++++++++++++++++++++++++++++++++++++++++++++---- recorder.c | 2 +- udata.h | 2 +- 5 files changed, 155 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 94515c2..25a9264 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,13 @@ This section lists the most important options of the Recorder with their long na `--precision` overrides the compiled-in default. (See "Precision" later.) -`--geokey` sets the Google API key for reverse geo lookups. If you do more than 2500 (currently) reverse-geo requests per day, you'll need an API key for Google's geocoding service. Specify that here. (Note: these limits have changed in May 2018; make sure to check Google's Map Project documentation before using this; we recommend using [OpenCage](doc/OPENCAGE.md) as reverse geo-encoder.) +`--geokey` sets the API key for reverse geo lookups. We support Google (legacy) and OpenCage which we recommend [OpenCage](doc/OPENCAGE.md). You will require an API key for both. For backwards-compatibility the API key for Google is used "as is", whereas you prefix the OpenCage API key with the string `"opencage:"`: +``` +--geokey "opencage:xxxxxxxxxxxxxxxxxxxxxx" # for OpenCage +--geokey "xxxxxxxxxxxxxxxxxxxxxx" # for Google +``` + +(The rules of the game for using Google as reverse geocoder changed in May 2018; make sure to check Google's Map Project documentation before using this) `--debug` enables a bit of additional debugging on stderr. @@ -632,9 +638,9 @@ isotst,vel,addr If not disabled with option `--norevgeo`, the Recorder will attempt to perform a reverse-geo lookup on the location coordinates it obtains. These can be either 1. obtained via a Lua function you define (see [doc/HOOKS.md](doc/HOOKS.md)) -2. obtained via a call to Google +2. obtained via a call to one of the supported reverse geocoders (see `--geokey`) -Lookups performed via Google are stored in an LMDB database. If a lookup is not possible, for example because you're over quota, the service isn't available, etc., Recorder keeps tracks of the coordinates which could *not* be resolved in a file named `missing`: +Results of lookups are stored in an LMDB database. If a lookup is not possible, for example because you're over quota, the service isn't available, etc., Recorder keeps tracks of the coordinates which could *not* be resolved in a file named `missing`: ``` $ cat store/ghash/missing @@ -663,7 +669,7 @@ and a precision of 2 would mean that a very large part of France resolves to a s ![geohash2](assets/geohash-2.png) -The bottom line: if you run the Recorder with just a few devices and want to know quite exactly where you've been, use a high precision (7 is probably good). If you, on the other hand, run Recorder with many devices and are only interested in where a device was approximately, lower the precision; this also has the effect that fewer reverse-geo lookups will be performed in the Google infrastructure. (Also: respect their quotas!) +The bottom line: if you run the Recorder with just a few devices and want to know quite exactly where you've been, use a high precision (7 is probably good). If you, on the other hand, run Recorder with many devices and are only interested in where a device was approximately, lower the precision; this also has the effect that fewer reverse-geo lookups will be performed in the geocoding service infrastructure. (Also: respect their quotas!) ### The geo cache diff --git a/doc/OPENCAGE.md b/doc/OPENCAGE.md index d123675..1e11505 100644 --- a/doc/OPENCAGE.md +++ b/doc/OPENCAGE.md @@ -2,7 +2,15 @@ We now (since May 2018) recommend using [OpenCage](https://geocoder.opencagedata.com) as reverse geo-coding provider: their [pricing](https://geocoder.opencagedata.com/pricing) is attractive and they currently offer a free tier which allows up to 2,500 requests per day. -In order to use OpenCage with the Recorder, proceed as follows: +Use the OpenCage API in Recorder simply by setting the `--geokey` option to the string `"opencage:"` with your API key concatenated to it. (Without the substring `opencage:` the Recorder falls back to using Google in order to maintain backwards-compatibility.) + +``` +--geokey "opencage:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +``` + +Be aware that the Recorder uses the following settings: `no_record=1&limit=1`. OpenCage documents the first as meaning it will not log the request, and that protects your privacy. + +In order to use OpenCage with the Recorder using Lua, proceed as follows: 1. Make sure you've built the Recorder with support for Lua 1. Install the required Lua modules: diff --git a/geo.c b/geo.c index 3405a39..acc016e 100644 --- a/geo.c +++ b/geo.c @@ -20,13 +20,21 @@ #include #include #include +#include #include #include "utstring.h" #include "geo.h" #include "json.h" #include "util.h" -#define GURL "%s://maps.googleapis.com/maps/api/geocode/json?latlng=%lf,%lf&sensor=false&language=EN" +typedef enum { + GOOGLE, + OPENCAGE +} geocoder; + +#define GOOGLE_URL "https://maps.googleapis.com/maps/api/geocode/json?latlng=%lf,%lf&sensor=false&language=EN&key=%s" + +#define OPENCAGE_URL "https://api.opencagedata.com/geocode/v1/json?q=%lf+%lf&key=%s&abbrv=1&no_record=1&limit=1&format=json" static CURL *curl; @@ -131,15 +139,113 @@ static int goog_decode(UT_string *geodata, UT_string *addr, UT_string *cc, UT_st return (1); } +static int opencage_decode(UT_string *geodata, UT_string *addr, UT_string *cc, UT_string *locality) +{ + JsonNode *json, *results, *address, *ac, *zeroth; + + /* + * We are parsing this. I want the formatted in `addr' and + * the country code short_name in `cc' + * + * { + * "documentation": "https://geocoder.opencagedata.com/api", + * "licenses": [ + * { + * "name": "CC-BY-SA", + * "url": "http://creativecommons.org/licenses/by-sa/3.0/" + * }, + * { + * "name": "ODbL", + * "url": "http://opendatacommons.org/licenses/odbl/summary/" + * } + * ], + * "rate": { + * "limit": 2500, + * "remaining": 2495, + * "reset": 1525392000 + * }, + * "results": [ + * { + * ... + * "components": { + * "city": "Sablonnières", + * "country": "France", + * "country_code": "fr", + * "place": "La Terre Noire", + * }, + * "formatted": "La Terre Noire, 77510 Sablonnières, France", + */ + + if ((json = json_decode(UB(geodata))) == NULL) { + return (0); + } + + if ((results = json_find_member(json, "results")) != NULL) { + if ((zeroth = json_find_element(results, 0)) != NULL) { + address = json_find_member(zeroth, "formatted"); + if ((address != NULL) && (address->tag == JSON_STRING)) { + utstring_printf(addr, "%s", address->string_); + } + } + + if ((ac = json_find_member(zeroth, "components")) != NULL) { + + /* + * { + * "ISO_3166-1_alpha-2": "FR", + * "_type": "place", + * "city": "Sablonnières", + * "country": "France", + * "country_code": "fr", + * "county": "Seine-et-Marne", + * "place": "La Terre Noire", + * "political_union": "European Union", + * "postcode": "77510", + * "state": "Île-de-France" + * } + */ + + JsonNode *j; + int have_cc = 0, have_locality = 0; + + if ((j = json_find_member(ac, "country_code")) != NULL) { + if (j->tag == JSON_STRING) { + char *bp = j->string_; + int ch; + + while (*bp) { + ch = (islower(*bp)) ? toupper(*bp) : *bp; + utstring_printf(cc, "%c", ch); + ++bp; + } + have_cc = 1; + } + } + + if ((j = json_find_member(ac, "city")) != NULL) { + if (j->tag == JSON_STRING) { + utstring_printf(locality, "%s", j->string_); + have_locality = 1; + } + } + } + } + + json_delete(json); + return (1); +} + JsonNode *revgeo(struct udata *ud, double lat, double lon, UT_string *addr, UT_string *cc) { static UT_string *url; static UT_string *cbuf; /* Buffer for curl GET */ static UT_string *locality = NULL; + long http_code; CURLcode res; int rc; JsonNode *geo; time_t now; + geocoder geocoder; if ((geo = json_mkobject()) == NULL) { return (NULL); @@ -155,13 +261,22 @@ JsonNode *revgeo(struct udata *ud, double lat, double lon, UT_string *addr, UT_s utstring_renew(cbuf); utstring_renew(locality); - if (ud && ud->geokey) { - utstring_printf(url, GURL, "https", lat, lon); - utstring_printf(url, "&key=%s", ud->geokey); - } else { - utstring_printf(url, GURL, "http", lat, lon); + if (!ud->geokey || !*ud->geokey) { + utstring_printf(addr, "Unknown (%lf,%lf)", lat, lon); + utstring_printf(cc, "__"); + return (geo); } + if (strncmp(ud->geokey, "opencage:", strlen("opencage:")) == 0) { + utstring_printf(url, OPENCAGE_URL, lat, lon, ud->geokey + strlen("opencage:")); + geocoder = OPENCAGE; + } else { + utstring_printf(url, GOOGLE_URL, lat, lon, ud->geokey); + geocoder = GOOGLE; + } + + // printf("--------------- %s\n", UB(url)); + curl_easy_setopt(curl, CURLOPT_URL, UB(url)); curl_easy_setopt(curl, CURLOPT_USERAGENT, "OwnTracks-Recorder/1.0"); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); @@ -172,8 +287,10 @@ JsonNode *revgeo(struct udata *ud, double lat, double lon, UT_string *addr, UT_s curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)cbuf); res = curl_easy_perform(curl); - if (res != CURLE_OK) { - utstring_printf(addr, "revgeo failed for (%lf,%lf)", lat, lon); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + + if (res != CURLE_OK || http_code != 200) { + utstring_printf(addr, "revgeo failed for (%lf,%lf): HTTP status_code==%ld", lat, lon, http_code); utstring_printf(cc, "__"); fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); @@ -181,9 +298,16 @@ JsonNode *revgeo(struct udata *ud, double lat, double lon, UT_string *addr, UT_s return (NULL); } - // printf("%s\n", UB(url)); + switch (geocoder) { + case GOOGLE: + rc = goog_decode(cbuf, addr, cc, locality); + break; + case OPENCAGE: + rc = opencage_decode(cbuf, addr, cc, locality); + break; + } - if (!(rc = goog_decode(cbuf, addr, cc, locality))) { + if (!rc) { json_delete(geo); return (NULL); } diff --git a/recorder.c b/recorder.c index 027d637..87c3af2 100644 --- a/recorder.c +++ b/recorder.c @@ -1168,7 +1168,7 @@ void usage(char *prog) #endif printf(" --precision ghash precision (dflt: %d)\n", GHASHPREC); printf(" --norec don't maintain REC files\n"); - printf(" --geokey optional Google reverse-geo API key\n"); + printf(" --geokey optional reverse-geo API key\n"); printf(" --debug additional debugging\n"); printf("\n"); printf("Options override these environment variables:\n"); diff --git a/udata.h b/udata.h index 08502b3..43dfa32 100644 --- a/udata.h +++ b/udata.h @@ -51,7 +51,7 @@ struct udata { struct gcache *keydb; /* encryption keys */ #endif char *label; /* Server label */ - char *geokey; /* Google reverse-geo API key */ + char *geokey; /* reverse-geo API key */ int debug; /* enable for debugging */ struct gcache *httpfriends; /* lmdb named database 'friends' */ struct gcache *wpdb; /* lmdb named database 'wp' (waypoints) */