geocode: add support for openstreetmap geocoding api

[nor...@perkeep.org]

https://github.com/perkeep/perkeep/commit/5af672029ec2999bb9dc4d9ddd2878a6c6f827ab

commit 5af672029ec2999bb9dc4d9ddd2878a6c6f827ab
Author: Will Norris <wi...@willnorris.com>
Date: Tue May 3 16:29:24 2022 -0700

geocode: add support for openstreetmap geocoding api

diff --git a/doc/geocoding.md b/doc/geocoding.md
index 27f9f86..a4b89e8 100644
--- a/doc/geocoding.md
+++ b/doc/geocoding.md
@@ -3,12 +3,12 @@
Geocoding is the process of converting a location name (like `nyc` or
`Argentina`) into GPS coordinates and bounding box(es).

-Perkeep's location search currently requires Google's Geocoding API,
-which now requires an API key. We do not currently provide a default,
-shared API key to use by default. (We might in the future.)
+Perkeep's location search will use Google's Geocoding API if a key is provided,
+otherwise it falls back to using OpenStreetMap's API.

-For now, you need to manually get your own Geocoding API key from Google and place it
-in your Perkeep configuration directory (run `pk env configdir` to
-find your configuration directory) in a file named `google-geocode.key`.
+To use Google's Geocoding API, you need to manually get your own API key from
+Google and place it in your Perkeep configuration directory (run `pk env
+configdir` to find your configuration directory) in a file named
+`google-geocode.key`.

-To get the key, see https://developers.google.com/maps/documentation/geocoding/start#get-a-key
+To get the Google API key, see https://developers.google.com/maps/documentation/geocoding/get-api-key
diff --git a/internal/geocode/geocode.go b/internal/geocode/geocode.go
index 86e94e4..de40643 100644
--- a/internal/geocode/geocode.go
+++ b/internal/geocode/geocode.go
@@ -24,15 +24,19 @@ import (
"fmt"
"io"
"log"
+ "net/http"
"net/url"
"os"
"path/filepath"
+ "strconv"
"strings"
"sync"

"perkeep.org/internal/osutil"
+ "perkeep.org/pkg/buildinfo"

"go4.org/ctxutil"
+ "go4.org/legal"
"go4.org/syncutil/singleflight"
"go4.org/wkfs"
"golang.org/x/net/context/ctxhttp"
@@ -113,7 +117,7 @@ func GetAPIKey() (string, error) {
return key, nil
}

-var ErrNoGoogleKey = errors.New("geocode: geocoding is not configured; see https://perkeep.org/doc/geocoding")
+var ErrNoGoogleKey = errors.New("geocode: Google API key not configured, using OpenStreetMap; see https://perkeep.org/doc/geocoding")

// Lookup returns rectangles for the given address. Currently the only
// implementation is the Google geocoding service.
@@ -130,36 +134,44 @@ func Lookup(ctx context.Context, address string) ([]Rect, error) {
}

key, err := GetAPIKey()
- if err != nil {
+ if err != nil && err != ErrNoGoogleKey {
return nil, err
}

rectsi, err := sf.Do(address, func() (interface{}, error) {
- // TODO: static data files from OpenStreetMap, Wikipedia, etc?
- urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
- res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
- if err != nil {
- log.Printf("geocode: HTTP error doing Google lookup: %v", err)
- return nil, err
- }
- defer res.Body.Close()
- rects, err := decodeGoogleResponse(res.Body)
- if err != nil {
- log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
+ if key != "" {
+ return lookupGoogle(ctx, address, key)
} else {
- log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
- }
- if err == nil {
- mu.Lock()
- cache[address] = rects
- mu.Unlock()
+ return lookupOpenStreetMap(ctx, address)
}
- return rects, err
})
if err != nil {
return nil, err
}
- return rectsi.([]Rect), nil
+ rects = rectsi.([]Rect)
+
+ mu.Lock()
+ cache[address] = rects
+ mu.Unlock()
+ return rects, nil
+}
+
+func lookupGoogle(ctx context.Context, address string, key string) ([]Rect, error) {
+ // TODO: static data files from OpenStreetMap, Wikipedia, etc?
+ urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
+ res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
+ if err != nil {
+ log.Printf("geocode: HTTP error doing Google lookup: %v", err)
+ return nil, err
+ }
+ defer res.Body.Close()
+ rects, err := decodeGoogleResponse(res.Body)
+ if err != nil {
+ log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
+ } else {
+ log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
+ }
+ return rects, err
}

type googleResTop struct {
@@ -194,3 +206,67 @@ func decodeGoogleResponse(r io.Reader) (rects []Rect, err error) {
}
return
}
+
+var openstreetmapUserAgent = fmt.Sprintf("perkeep/%v", buildinfo.Summary())
+
+func lookupOpenStreetMap(ctx context.Context, address string) ([]Rect, error) {
+ // TODO: static data files from OpenStreetMap, Wikipedia, etc?
+ urlStr := "https://nominatim.openstreetmap.org/search?format=json&limit=1&q=" + url.QueryEscape(address)
+ req, err := http.NewRequest("GET", urlStr, nil)
+ if err != nil {
+ log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
+ return nil, err
+ }
+ // Nominatim Usage Policy requires a user agent (https://operations.osmfoundation.org/policies/nominatim/)
+ req.Header.Set("User-Agent", openstreetmapUserAgent)
+ res, err := ctxhttp.Do(ctx, ctxutil.Client(ctx), req)
+ if err != nil {
+ log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
+ return nil, err
+ }
+ defer res.Body.Close()
+ rects, err := decodeOpenStreetMapResponse(res.Body)
+ if err != nil {
+ log.Printf("geocode: error decoding OpenStreetMap geocode response for %q: %v", address, err)
+ } else {
+ log.Printf("geocode: OpenStreetMap lookup (%q) = %#v", address, rects)
+ }
+ return rects, err
+}
+
+type openstreetmapResult struct {
+ // BoundingBox is encoded as four floats (encoded as strings) in order: SW Lat, NE Lat, SW Long, NE Long
+ BoundingBox []string `json:"boundingbox"`
+}
+
+func decodeOpenStreetMapResponse(r io.Reader) (rects []Rect, err error) {
+ var osmResults []*openstreetmapResult
+ if err := json.NewDecoder(r).Decode(&osmResults); err != nil {
+ return nil, err
+ }
+ for _, res := range osmResults {
+ if len(res.BoundingBox) == 4 {
+ var coords []float64
+ for _, b := range res.BoundingBox {
+ f, err := strconv.ParseFloat(b, 64)
+ if err != nil {
+ return nil, err
+ }
+ coords = append(coords, f)
+ }
+ rect := Rect{
+ NorthEast: LatLong{Lat: coords[1], Long: coords[3]},
+ SouthWest: LatLong{Lat: coords[0], Long: coords[2]},
+ }
+ rects = append(rects, rect)
+ }
+ }
+
+ return
+}
+
+func init() {
+ legal.RegisterLicense(`
+Mapping data and services copyright OpenStreetMap contributors, ODbL 1.0.
+https://osm.org/copyright.`)
+}
diff --git a/internal/geocode/geocode_test.go b/internal/geocode/geocode_test.go
index 9a3bc2c..06a38f4 100644
--- a/internal/geocode/geocode_test.go
+++ b/internal/geocode/geocode_test.go
@@ -236,3 +236,57 @@ var googleUSA = `
"status" : "OK"
}
`
+
+func TestDecodeOpenStreetMapResponse(t *testing.T) {
+ tests := []struct {
+ name string
+ res string
+ want []Rect
+ }{
+ {
+ name: "moscow",
+ res: openstreetmapMoscow,
+ want: []Rect{
+ {
+ NorthEast: LatLong{pf("55.9577717"), pf("37.9674277")},
+ SouthWest: LatLong{pf("55.4913076"), pf("37.290502")},
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ rects, err := decodeOpenStreetMapResponse(strings.NewReader(tt.res))
+ if err != nil {
+ t.Errorf("Decoding %s: %v", tt.name, err)
+ continue
+ }
+ if !reflect.DeepEqual(rects, tt.want) {
+ t.Errorf("Test %s: wrong rects\n Got %#v\nWant %#v", tt.name, rects, tt.want)
+ }
+ }
+}
+
+// https://nominatim.openstreetmap.org/search?format=json&limit=1&q=moscow
+var openstreetmapMoscow = `
+[
+ {
+ "place_id": 282700412,
+ "licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
+ "osm_type": "relation",
+ "osm_id": 2555133,
+ "boundingbox": [
+ "55.4913076",
+ "55.9577717",
+ "37.290502",
+ "37.9674277"
+ ],
+ "lat": "55.7504461",
+ "lon": "37.6174943",
+ "display_name": "Москва, Центральный федеральный округ, Россия",
+ "class": "place",
+ "type": "city",
+ "importance": 0.7908193282833463,
+ "icon": "https://nominatim.openstreetmap.org/ui/mapicons//poi_place_city.p.20.png"
+ }
+ ]
+ `
diff --git a/pkg/server/help.go b/pkg/server/help.go
index 85285f6..30907df 100644
--- a/pkg/server/help.go
+++ b/pkg/server/help.go
@@ -57,6 +57,10 @@ const helpHTML string = `<html>

<h3>Anything Else?</h3>
<p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p>
+
+ <h3>Attribution</h3>
+ <p>Various mapping data and services <a href="https://osm.org/copyright">copyright OpenStreetMap contributors</a>, ODbL 1.0.</p>
+
</body>
</html>`

diff --git a/server/perkeepd/perkeepd.go b/server/perkeepd/perkeepd.go
index 5350074..85bbf30 100644
--- a/server/perkeepd/perkeepd.go
+++ b/server/perkeepd/perkeepd.go
@@ -381,9 +381,9 @@ func checkGeoKey() error {
}
if env.OnGCE() {
keyPath = strings.TrimPrefix(keyPath, "/gcs/")
- return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in your VM's configuration bucket as: %v", keyPath)
+ return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in your VM's configuration bucket as: %v", keyPath)
}
- return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in Perkeep's configuration directory as: %v", keyPath)
+ return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in Perkeep's configuration directory as: %v", keyPath)
}

// main wraps Main so tests (which generate their own func main) can still run Main.