Skip to content

Commit

Permalink
data for postgis provider tests included in the CI. additional postgi…
Browse files Browse the repository at this point in the history 
…s provider test. !zoom! token tests. closes go-spatial#88
  • Loading branch information
@ARolek
ARolek committed Mar 12, 2017
1 parent 3d74016 commit 1ad1ff9
Show file tree
Hide file tree
Showing 4 changed files with 176 additions and 64 deletions.
13 changes: 9 additions & 4 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ language: go
go:
- tip

notifications:
slack:
secure: TzRnZdJ1dhQJg9nby2oJ6qFj9Bl20VJ2aJXmXwADsj2ck/UGsFWpYNnqZVWvZomCZEJinFpA/h2TG1odEzg1BpCT0+rLIGQDAhIMVuici+nIs+DyPQmL2owG1LZ6/hzX09Y1mIZci0nmnI1wousfM8WDPRq5NSOsBeAMlixGR4g4lF8cp/R0B6CtcewtQx4RicdPJtopHDwXm5KI1/7euFcJwu15FWwWoUp6mOw+r44T1Md+qbqDKBbLSy+YvMkPcxI7HeECa7OP9OjkeUfU0BoMP6qqo9QTVX5v6lOHxA+FhueyyPrSTtNV377tilDy88kytqu6w8gcrxZOddj7nFbv6ErjfVaMfeepzk/HH2bzlLevvL+ukeJKrBNdmPuDRU3BqG5ygrdGAQUNpM7VSXLDcGrM2+ocVn/HO1NyXWwpejq7Gq0HWhNkYPHG5c7+khkQkRGJtg0wp9R3qXDAcRHLR+CHh89g5Blz4Wo6R/Q48MLJycbH/W56zsoe0xQ5Ww2eti16uSFE4kkiPt99uewWkPSfEZn4pyk+xCwDxgNDNu/BaV+Mjm4JTL1GjWalto+NLK7NsEg+6sEYok0CHkUb8MQV++r1KVfIaxC3zVuojIiOSB9/+JPtPd9w2iuVOlz0rrbXfKkkSf5wcmOguqk3aAfAgmcjAV9p1kI9pSc=
env:
global:
- RUN_POSTGIS_TESTS=yes

install:
- bash ci/config_postgis.sh

before_deploy:
- go get github.com/mitchellh/gox
Expand Down Expand Up @@ -38,4 +41,6 @@ deploy:

notifications:
email:
on_success: never
on_success: never
slack:
secure: TzRnZdJ1dhQJg9nby2oJ6qFj9Bl20VJ2aJXmXwADsj2ck/UGsFWpYNnqZVWvZomCZEJinFpA/h2TG1odEzg1BpCT0+rLIGQDAhIMVuici+nIs+DyPQmL2owG1LZ6/hzX09Y1mIZci0nmnI1wousfM8WDPRq5NSOsBeAMlixGR4g4lF8cp/R0B6CtcewtQx4RicdPJtopHDwXm5KI1/7euFcJwu15FWwWoUp6mOw+r44T1Md+qbqDKBbLSy+YvMkPcxI7HeECa7OP9OjkeUfU0BoMP6qqo9QTVX5v6lOHxA+FhueyyPrSTtNV377tilDy88kytqu6w8gcrxZOddj7nFbv6ErjfVaMfeepzk/HH2bzlLevvL+ukeJKrBNdmPuDRU3BqG5ygrdGAQUNpM7VSXLDcGrM2+ocVn/HO1NyXWwpejq7Gq0HWhNkYPHG5c7+khkQkRGJtg0wp9R3qXDAcRHLR+CHh89g5Blz4Wo6R/Q48MLJycbH/W56zsoe0xQ5Ww2eti16uSFE4kkiPt99uewWkPSfEZn4pyk+xCwDxgNDNu/BaV+Mjm4JTL1GjWalto+NLK7NsEg+6sEYok0CHkUb8MQV++r1KVfIaxC3zVuojIiOSB9/+JPtPd9w2iuVOlz0rrbXfKkkSf5wcmOguqk3aAfAgmcjAV9p1kI9pSc=
20 changes: 20 additions & 0 deletions ci/config_postgis.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

set -ex

# fetch our test data and import it into Postgres
configure_postgis() {
local test_data="tegola.backup"
local test_data_url="https://s3-us-west-1.amazonaws.com/tegola-test-data/tegola-postgis-test-data.backup"

# fetch our test data
curl $test_data_url > $test_data

# import the data to postgres
pg_restore -C -d postgres $test_data

# clean up our test data
rm $test_data
}

configure_postgis
80 changes: 46 additions & 34 deletions provider/postgis/postgis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ import (
"github.com/terranodo/tegola/wkb"
)

// Provider provides the postgis data provider.
type Provider struct {
config pgx.ConnPoolConfig
pool *pgx.ConnPool
// map of layer name and corrosponding sql
layers map[string]Layer
srid int
}

// layer holds information about a query.
type Layer struct {
// The Name of the layer
Expand All @@ -31,30 +40,26 @@ type Layer struct {
SRID int
}

// Provider provides the postgis data provider.
type Provider struct {
config pgx.ConnPoolConfig
pool *pgx.ConnPool
// map of layer name and corrosponding sql
layers map[string]Layer
srid int
}

const (
bboxToken = "!BBOX!"
zoomToken = "!ZOOM!"
)

// We quote the field and table names to prevent colliding with postgres keywords.
const stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + bboxToken
const (
// We quote the field and table names to prevent colliding with postgres keywords.
stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + bboxToken

// SQL to get the column names, without hitting the information_schema. Though it might be better to hit the information_schema.
const fldsSQL = `SELECT * FROM %[1]v LIMIT 0;`
// SQL to get the column names, without hitting the information_schema. Though it might be better to hit the information_schema.
fldsSQL = `SELECT * FROM %[1]v LIMIT 0;`
)

const Name = "postgis"
const DefaultPort = 5432
const DefaultSRID = tegola.WebMercator
const DefaultMaxConn = 5

const (
DefaultPort = 5432
DefaultSRID = tegola.WebMercator
DefaultMaxConn = 5
)

const (
ConfigKeyHost = "host"
Expand Down Expand Up @@ -87,6 +92,7 @@ func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string) (sql st
return "", err
}
defer rows.Close()

fdescs := rows.FieldDescriptions()
if len(fdescs) == 0 {
return "", fmt.Errorf("No fields were returned for table %v", tblname)
Expand All @@ -100,6 +106,7 @@ func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string) (sql st
for i := range flds {
flds[i] = fmt.Sprintf(`"%v"`, flds[i])
}

var fgeom int = -1
var fgid bool
for i, f := range flds {
Expand All @@ -118,29 +125,33 @@ func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string) (sql st
} else {
flds[fgeom] = fmt.Sprintf(`ST_AsBinary("%v") AS "%[1]v"`, l.GeomFieldName)
}

if !fgid {
flds = append(flds, fmt.Sprintf(`"%v"`, l.IDFieldName))
}

selectClause := strings.Join(flds, ", ")

return fmt.Sprintf(stdSQL, selectClause, tblname, l.GeomFieldName), nil
}

// NewProvider Setups and returns a new postgis provider or an error; if something
// is wrong. The function will validate that the config object looks good before
// trying to create a driver. This means that the Provider expects the following
// fields to exists in the provided map[string]interface{} map:
// host string — the host to connect to.
// port uint16 — the port to connect on.
// database string — the database name
// user string — the user name
// password string — the Password
// max_connections *uint8 // Default is 5 if nil, 0 means no max.
// layers map[string]struct{ — This is map of layers keyed by the layer name.
// tablename string || sql string — This is the sql to use or the tablename to use with the default query.
// fields []string — This is a list, if this is nil or empty we will get all fields.
// geometry_fieldname string — This is the field name of the geometry, if it's an empty string or nil, it will defaults to 'geom'.
// id_fieldname string — This is the field name for the id property, if it's an empty string or nil, it will defaults to 'gid'.
// }
// NewProvider Setups and returns a new postgis provider or an error; if something
// is wrong. The function will validate that the config object looks good before
// trying to create a driver. This means that the Provider expects the following
// fields to exists in the provided map[string]interface{} map:
//
// host (string) — the host to connect to.
// port (uint16) — the port to connect on.
// database (string) — the database name
// user (string) — the user name
// password (string) — the Password
// max_connections (*uint8) // Default is 5 if nil, 0 means no max.
// layers (map[string]struct{}) — This is map of layers keyed by the layer name.
// tablename (string || sql string) — This is the sql to use or the tablename to use with the default query.
// fields ([]string) — This is a list, if this is nil or empty we will get all fields.
// geometry_fieldname (string) — This is the field name of the geometry, if it's an empty string or nil, it will defaults to 'geom'.
// id_fieldname (string) — This is the field name for the id property, if it's an empty string or nil, it will defaults to 'gid'.
//
func NewProvider(config map[string]interface{}) (mvt.Provider, error) {
// Validate the config to make sure it has the values I care about and the types for those values.
c := dict.M(config)
Expand Down Expand Up @@ -222,11 +233,13 @@ func NewProvider(config map[string]interface{}) (mvt.Provider, error) {
if err != nil {
return nil, fmt.Errorf("For layer (%v) %v %v field had the following error: %v", i, lname, ConfigKeyFields, err)
}

geomfld := "geom"
geomfld, err = vc.String(ConfigKeyGeomField, &geomfld)
if err != nil {
return nil, fmt.Errorf("For layer (%v) %v : %v", i, lname, err)
}

idfld := "gid"
idfld, err = vc.String(ConfigKeyGeomIDField, &idfld)
if err != nil {
Expand All @@ -241,10 +254,9 @@ func NewProvider(config map[string]interface{}) (mvt.Provider, error) {
if err != nil {
return nil, fmt.Errorf("for %v layer(%v) %v has an error: %v", i, lname, ConfigKeyTablename, err)
}
var sql string

var sql string
sql, err = vc.String(ConfigKeySQL, &sql)

if err != nil {
return nil, fmt.Errorf("for %v layer(%v) %v has an error: %v", i, lname, ConfigKeySQL, err)
}
Expand Down
127 changes: 101 additions & 26 deletions provider/postgis/postgis_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package postgis_test

import (
"log"
"os"
"testing"

Expand All @@ -10,45 +9,119 @@ import (
)

func TestNewProvider(t *testing.T) {
// The database connection string have the following JSON format:
// { "host" : "host", port
if os.Getenv("RUN_POSTGRESS_TEST") == "" {
if os.Getenv("RUN_POSTGIS_TEST") != "yes" {
return
}

config := map[string]interface{}{
postgis.ConfigKeyHost: "localhost",
postgis.ConfigKeyPort: int64(5432),
postgis.ConfigKeyDB: "gdey",
postgis.ConfigKeyUser: "gdey",
postgis.ConfigKeyPassword: "",
postgis.ConfigKeyLayers: map[string]map[string]interface{}{
"buildings": map[string]interface{}{
postgis.ConfigKeyTablename: "gis.zoning_base_3857",
testcases := []struct {
config map[string]interface{}
}{
{
config: map[string]interface{}{
postgis.ConfigKeyHost: "localhost",
postgis.ConfigKeyPort: int64(5432),
postgis.ConfigKeyDB: "tegola",
postgis.ConfigKeyUser: "postgres",
postgis.ConfigKeyPassword: "",
postgis.ConfigKeyLayers: []map[string]interface{}{
{
postgis.ConfigKeyLayerName: "land",
postgis.ConfigKeyTablename: "ne_10m_land_scale_rank",
},
},
},
},
}
p, err := postgis.NewProvider(config)
if err != nil {
t.Errorf("Failed to create a new provider. %v", err)

for i, tc := range testcases {
_, err := postgis.NewProvider(tc.config)
if err != nil {
t.Errorf("Failed test %v. Unable to create a new provider. err: %v", i, err)
return
}
}
}

func TestMVTLayer(t *testing.T) {
if os.Getenv("RUN_POSTGIS_TEST") != "yes" {
return
}

tile := tegola.Tile{
Z: 15,
X: 12451,
Y: 18527,
testcases := []struct {
config map[string]interface{}
tile tegola.Tile
expectedFeatureCount int
}{
{
config: map[string]interface{}{
postgis.ConfigKeyHost: "localhost",
postgis.ConfigKeyPort: int64(5432),
postgis.ConfigKeyDB: "tegola",
postgis.ConfigKeyUser: "postgres",
postgis.ConfigKeyPassword: "",
postgis.ConfigKeyLayers: []map[string]interface{}{
{
postgis.ConfigKeyLayerName: "land",
postgis.ConfigKeyTablename: "ne_10m_land_scale_rank",
},
},
},
tile: tegola.Tile{
Z: 1,
X: 1,
Y: 1,
},
expectedFeatureCount: 614,
},
{
config: map[string]interface{}{
postgis.ConfigKeyHost: "localhost",
postgis.ConfigKeyPort: int64(5432),
postgis.ConfigKeyDB: "tegola",
postgis.ConfigKeyUser: "postgres",
postgis.ConfigKeyPassword: "",
postgis.ConfigKeyLayers: []map[string]interface{}{
{
postgis.ConfigKeyLayerName: "land",
postgis.ConfigKeySQL: "SELECT gid, ST_AsBinary(geom) FROM ne_10m_land_scale_rank WHERE scalerank=!ZOOM! AND geom && !BBOX!",
},
},
},
tile: tegola.Tile{
Z: 1,
X: 1,
Y: 1,
},
expectedFeatureCount: 23,
},
}
l, err := p.MVTLayer("buildings", tile, map[string]interface{}{"class": "park"})
if err != nil {
t.Errorf("Failed to create mvt layer. %v", err)
return

for i, tc := range testcases {
p, err := postgis.NewProvider(tc.config)
if err != nil {
t.Errorf("Failed test %v. Unable to create a new provider. err: %v", i, err)
return
}

// iterate our configured layers
for _, tcLayer := range tc.config[postgis.ConfigKeyLayers].([]map[string]interface{}) {
layerName := tcLayer[postgis.ConfigKeyLayerName].(string)

l, err := p.MVTLayer(layerName, tc.tile, map[string]interface{}{})
if err != nil {
t.Errorf("Failed to create mvt layer. %v", err)
return
}

if len(l.Features()) != tc.expectedFeatureCount {
t.Errorf("Failed test %v. Expected feature count (%v), got (%v)", i, tc.expectedFeatureCount, len(l.Features()))
return
}
}
}
log.Printf("Go to following layer %v\n", l)
}

func TestReplaceTokens(t *testing.T) {

testcases := []struct {
layer postgis.Layer
tile tegola.Tile
Expand Down Expand Up @@ -84,10 +157,12 @@ func TestReplaceTokens(t *testing.T) {
sql, err := postgis.ReplaceTokens(&tc.layer, tc.tile)
if err != nil {
t.Errorf("Failed test %v. err: %v", i, err)
return
}

if sql != tc.expected {
t.Errorf("Failed test %v. Expected (%v), got (%v)", i, tc.expected, sql)
return
}
}
}

0 comments on commit 1ad1ff9

Please sign in to comment.