Skip to content

Commit

Permalink
v0.2.0
Browse files Browse the repository at this point in the history 
Former-commit-id: ad1f914
  • Loading branch information
@atlasdatatech
atlasdatatech committed Dec 8, 2020
1 parent ad3ec4d commit de3f8a0
Show file tree
Hide file tree
Showing 10 changed files with 18 additions and 191 deletions.
4 changes: 2 additions & 2 deletions conf.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
directory ="output"
[task]
#number of fetchers
workers = 16
workers = 100
#number of savers
savepipe = 1

Expand All @@ -17,7 +17,7 @@
name = "streets-v8"
#max and min level for mbtiles
min = 0
max = 10
max = 3
#can be pbf/png/jpg
format = "jpg"
#can be xyz/tm
Expand Down
2 changes: 0 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,13 @@ go 1.15

require (
github.com/antonfisher/nested-logrus-formatter v1.3.0
github.com/atlasdatatech/orb v0.1.7-0.20200901063007-648f91dfac36
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/mattn/go-sqlite3 v1.14.2 // indirect
github.com/paulmach/orb v0.1.6
github.com/shaxbee/go-spatialite v0.0.0-20180425212100-9b4c81899e0e
github.com/shiena/ansicolor v0.0.0-20200830101100-9405ca8e49f3
github.com/sirupsen/logrus v1.6.0
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.3.0
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
gopkg.in/cheggaaa/pb.v1 v1.0.28
)
Expand Down
6 changes: 0 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,11 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/antonfisher/nested-logrus-formatter v1.1.0 h1:wb5SkAtQD/VMTOkYimj8PtdNvbNEs0QWOQXSZAw/Ars=
github.com/antonfisher/nested-logrus-formatter v1.1.0/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA=
github.com/antonfisher/nested-logrus-formatter v1.3.0 h1:8zixYquU1Odk+vzAaAQPAdRh1ZjmUXNQ1T+dUBvlhVo=
github.com/antonfisher/nested-logrus-formatter v1.3.0/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/atlasdatatech/orb v0.1.7-0.20200901063007-648f91dfac36 h1:8vMdrLLoz3ccztngz8A+ZVsVK/z9H1EvZ1hgsGCFZkU=
github.com/atlasdatatech/orb v0.1.7-0.20200901063007-648f91dfac36/go.mod h1:2wfQTtdC46BmaEh85mWP9jPpVBn45emrPsRVf/ZPdKE=
github.com/atlasdatatech/orb v0.2.2 h1:eJvcL8taHRzncteaHjfIyW6WZ1vbzTnS/ffABadLr10=
github.com/atlasdatatech/orb v0.2.2/go.mod h1:pPwxxs3zoAyosNSbNKn1jiXV2+oovRDObDKfTvRegDI=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand Down Expand Up @@ -147,8 +143,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/paulmach/orb v0.1.6 h1:C8klK4r0mR0MnfSk+GvEFFKLrQVwjQ+FlhtXgpaupjg=
github.com/paulmach/orb v0.1.6/go.mod h1:pPwxxs3zoAyosNSbNKn1jiXV2+oovRDObDKfTvRegDI=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down
70 changes: 2 additions & 68 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func init() {
log.SetLevel(log.DebugLevel)
}
func usage() {
fmt.Fprintf(os.Stderr, `tiler version: tiler/0.9.19
fmt.Fprintf(os.Stderr, `tiler version: tiler/v0.1.0
Usage: tiler [-h] [-c filename]
`)
flag.PrintDefaults()
Expand Down Expand Up @@ -70,6 +70,7 @@ func main() {
flag.Usage()
return
}

if cf == "" {
cf = "conf.toml"
}
Expand All @@ -83,10 +84,6 @@ func main() {
Schema: viper.GetString("tm.schema"),
JSON: viper.GetString("tm.json"),
URL: viper.GetString("tm.url"),
// URL: "http://mt0.google.cn/vt/lyrs=s&hl=zh-CN&x={x}&y={y}&z={z}", ///data/landcover/{z}/{x}/{y}.pbf?key=hWWfWrAiWGtv68r8wA6D
// URL: "http://tiles.emapgo.cn/data/emg.china-streets/{z}/{x}/{y}.pbf",
// URL: "http://datahive.minedata.cn/data/Buildingmore/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>14
// URL: "http://datahive.minedata.cn/mergeddata/Adminflag,Annotation,Poi,Ptline,Railway,Road,Villtown,Worldannotation/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873",
}
type cfgLayer struct {
Min int
Expand Down Expand Up @@ -115,67 +112,4 @@ func main() {
task.Download()
secs := time.Since(start).Seconds()
fmt.Printf("\n%.3fs finished...", secs)

// for z := 0; z <= 6; z++ {
// c := loadCollection("./geojson/z1-6.global.geojson")
// layer := Layer{
// // URL: "http://datahive.minedata.cn/mergeddata/Adminflag,Annotation,Poi,Ptline,Railway,Road,Villtown,Worldannotation/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873",
// URL: "http://datahive.minedata.cn/data/ResidentialPolygon/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873",
// // URL: "http://tiles.emapgo.cn/data/emg.china-streets/{z}/{x}/{y}.pbf",
// Zoom: z,
// Collection: c,
// }
// layers = append(layers, layer)
// }

// for z := 14; z <= 17; z++ {
// c := loadCollection("./geojson/beijing.geojson")
// layer := Layer{
// // URL: "http://datahive.minedata.cn/mergeddata/Adminflag,Annotation,Poi,Ptline,Railway,Road,Villtown,Worldannotation/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873",//merge8
// // URL: "http://datahive.minedata.cn/data/Waterface/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873",
// // URL: "http://datahive.minedata.cn/data/Greenface/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873",
// // URL: "http://datahive.minedata.cn/data/Landuse/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>12
// // URL: "http://datahive.minedata.cn/data/Ptstop/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>13
// // URL: "http://datahive.minedata.cn/data/ResidentialPolygon/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>13
// // URL: "http://datahive.minedata.cn/data/Buildingmore/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5874", //>14
// // URL: "http://datahive.minedata.cn/data/Zlevel/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>15
// // URL: "http://datahive.minedata.cn/data/Subwaypolygon/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>15
// // URL: "http://datahive.minedata.cn/data/Ptexit/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>16
// // URL: "http://datahive.minedata.cn/data/Trafficlight/{z}/{x}/{y}?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873", //>16
// // URL: "http://tiles.emapgo.cn/data/emg.china-streets/{z}/{x}/{y}.pbf",
// Zoom: z,
// Collection: c,
// }
// layers = append(layers, layer)
// }

// for z := 11; z <= 13; z++ {
// c := loadCollection("./geojson/z11-13.gansu.geojson")
// layer := Layer{
// URL: "http://mt0.google.cn/vt/lyrs=s&x={x}&y={y}&z={z}",
// Zoom: z,
// Collection: c,
// }
// layers = append(layers, layer)
// }

// for z := 14; z <= 16; z++ {
// c := loadCollection("./geojson/z14-16.lanzhou.geojson")
// layer := Layer{
// URL: "http://mt0.google.cn/vt/lyrs=s&x={x}&y={y}&z={z}",
// Zoom: z,
// Collection: c,
// }
// layers = append(layers, layer)
// }
// for z := 17; z <= 18; z++ {
// c := loadCollection("./geojson/z17-18.lanzhou.geojson")
// layer := Layer{
// URL: "http://mt0.google.cn/vt/lyrs=s&x={x}&y={y}&z={z}",
// Zoom: z,
// Collection: c,
// }
// layers = append(layers, layer)
// }

}
7 changes: 2 additions & 5 deletions map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ func CreateTileMap(url string) {
//GetTileMapList 获取初始化默认地图列表
func GetTileMapList() map[int]TileMap {
tml := make(map[int]TileMap)
var list = []string{"http://mt2.google.com/vt/lyrs=y&x={x}&y={y}&z={z}",
var list = []string{"http://mt0.google.com/vt/lyrs=y&x={x}&y={y}&z={z}",
"http://tile.openstreetmap.org/{z}/{x}/{y}.png",
"http://api.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.png?access_token=pk.eyJ1IjoiYWVyb3Zpc2lvbmtlc3RyZWwiLCJhIjoiY2l5bDhzYTVqMDAxNDJ3bGp1ZHA2cmtiaCJ9.8o3pqTWKiOV8RhjNGFW0rg",
// "http://mt0.google.cn/vt/lyrs=s&hl=zh-CN&x=214130&y=114212&z=18",
"http://mt0.google.cn/vt/lyrs=y&hl=zh-CN&x={x}&y={y}&z={z}",
"http://mt0.google.cn/vt/lyrs=s&x={x}&y={y}&z={z}",
"http://mt2.google.cn/vt/lyrs=y@258000000&hl=zh-CN&gl=CN&src=app&x=214130&y=114212&z=18&s=Ga", //m：路线图,t：地形图,p：带标签的地形图,s：卫星图,y：带标签的卫星图,h：标签层（路名、地名等）
"http://mt0.google.cn/vt/lyrs=y@258000000&hl=zh-CN&gl=CN&src=app&x=214130&y=114212&z=18&s=Ga", //m：路线图,t：地形图,p：带标签的地形图,s：卫星图,y：带标签的卫星图,h：标签层（路名、地名等）
}
for i, v := range list {
fmt.Println(i, v)
Expand All @@ -55,7 +55,4 @@ func (m TileMap) getTileURL(t maptile.Tile) string {
url = strings.Replace(url, "{y}", strconv.Itoa(int(t.Y)), -1)
url = strings.Replace(url, "{z}", strconv.Itoa(int(t.Z)), -1)
return url
// return fmt.Sprintf(`http://mt1.google.cn/vt/hl=zh-CN&gl=cn&x=%d&y=%d&zoom=%d&lyrs=m`, t.X, t.Y, 17-t.Z) //google
// return fmt.Sprintf(`http://datahive.minedata.cn/data/Buildingmore/%d/%d/%d?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873`, t.Z, t.X, t.Y)
// return fmt.Sprintf(`http://datahive.minedata.cn/data/ResidentialPolygon/%d/%d/%d?token=f7bf94956c3d4693bab79b5a63498f61&solu=5873`, t.Z, t.X, t.Y)
}
1 change: 0 additions & 1 deletion mod_spatialite.dll.REMOVED.git-id
View file

This file was deleted.

53 changes: 0 additions & 53 deletions spatialite.go
View file

This file was deleted.

40 changes: 0 additions & 40 deletions spatialite_test.go
View file

This file was deleted.

26 changes: 12 additions & 14 deletions task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,8 @@ func NewTask(layers []Layer, m TileMap) *Task {
if layers[i].URL == "" {
layers[i].URL = m.URL
}
t := time.Now()
layers[i].Count = tilecover.CollectionCount(layers[i].Collection, maptile.Zoom(layers[i].Zoom))
fmt.Println(time.Since(t))
fmt.Println(layers[i].Zoom, layers[i].Count)
log.Printf("zoom: %d, tiles: %d \n", layers[i].Zoom, layers[i].Count)
task.Total += layers[i].Count
}
task.abort = make(chan struct{})
Expand Down Expand Up @@ -296,7 +294,7 @@ func (task *Task) tileFetcher(t maptile.Tile, url string) {

//DownloadZoom 下载指定层级
func (task *Task) downloadLayer(layer Layer) {
bar := pb.New64(layer.Count).Prefix(fmt.Sprintf("Zoom %d : ", layer.Zoom)).Postfix("\n")
bar := pb.New64(layer.Count).Prefix(fmt.Sprintf("Zoom %d : ", layer.Zoom))
// bar.SetRefreshRate(time.Second)
bar.Start()
// bar.SetMaxWidth(300)
Expand All @@ -314,21 +312,21 @@ func (task *Task) downloadLayer(layer Layer) {
task.wg.Add(1)
go task.tileFetcher(tile, layer.URL)
case <-task.abort:
log.Infof("task %s got canceled.", task.ID)
log.Infof("Task %s got canceled.", task.ID)
close(tilelist)
case <-task.pause:
log.Infof("task %s suspended.", task.ID)
log.Infof("Task %s suspended.", task.ID)
select {
case <-task.play:
log.Infof("task %s go on.", task.ID)
log.Infof("Task %s go on.", task.ID)
case <-task.abort:
log.Infof("task %s got canceled.", task.ID)
log.Infof("Task %s got canceled.", task.ID)
close(tilelist)
}
}
}
task.wg.Wait()
bar.FinishPrint(fmt.Sprintf("Task %s zoom %d finished ~", task.ID, layer.Zoom))
bar.FinishPrint(fmt.Sprintf("Task %s Zoom %d finished ~", task.ID, layer.Zoom))
}

//DownloadZoom 下载指定层级
Expand All @@ -353,10 +351,10 @@ func (task *Task) downloadGeom(geom orb.Geometry, zoom int) {
task.wg.Add(1)
go task.tileFetcher(tile, task.TileMap.URL)
case <-task.abort:
log.Infof("task %s got canceled.", task.ID)
log.Infof("Task %s got canceled.", task.ID)
close(tilelist)
case <-task.pause:
log.Infof("task %s suspended.", task.ID)
log.Infof("Task %s suspended.", task.ID)
select {
case <-task.play:
log.Infof("task %s go on.", task.ID)
Expand All @@ -367,7 +365,7 @@ func (task *Task) downloadGeom(geom orb.Geometry, zoom int) {
}
}
task.wg.Wait()
bar.FinishPrint(fmt.Sprintf("task %s zoom %d finished ~", task.ID, zoom))
bar.FinishPrint(fmt.Sprintf("Task %s Zoom %d finished ~", task.ID, zoom))
}

//Download 开启下载任务
Expand All @@ -386,7 +384,7 @@ func (task *Task) Download() {
task.downloadLayer(layer)
}
task.wg.Wait()
task.Bar.FinishPrint(fmt.Sprintf("task %s finished ~", task.ID))
task.Bar.FinishPrint(fmt.Sprintf("Task %s finished ~", task.ID))
}

//DownloadDepth 深度优先下载
Expand Down Expand Up @@ -454,5 +452,5 @@ func (task *Task) DownloadDepth() {
task.wg.Wait() //wait for saving
bar.FinishPrint(fmt.Sprintf("zoom %d finished ~", layer.Zoom))
}
task.Bar.FinishPrint(fmt.Sprintf("task %s finished ~", task.ID))
task.Bar.FinishPrint(fmt.Sprintf("Task %s finished ~", task.ID))
}
Binary file removed zlib1.dll
View file
Binary file not shown.

0 comments on commit de3f8a0

Please sign in to comment.