Skip to content

Commit

Permalink
修改data预处理方法
Browse files Browse the repository at this point in the history
  • Loading branch information
@yiducn
yiducn committed Apr 13, 2016
1 parent 5497aba commit 49e0a35
Show file tree
Hide file tree
Showing 7 changed files with 354,307 additions and 15 deletions.
42 changes: 41 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<geotools.version>15-SNAPSHOT</geotools.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -51,7 +52,46 @@
<artifactId>google-maps-services</artifactId>
<version>0.1.7</version>
</dependency>

<dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-shapefile</artifactId>
<version>${geotools.version}</version>
</dependency>
<dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-swing</artifactId>
<version>${geotools.version}</version>
</dependency>
<dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-geojson</artifactId>
<version>${geotools.version}</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
</dependency>

</dependencies>
<repositories>
<repository>
<id>maven2-repository.dev.java.net</id>
<name>Java.net repository</name>
<url>http://download.java.net/maven/2</url>
</repository>
<repository>
<id>osgeo</id>
<name>Open Source Geospatial Foundation Repository</name>
<url>http://download.osgeo.org/webdav/geotools/</url>
</repository>
<repository>
<snapshots>
<enabled>true</enabled>
</snapshots>
<id>boundless</id>
<name>Boundless Maven Repository</name>
<url>http://repo.boundlessgeo.com/main</url>
</repository>
</repositories>
</project>
117 changes: 107 additions & 10 deletions src/main/java/org/duyi/AllParsersAir.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,15 @@ public class AllParsersAir {

public static void main(String[] args){
// preProcess();
removeDuplicate();
// removeDuplicate();
// meteorologicalStationsToMongo();
// geoCodingGoogle();
// insertCityCode();
// insertProvinceName();
// updateProvinceName();
updateProvinceName();
updateCityName();
// updateCityName4Daily();
// interpolateDailyData();
}

/**
Expand Down Expand Up @@ -154,9 +157,9 @@ private static void updateProvinceName(){
//get city codes
HashMap<String, String> provCode = new HashMap<String, String>();
MongoClient client = new MongoClient("127.0.0.1");
MongoDatabase db = client.getDatabase("pm");
MongoCollection locWithLL = db.getCollection("loc_ll_google");
MongoCursor cursor = locWithLL.find().iterator();
MongoDatabase db = client.getDatabase("airdb");
MongoCollection stations = db.getCollection("pm_stations");
MongoCursor cursor = stations.find().iterator();
Document d;
while(cursor.hasNext()){
d = (Document)cursor.next();
Expand All @@ -168,13 +171,72 @@ private static void updateProvinceName(){
cursor = month.find().iterator();
while(cursor.hasNext()){
d = (Document)cursor.next();
String code = d.getString("code");
String code = ((Document)d.get("_id")).getString("code");
String province = provCode.get(code);
// System.out.println(province);
month.updateOne(new Document("_id", d.get("_id")),
new Document("$set", new Document("province", province)));
}
}
/**
* 将城市名称添加到pmdata_month中
* 在后调用
*/
private static void updateCityName(){
//get city codes
HashMap<String, String> cityCode = new HashMap<String, String>();
MongoClient client = new MongoClient("127.0.0.1");
MongoDatabase db = client.getDatabase("airdb");
MongoCollection stations = db.getCollection("pm_stations");
MongoCursor cursor = stations.find().iterator();
Document d;
while(cursor.hasNext()){
d = (Document)cursor.next();
cityCode.put(d.getString("code"), d.getString("city"));
// System.out.println(d.getString("code") + ":" + d.getString("city"));
}

MongoCollection month = db.getCollection("pmdata_month");
cursor = month.find().iterator();
while(cursor.hasNext()){
d = (Document)cursor.next();
String code = ((Document)d.get("_id")).getString("code");
String province = cityCode.get(code);
// System.out.println(province);
month.updateOne(new Document("_id", d.get("_id")),
new Document("$set", new Document("city", province)));
}
}

/**
* 将城市名称添加到pmdata_day中
* 在后调用
*/
private static void updateCityName4Daily(){
//get city codes
HashMap<String, String> cityCode = new HashMap<String, String>();
MongoClient client = new MongoClient("127.0.0.1");
MongoDatabase db = client.getDatabase("airdb");
MongoCollection stations = db.getCollection("pm_stations");
MongoCursor cursor = stations.find().iterator();
Document d;
while(cursor.hasNext()){
d = (Document)cursor.next();
cityCode.put(d.getString("code"), d.getString("city"));
// System.out.println(d.getString("code") + ":" + d.getString("city"));
}

MongoCollection month = db.getCollection("pmdata_day");
cursor = month.find().iterator();
while(cursor.hasNext()){
d = (Document)cursor.next();
String code = ((Document)d.get("_id")).getString("code");
String province = cityCode.get(code);
// System.out.println(province);
month.updateOne(new Document("_id", d.get("_id")),
new Document("$set", new Document("city", province)));
}
}

/**
* 将气象站数据处理后写入数据库
Expand Down Expand Up @@ -225,9 +287,9 @@ else if(type.equals("一般站"))
*/
private static void geoCodingGoogle(){
MongoClient client = new MongoClient("127.0.0.1");
MongoDatabase db = client.getDatabase("pm");
MongoDatabase db = client.getDatabase("airdb");
MongoCollection location = db.getCollection("location");
MongoCollection locWithLL = db.getCollection("loc_ll_google");
MongoCollection locWithLL = db.getCollection("pm_stations");
MongoCursor cursor = location.find().iterator();
GeoApiContext context = new GeoApiContext().setApiKey(API_KEY_GOOGLE);
Document d;
Expand Down Expand Up @@ -469,8 +531,8 @@ private static void writeStationToDB(){
String line = br.readLine();

MongoClient client = new MongoClient("127.0.0.1");
MongoDatabase db = client.getDatabase("pm");
MongoCollection location = db.getCollection("location");
MongoDatabase db = client.getDatabase("airdb");
MongoCollection location = db.getCollection("pm_stations");
Document doc = null;
ArrayList<String> station = new ArrayList<String>();

Expand Down Expand Up @@ -613,4 +675,39 @@ private static void preprocess(){
}
client.close();
}

static void interpolateDailyData(){
// HashMap<String, String> codeAName = parseLocationNames();
// Calendar c = Calendar.getInstance(TimeZone.getTimeZone("GMT+08:00"));
// MongoClient client = new MongoClient("127.0.0.1");
// MongoDatabase db = client.getDatabase("airdb");
// MongoCollection pm = db.getCollection("pmdata_day");
// Document sort = new Document();
// sort.put("code", -1);
// sort.put("time", 1);
// MongoCursor cur = pm.find().sort(sort).iterator();
// Document pre = null;
// while(cur.hasNext()){
// if(pre == null){
// pre = (Document)cur.next();
// continue;
// }
// Document current = (Document)cur.next();
// c.setTime(current.getDate("time"));
// int newDay = c.get(Calendar.DAY_OF_YEAR);
// c.setTime(pre.getDate("time"));
// int preDay = c.get(Calendar.DAY_OF_YEAR);
//
// if(newDay - preDay > 1){
// for(int i = 0; i < (newDay-preDay-1); i ++) {
// Document d = pre;
// ((Document)d.get("_id")).
// }
// d.put()
// }
// pre = current;
//
// }
// client.close();
}
}
121 changes: 121 additions & 0 deletions src/main/java/org/duyi/GeoJsonParser.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package org.duyi;

import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.vividsolutions.jts.geom.*;
import org.bson.Document;
import org.geotools.data.simple.SimpleFeatureIterator;
import org.geotools.feature.FeatureCollection;
import org.geotools.feature.FeatureIterator;
import org.geotools.geojson.feature.FeatureJSON;
import org.geotools.geojson.geom.GeometryJSON;
import org.geotools.geometry.jts.JTS;
import org.geotools.geometry.jts.JTSFactoryFinder;
import org.geotools.referencing.GeodeticCalculator;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.opengis.feature.Feature;

import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;

/**
* Created by yidu on 3/30/16.
*/
public class GeoJsonParser {

public static void main(String[] args){
try {
//input : codeList distance
//output : cluster
//计算codeList的中心,可以在前端计算好
//循环,找到所有满足要求的点
//循环feature,对每一个feature,判断点是否在范围内,如果在,聚类

MongoClient client = new MongoClient("127.0.0.1");
MongoDatabase db = client.getDatabase("airdb");
MongoCollection coll = db.getCollection("pm_stations");
MongoCursor cur = coll.find().iterator();
JSONObject oneStation;
ArrayList<JSONObject> filtered = new ArrayList<JSONObject>();
double maxDistance = 603153.1314;//最大距离约束
//中心 经度\纬度116°23′17〃，北纬：39°54′27;116.5, 40
String[] codes = {"1001A", "1002A"};
Document d;
while(cur.hasNext()){
d = (Document)cur.next();
double lon = d.getDouble("lon");
double lat = d.getDouble("lat");
GeodeticCalculator calc = new GeodeticCalculator();
// mind, this is lon/lat
calc.setStartingGeographicPoint(lon, lat);
calc.setDestinationGeographicPoint(116.4, 40);
double distance = calc.getOrthodromicDistance();

//距离在最大距离之外的去除
if(distance > maxDistance)
continue;

//去除自己
boolean self = false;
for(int i = 0; i < codes.length; i ++) {
if (d.get("code").equals(codes[i]))
self = true;
}
if(self)
continue;


oneStation = new JSONObject();
GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory();
try {
oneStation.put("city", d.getString("city"));
oneStation.put("station", d.getString("name"));
oneStation.put("longitude", d.getDouble("lon"));
oneStation.put("latitude", d.getDouble("lat"));
oneStation.put("code", d.getString("code"));
Coordinate coord = new Coordinate(d.getDouble("lon"), d.getDouble("lat"));
Point point = geometryFactory.createPoint(coord);
oneStation.put("point", point);
} catch (JSONException e) {
e.printStackTrace();
}
filtered.add(oneStation);
}

FeatureJSON fj = new FeatureJSON();
FeatureCollection fc = fj.readFeatureCollection(new FileInputStream(new File("/Users/yidu/dev/airvisprocessing/src/main/java/org/duyi/china_cities.json")));
FeatureIterator iterator = fc.features();
ArrayList<Geometry> cityArea = new ArrayList<Geometry>();
ArrayList cluster = new ArrayList();

try {
while( iterator.hasNext() ){
Feature feature = iterator.next();
Geometry value = (Geometry)feature.getDefaultGeometryProperty().getValue();
cityArea.add(value);
}

for(int i = 0; i < cityArea.size(); i ++){
ArrayList oneCluster = new ArrayList();
for(int j = 0; j < filtered.size(); j ++){
if(cityArea.get(i).contains((Geometry)filtered.get(j).get("point"))){
oneCluster.add(filtered);
}
}
if(!oneCluster.isEmpty())
cluster.add(oneCluster);
}
}
finally {
iterator.close();
}
}catch(Exception e){
e.printStackTrace();
}
}
}
Loading

0 comments on commit 49e0a35

Please sign in to comment.