Skip to content

Commit

Permalink
Allow accepting predicted score as ground truth
Browse files Browse the repository at this point in the history 
For areas a negative answer makes sense, score it as 0, but a generic
positive answer doesn't (what should the area be in that case?). Instead
add a new option to accept the prediction as ground truth. This probably
leads to a new, longer, loop of predict, add to OSM if needed, ignore
predicted score, and then later run an import of scores from OSM,
accepting the scores that come from the newly drawn features.
  • Loading branch information
Martin Nyhus committed Feb 13, 2024
1 parent 6ca8094 commit 15ee9b5
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 33 deletions.
91 changes: 78 additions & 13 deletions database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,12 @@ def tiles_for_scoring(self, current_model, feature_name, limit):
return (tiles, count)

def tiles_for_review(self, feature_name, limit):
if feature_name == 'solar_area':
return self.tiles_for_review_area(feature_name, limit)
else:
return self.tiles_for_review_normal(feature_name, limit)

def tiles_for_review_normal(self, feature_name, limit):
review_most_likely = True

ordering = 'abs(score - 0.5) asc'
Expand Down Expand Up @@ -180,6 +186,53 @@ def tiles_for_review(self, feature_name, limit):
limit])
return c.fetchall()

def tiles_for_review_area(self, feature_name, limit):
with self.transaction('get_tiles_for_review_area') as c:
model_query = '''select model_version
from scores
where
feature_name = ?
and timestamp in (
select max(timestamp)
from scores
natural left join (
select 1 as t, tile_hash
from true_score
where feature_name = ?)
where
t is null
and feature_name = ?
)
limit 1
'''

c.execute(model_query, [feature_name, feature_name, feature_name])
model_version = c.fetchone()
if model_version is None:
return []

query_fmt = '''select scores.tile_hash, z, x, y, score, model_version
from scores
left join (
select 1 as t, tile_hash
from true_score
where feature_name = ?) as ground_truth
on ground_truth.tile_hash = scores.tile_hash
natural join tile_positions
where t is null
and feature_name = ?
and score is not null
and model_version = ?
order by score desc
limit ?
'''
query = query_fmt
c.execute(query, [feature_name,
feature_name,
model_version[0],
limit])
return c.fetchall()

def tiles_with_solar(self):
with self.transaction('get_tiles_with_solar') as c:
c.execute('''select tile_hash
Expand All @@ -188,15 +241,6 @@ def tiles_with_solar(self):
''')
return c.fetchall()

def remove_score(self, feature_name, tile_hash):
with self.transaction() as c:
c.execute('''delete from scores
where
feature_name = ?
and tile_hash = ?
''',
[feature_name, tile_hash])


def get_tile_hash(cursor, z, x, y):
assert type(z) == int
Expand Down Expand Up @@ -272,6 +316,18 @@ def add_tile_hash(cursor, z, x, y, tile_hash):
(tile_hash, z, x, y, timestamp))


def get_score(cursor, tile_hash, feature_name):
cursor.execute('''select score
from scores
where
tile_hash = ?
and feature_name = ?
''',
[tile_hash, feature_name])
row = cursor.fetchone()
return row[0] if row else None


def write_score(cursor, tile_hash, feature_name, score, model_version,
timestamp):
assert type(tile_hash) == str
Expand All @@ -292,6 +348,15 @@ def write_score(cursor, tile_hash, feature_name, score, model_version,
(tile_hash, feature_name, score, model_version, timestamp))


def remove_score(cursor, feature_name, tile_hash):
cursor.execute('''delete from scores
where
feature_name = ?
and tile_hash = ?
''',
[feature_name, tile_hash])


def set_has_feature(cursor, tile_hash, feature_name, has_feature):
assert type(tile_hash) == str
assert type(feature_name) == str
Expand Down Expand Up @@ -350,21 +415,21 @@ def mark_checked(cursor, z, x, y):


def training_tiles(cursor, feature_name):
cursor.execute('''select tile_hash, has_feature, 0
cursor.execute('''select tile_hash, score, 0
from training_set
natural join tile_positions
natural join has_feature
natural join true_score
where feature_name = ?
''',
[feature_name])
return cursor.fetchall()


def validation_tiles(cursor, feature_name):
cursor.execute('''select tile_hash, has_feature, score
cursor.execute('''select tile_hash, true_score.score, scores.score
from validation_set
natural join tile_positions
natural left join has_feature
natural left join true_score
natural left join scores
where feature_name = ?
''',
Expand Down
50 changes: 30 additions & 20 deletions web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@
import sqlite3

import database
import feature
import util


db_path = None
nib_key_path = None
tile_path = None
feature = None
feature_name = None


app = flask.Flask(__name__, static_url_path='')
Expand All @@ -32,7 +33,7 @@ def send_script():
@app.route('/api/review/next_tile')
def get_next_tile_for_review():
db = database.Database(db_path)
tiles = db.tiles_for_review(feature, limit=1)
tiles = db.tiles_for_review(feature_name, limit=1)
if not tiles:
return '', 204

Expand Down Expand Up @@ -66,38 +67,47 @@ def score_neighbours(cursor, own_hash):
neighbour_y):
database.write_score(cursor,
neighbour_hash,
feature,
feature_name,
1.0,
'neighbour',
now)


def _write_ground_truth(tile_hash, feature_name, response, cursor):
if response == 'accept':
predicted_score = database.get_score(cursor, tile_hash, feature_name)
database.set_true_score(cursor, tile_hash, feature_name, predicted_score)
elif response == 'true':
database.set_has_feature(cursor, tile_hash, feature_name, True)
score_neighbours(cursor, tile_hash)
elif response == 'false':
if feature_name == 'solar_area':
database.set_true_score(cursor, tile_hash, feature_name, 0.0)
else:
database.set_has_feature(cursor, tile_hash, feature_name, False)
elif response == 'skip':
database.remove_score(cursor, feature_name, tile_hash)
else:
raise ValueError


@app.route('/api/review/response', methods=['POST'])
def accept_tile_response():
body = flask.request.json
tile_hash = body['tile_hash']
response = body['response']

if response == 'true':
has_feature = True
elif response == 'false':
has_feature = False
elif response == 'skip':
has_feature = None
else:
return 400, 'Bad "response" value'
if response not in ['accept', 'false', 'skip', 'true']:
return 'Bad "response" value', 400

if feature_name == 'solar_area' and response == 'true':
return 'Can\'t use "true" with solar_area', 400

db = database.Database(db_path)
while True:
try:
if has_feature is not None:
with db.transaction() as c:
database.set_has_feature(c, tile_hash, feature,
has_feature)
if has_feature:
score_neighbours(c, tile_hash)
else:
db.remove_score(feature, tile_hash)
with db.transaction('write_ground_truth') as c:
_write_ground_truth(tile_hash, feature_name, response, c)
break
except sqlite3.IntegrityError as e:
print(e)
Expand Down Expand Up @@ -169,6 +179,6 @@ def get_nib_tile(z, x, y):
db_path = pathlib.Path(args.database)
nib_key_path = pathlib.Path(args.NiB_key)
tile_path = pathlib.Path(args.tile_path)
feature = args.feature
feature_name = args.feature

app.run('0.0.0.0', 5000)
2 changes: 2 additions & 0 deletions web/static/script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ function key_handler(event) {
review_surrounding_tiles();
} else if (event.key === "y") {
submit_result('true');
} else if (event.key === "a") {
submit_result('accept');
}
}

Expand Down

0 comments on commit 15ee9b5

Please sign in to comment.