diff --git a/api/README.md b/api/README.md index 4d3622f9b..732300527 100644 --- a/api/README.md +++ b/api/README.md @@ -5,7 +5,7 @@ We package useful components developed in the Camera Traps project into APIs and ## Detector -Our animal detection model ([MegaDetector](https://github.com/Microsoft/CameraTraps#megadetector)) trained on camera trap images from a variety of ecosystems is exposed through two APIs, one for real-time applications or small batches of test images (synchronous API), and one for processing large collections of images (batch processing API). These APIs can be adapted to deploy any algorithms or models - see our tutorial in the [AI for Earth API Framework](https://github.com/Microsoft/AIforEarth-API-Development) repo. +Our animal detection model ([MegaDetector](https://github.com/Microsoft/CameraTraps#megadetector)) trained on camera trap images from a variety of ecosystems is exposed through two APIs, one for real-time applications or small batches of test images (synchronous API), and one for processing large collections of images (batch processing API). These APIs can be adapted to deploy any algorithms or models – see our tutorial in the [AI for Earth API Framework](https://github.com/Microsoft/AIforEarth-API-Development) repo. ### Synchronous API @@ -22,3 +22,8 @@ This API runs the detector on up to 2 million images in one request using [Azure Upcoming improvements: - [ ] Adapt `runserver.py` to use the newest version of the AI4E API Framework - [ ] More checks on the input container and image list SAS keys + + +## Integration with other tools + +The “integration” folder contains guidelines and postprocessing scripts for using the output of our API in other applications. diff --git a/api/integration/MLDebugTemplate.tdb b/api/integration/MLDebugTemplate.tdb new file mode 100644 index 000000000..fa8063727 Binary files /dev/null and b/api/integration/MLDebugTemplate.tdb differ diff --git a/api/integration/images/tl_boxes.jpg b/api/integration/images/tl_boxes.jpg new file mode 100644 index 000000000..47c652f5a Binary files /dev/null and b/api/integration/images/tl_boxes.jpg differ diff --git a/api/integration/images/tl_confidence.jpg b/api/integration/images/tl_confidence.jpg new file mode 100644 index 000000000..cd50441cd Binary files /dev/null and b/api/integration/images/tl_confidence.jpg differ diff --git a/api/integration/images/tl_template.jpg b/api/integration/images/tl_template.jpg new file mode 100644 index 000000000..0ed704327 Binary files /dev/null and b/api/integration/images/tl_template.jpg differ diff --git a/api/integration/prepare_api_output_for_timelapse.py b/api/integration/prepare_api_output_for_timelapse.py new file mode 100644 index 000000000..60bd426b5 --- /dev/null +++ b/api/integration/prepare_api_output_for_timelapse.py @@ -0,0 +1,200 @@ +# +# prepare_api_output_for_timelapse.py +# +# Takes output from the batch API and does some conversions to prepare +# it for use in Timelapse. +# +# Specifically: +# +# * Removes the class field from each bounding box +# * Optionally does query-based subsetting of rows +# * Optionally does a search and replace on filenames +# * Replaces backslashes with forward slashes +# * Renames "detections" to "predicted_boxes" +# +# Note that "relative" paths as interpreted by Timelapse aren't strictly relative as +# of 6/5/2019. If your project is in: +# +# c:\myproject +# +# ...and your .tdb file is: +# +# c:\myproject\blah.tdb +# +# ...and you have an image at: +# +# c:\myproject\imagefolder1\img.jpg +# +# The .csv that Timelapse sees should refer to this as: +# +# myproject/imagefolder1/img.jpg +# +# ...*not* as: +# +# imagefolder1/img.jpg +# +# Hence all the search/replace functionality in this script. It's very straightforward +# once you get this and doesn't take time, but it's easy to forget to do this. This will +# be fixed in an upcoming release. +# + +#%% Constants and imports + +# Python standard +import csv +import os + +# pip-installable +from tqdm import tqdm + +# AI4E repos, expected to be available on the path +from api.batch_processing.load_api_results import load_api_results +import matlab_porting_tools as mpt + + +#%% Helper classes + +class TimelapsePrepOptions: + + # Only process rows matching this query (if not None); this is processed + # after applying os.normpath to filenames. + query = None + + # If not none, replace the query token with this + replacement = None + + # If not none, prepend matching filenames with this + prepend = None + + removeClassLabel = False + nRows = None + temporaryMatchColumn = '_bMatch' + + +#%% Helper functions + +def process_row(row,options): + + if options.removeClassLabel: + + detections = row['detections'] + for iDetection,detection in enumerate(detections): + detections[iDetection] = detection[0:5] + + # If there's no query, we're just pre-pending + if options.query is None: + + row[options.temporaryMatchColumn] = True + if options.prepend is not None: + row['image_path'] = options.prepend + row['image_path'] + + else: + + fn = row['image_path'] + if options.query in os.path.normpath(fn): + + row[options.temporaryMatchColumn] = True + + if options.prepend is not None: + row['image_path'] = options.prepend + row['image_path'] + + if options.replacement is not None: + fn = fn.replace(options.query,options.replacement) + row['image_path'] = fn + + return row + + +#%% Main function + +def prepare_api_output_for_timelapse(inputFilename,outputFilename,options): + + if options is None: + options = TimelapsePrepOptions() + + if options.query is not None: + options.query = os.path.normpath(options.query) + + detectionResults = load_api_results(inputFilename,nrows=options.nRows) + nRowsLoaded = len(detectionResults) + + # Create a temporary column we'll use to mark the rows we want to keep + detectionResults[options.temporaryMatchColumn] = False + + # This is the main loop over rows + tqdm.pandas() + detectionResults = detectionResults.progress_apply(lambda x: process_row(x,options), axis=1) + + print('Finished main loop, post-processing output') + + # Trim to matching rows + detectionResults = detectionResults.loc[detectionResults[options.temporaryMatchColumn]] + print('Trimmed to {} matching rows (from {})'.format(len(detectionResults),nRowsLoaded)) + + detectionResults = detectionResults.drop(columns=options.temporaryMatchColumn) + + # Timelapse legacy issue; we used to call this column 'predicted_boxes' + detectionResults.rename(columns={'detections':'predicted_boxes'},inplace=True) + detectionResults['image_path'] = detectionResults['image_path'].str.replace('\\','/') + + # Write output + # write_api_results(detectionResults,outputFilename) + detectionResults.to_csv(outputFilename,index=False,quoting=csv.QUOTE_MINIMAL) + + return detectionResults + + +#%% Interactive driver + +if False: + + #%% + + inputFilename = r"D:\temp\demo_images\snapshot_serengeti\detections.csv" + outputFilename = mpt.insert_before_extension(inputFilename,'for_timelapse') + + options = TimelapsePrepOptions() + options.prepend = '' + options.replacement = 'snapshot_serengeti' + options.query = r'd:\temp\demo_images\snapshot_serengeti' + options.nRows = None + options.removeClassLabel = True + + detectionResults = prepare_api_output_for_timelapse(inputFilename,outputFilename,options) + print('Done, found {} matches'.format(len(detectionResults))) + + +#%% Command-line driver (** outdated **) + +import argparse +import inspect + +# Copy all fields from a Namespace (i.e., the output from parse_args) to an object. +# +# Skips fields starting with _. Does not check existence in the target object. +def argsToObject(args, obj): + + for n, v in inspect.getmembers(args): + if not n.startswith('_'): + # print('Setting {} to {}'.format(n,v)) + setattr(obj, n, v); + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('inputFile') + parser.add_argument('outputFile') + parser.add_argument('--query', action='store', type=str, default=None) + parser.add_argument('--prepend', action='store', type=str, default=None) + parser.add_argument('--replacement', action='store', type=str, default=None) + args = parser.parse_args() + + # Convert to an options object + options = TimelapsePrepOptions() + argsToObject(args,options) + + prepare_api_output_for_timelapse(args.inputFile,args.outputFile,args.query,options) + +if __name__ == '__main__': + + main() diff --git a/api/integration/timelapse.md b/api/integration/timelapse.md new file mode 100644 index 000000000..968cf84f6 --- /dev/null +++ b/api/integration/timelapse.md @@ -0,0 +1,77 @@ +# Overview + +[Timelapse](http://saul.cpsc.ucalgary.ca/timelapse/) is an open-source tool for annotating camera trap images. We have worked with the Timelapse developer to integrate the output of our API into Timelapse, so a user can: + +- Select or sort images based on whether they contain people or animals +- View bounding boxes during image annotation (which can speed up review) + +This page contains instructions about how to load our API output into Timelapse. It assumes familiarity with Timelapse, most importantly with the concept of Timlapse templates. + + +# Download the ML-enabled version of Timelapse + +This feature is not in the stable release of Timelapse yet; you can download from (obfuscated URL) or, if you’re feeling ambitious, you can build from source on the [machinelearning-experimental](https://github.com/saulgreenberg/Timelapse/tree/machinelearning-experimental) branch of the Timelapse repo. + + +# Prepare your Timelapse template + +Using the Timelapse template editor, add two fields to your template (which presumably already contains lots of other things specific to your project): + +- Confidence (of type “note”, i.e., string) +- BoundingBoxes (of type “note”, i.e., string) + + + +These fields will be used internally by Timelapse to store the results you load from our API. + +A sample template containing these fields is available [here](MLDebugTemplate.tdb). + + +# Create your Timelapse database + +...exactly the way you would for any other Timelapse project. Specifically, put your .tdb file in the root directory of your project, and load it with file → load template, then let it load all the images (can take a couple hours if you have millions of images). This should create your database (.ddb file). + + +# Prepare API output for Timelapse + +This is a temporary step, used only while we're reconciling the output format expected by Timelapse with the output format currently produced by our API. + +Use the script [prepare_api_output_for_timelapse.py](prepare_api_output_for_timelapse.py). Because this is temporary, I’m not going to document it here, but the script is reasonably well-commented. + + +# Load ML results into Timelapse + +Click recognition → import recognition data, and point it to the Timelapse-ready .csv file. It doesn’t matter where this file is, though it’ probably cleanest to put it in the same directory as your template/database. + +This step can also take a few hours if you have lots of images. + + +# Do useful stuff with your ML results! + +Now that you’ve loaded ML results, there are two major differences in your Timelapse workflow... first, and most obvious, there are bounding boxes around animals: + + + +
This is fun; we love both animals and bounding boxes. But far more important is the fact that you can select images based on whether they contain animals. We recommend the following workflow: + +## Confidence level selection + +Find the confidence threshold that you’re comfortable using to discard images, by choosing select → custom selection → confidence < [some number]. 0.6 is a decent starting point. Note that you need to type 0.6, rather than .6, i.e. numbers other than 1.0 need to include a leading zero. + + + +
Now you should only be seeing images with no animals... if you see animals, something is amiss. You can use the “play forward quickly” button to very rapidly assess whether there are animals hiding here. If you’re feeling comfortable... + +## Labeling + +Change the selection to confidence >= [your threshold]. Now you should be seeing mostly images with animals, though you probably set that threshold low enough that you’re still seeing some empty images. At this point, go about your normal Timelapse business, without wasting all that time on empty images! + + +# In the works... + +Right now animals and people are treated as one entity; we hope to allow selection separately based on animals, people, or both. + + + + + diff --git a/classification/api/api_apply_classifier_single_node.py b/classification/api/api_apply_classifier_single_node.py new file mode 100644 index 000000000..58c542eca --- /dev/null +++ b/classification/api/api_apply_classifier_single_node.py @@ -0,0 +1,269 @@ +###### +# +# detect_and_predict_image.py +# +# Runs both a detector and a classifier on a given image. +# +###### + +#%% Constants, imports, environment + +import tensorflow as tf +import numpy as np +import time +import argparse +import PIL +import json +import humanfriendly + +# Minimum detection confidence for showing a bounding box on the output image +DEFAULT_CONFIDENCE_THRESHOLD = 0.85 + +# Number of top-scoring classes to show at each bounding box +NUM_ANNOTATED_CLASSES = 3 + +# Enlargment factor applied to boxes before passing them to the classifier +# Provides more context and can lead to better results +PADDING_FACTOR = 1.6 + +# List of detection categories, for which we will run the classification +# Currently there are {"1": "animal", "2": "person", "4": "vehicle"} +# Please use strings here +DETECTION_CATEGORY_WHITELIST = ['1'] +assert all([isinstance(x, str) for x in DETECTION_CATEGORY_WHITELIST]) + + +#%% Core detection functions + +def load_model(checkpoint): + """ + Load a detection model (i.e., create a graph) from a .pb file + """ + + print('Creating Graph...') + graph = tf.Graph() + with graph.as_default(): + od_graph_def = tf.GraphDef() + with tf.gfile.GFile(checkpoint, 'rb') as fid: + serialized_graph = fid.read() + od_graph_def.ParseFromString(serialized_graph) + tf.import_graph_def(od_graph_def, name='') + print('...done') + + return graph + + +def add_classification_categories(json_object, classes_file): + ''' + Reads the name of classes from the file *classes_file* and adds them to + the JSON object *json_object*. The function assumes that the first line + corresponds to output no. 0, i.e. we use 0-based indexing. + + Modifies json_object in-place. + + Args: + json_object: an object created from a json in the format of the detection API output + classes_file: the list of classes that correspond to the output elements of the classifier + + Return: + The modified json_object with classification_categories added. If the field 'classification_categories' + already exists, then this function is a no-op. + ''' + + if 'classification_categories' not in json_object.keys(): + + # Read the name of all classes + with open(classes_file, 'rt') as fi: + class_names = fi.read().splitlines() + # remove empty lines + class_names = [cn for cn in class_names if cn.strip()] + + # Create field with name *classification_categories* + json_object['classification_categories'] = dict() + # Add classes using 0-based indexing + for idx, name in enumerate(class_names): + json_object['classification_categories']['%i'%idx] = name + else: + print('WARNING: The input json already contains the list of classification categories.') + + return json_object + + +def classify_boxes(classification_graph, json_with_classes, confidence_threshold=DEFAULT_CONFIDENCE_THRESHOLD, + detection_category_whitelist=DETECTION_CATEGORY_WHITELIST, padding_factor=PADDING_FACTOR, + num_annotated_classes=NUM_ANNOTATED_CLASSES): + ''' + Takes a classification model and applies it to all detected boxes with a detection confidence + larger than confidence_threshold. + + Args: + classification_graph: frozen graph model that includes the TF-slim preprocessing. i.e. it will be given a cropped + images with values in [0,1] + json_with_classes: Object created from the json file that is generated by the detection API. However, the + field 'classification_categories' is already added. The script assumes 0-based indexing. + confidence_threshold: Only classify boxes with a threshold larger than this + detection_category_whitelist : Only boxes with this detection category will be classified + padding_factor: The function will enlarge the bounding boxes by this factor before passing them to the + classifier. + num_annotated_classes: Number of top-scoring class predictions to store in the json + + Returns the updated json object. Classification results are added as field 'classifications' to all elements images/detections + assuming a 0-based indexing of the classifier output, i.e. output with index 0 has the class key '0' + ''' + + # Make sure we have the right json object + assert 'classification_categories' in json_with_classes.keys() + assert isinstance(detection_category_whitelist, list) + assert all([isinstance(x, str) for x in detection_category_whitelist]) + + with classification_graph.as_default(): + + with tf.Session(graph=classification_graph) as sess: + + # Get input and output tensors of classification model + image_tensor = classification_graph.get_tensor_by_name('input:0') + predictions_tensor = classification_graph.get_tensor_by_name('output:0') + predictions_tensor = tf.squeeze(predictions_tensor, [0]) + + # For each image + nImages = len(json_with_classes['images']) + for iImage in range(0,nImages): + + image_description = json_with_classes['images'][iImage] + + # Read image + try: + image_path = image_description['file'] + image_data = np.array(PIL.Image.open(image_path).convert("RGB")) + # Scale pixel values to [0,1] + image_data = image_data / 255 + image_height, image_width, _ = image_data.shape + except KeyboardInterrupt as e: + raise e + except: + print('Couldn\' load image {}'.format(image_path)) + continue + + # For each box + nDetections = len(image_description['detections']) + for iBox in range(nDetections): + + cur_detection = image_description['detections'][iBox] + + # Skip detections with low confidence + if cur_detection['conf'] < confidence_threshold: + continue + + # Skip if detection category is not in whitelist + if not cur_detection['category'] in detection_category_whitelist: + continue + + # Skip if already classified + if 'classifications' in cur_detection.keys() and len(cur_detection['classifications']) > 0: + continue + + # Get current box in relative coordinates and format [ymin, xmin, ymax, xmax] + # Store it as 1x4 numpy array so we can re-use the generic multi-box padding code + box_coords = np.array([cur_detection['bbox']]) + # Convert normalized coordinates to pixel coordinates + box_coords_abs = (box_coords * np.tile([image_height, image_width], (1,2))) + # Pad the detected animal to a square box and additionally by PADDING_FACTOR, the result will be in crop_boxes + # However, we need to make sure that it box coordinates are still within the image + bbox_sizes = np.vstack([box_coords_abs[:,2] - box_coords_abs[:,0], box_coords_abs[:,3] - box_coords_abs[:,1]]).T + offsets = (padding_factor * np.max(bbox_sizes, axis=1, keepdims=True) - bbox_sizes) / 2 + crop_boxes = box_coords_abs + np.hstack([-offsets,offsets]) + crop_boxes = np.maximum(0,crop_boxes).astype(int) + # Get the first (and only) row as our bbox to classify + crop_box = crop_boxes[0] + + # Get the image data for that box + cropped_img = image[crop_box[0]:crop_box[2], crop_box[1]:crop_box[3]] + # Run inference + predictions = sess.run(predictions_tensor, feed_dict={image_tensor: cropped_img}) + species_scores[iImage].append(predictions) + + # Add an empty list to the json for our predictions + cur_detection['classifications'] = list() + # Add the *num_annotated_classes* top scoring classes + for class_idx in np.argsort(-predictions)[:num_annotated_classes]: + cur_detection['classifications'].append(['%i'%class_idx, predictions[class_idx]]) + + # ...for each box + + # species_scores should have shape len(images) x len(boxes) x num_species + assert len(species_scores[iImage]) == len(boxes[iImage]) + + # ...for each image + + # ...with tf.Session + + # with classification_graph + + return json_with_classes + + +def load_and_run_classifier(classifier_file, classes_file, detector_json_file, output_json_file, + confidence_threshold=DEFAULT_CONFIDENCE_THRESHOLD, padding_factor=PADDING_FACTOR, + num_annotated_classes=NUM_ANNOTATED_CLASSES, detection_category_whitelist=DETECTION_CATEGORY_WHITELIST, + detection_graph=None, classification_graph=None): + + # Load classification model + if classification_graph is None: + classification_graph = load_model(classifier_file) + + # Load detector json + with open(detector_json_file, 'rt') as fi: + detector_json = json.load(fi) + + # Add classes to detector_json + updated_json = add_classification_categories(detector_json, classes_file) + + # Run classifier on all images, changes will be writting directly to the json + startTime = time.time() + updated_json = classify_boxes(classification_graph, updated_json, confidence_threshold, detection_category_whitelist, + padding_factor, num_annotated_classes) + elapsed = time.time() - startTime + print("Done running detector and classifier on {} files in {}".format(len(images), + humanfriendly.format_timespan(elapsed))) + + # Write output json + with open(output_json_file, 'wt') as fi: + json.dump(fi, updated_json) + + return detection_graph, classification_graph + + +#%% Command-line driver + +def main(): + parser = argparse.ArgumentParser(description='Applies a classifier to all detected boxes of the detection API output.') + parser.add_argument('classifier_file', type=str, help='Frozen graph for classification including pre-processing. The graphs ' + \ + ' will receive an image with values in [0,1], so double check that you use the correct model. The script ' + \ + ' `export_inference_graph_serengeti.sh` shows how to create such a model', + metavar='PATH_TO_CLASSIFIER_W_PREPROCESSING') + parser.add_argument('classes_file', action='store', type=str, help='File with the class names. Each line should contain ' + \ + ' one name and the first line should correspond to the first output, the second line to the second model output, etc.') + parser.add_argument('detector_json_file', type=str, help='JSON file that was produced by the detection API.') + parser.add_argument('output_json_file', type=str, help='Path to output file, will be in JSON format.') + parser.add_argument('--image_dir', action='store', type=str, default='', help='Directory to search for images, with optional recursion') + parser.add_argument('--threshold', action='store', type=float, default=DEFAULT_CONFIDENCE_THRESHOLD, + help="Confidence threshold, don't render boxes below this confidence. Default: %.2f"%DEFAULT_CONFIDENCE_THRESHOLD) + parser.add_argument('--padding_factor', action='store', type=float, default=PADDING_FACTOR, + help="Enlargement factor for bounding boxes before they are passed to the classifier. Default: %.2f"%PADDING_FACTOR) + parser.add_argument('--num_annotated_classes', action='store', type=int, default=NUM_ANNOTATED_CLASSES, + help='Number of top-scoring classes to add to the output for each bounding box, default: %d'%NUM_ANNOTATED_CLASSES) + parser.add_argument('--detection_category_whitelist', type=str, nargs='+', default=DETECTION_CATEGORY_WHITELIST, + help='We will run the detector on all detections with these detection categories, default: ' + ' '.join(DETECTION_CATEGORY_WHITELIST)) + args = parser.parse_args() + + + load_and_run_classifier(classifier_file=args.classifier_file, classes_file=args.classes_file, + detector_json_file=args.detector_json_file, output_json_file=args.output_json_file, + confidence_threshold=args.threshold, padding_factor=args.padding_factor, + num_annotated_classes=args.num_annotated_classes, detection_category_whitelist=args.detection_category_whitelist) + + + +if __name__ == '__main__': + + main()