forked from PaddlePaddle/PaddleOCR
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request PaddlePaddle#4204 from LDOUBLEV/fix_eval
[benchmark] add DB training benchmark
- Loading branch information
Showing
7 changed files
with
573 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,273 @@ | ||
| # copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| from __future__ import print_function | ||
|
|
||
| import argparse | ||
| import json | ||
| import os | ||
| import re | ||
| import traceback | ||
|
|
||
|
|
||
| def parse_args(): | ||
| parser = argparse.ArgumentParser(description=__doc__) | ||
| parser.add_argument( | ||
| "--filename", type=str, help="The name of log which need to analysis.") | ||
| parser.add_argument( | ||
| "--log_with_profiler", type=str, help="The path of train log with profiler") | ||
| parser.add_argument( | ||
| "--profiler_path", type=str, help="The path of profiler timeline log.") | ||
| parser.add_argument( | ||
| "--keyword", type=str, help="Keyword to specify analysis data") | ||
| parser.add_argument( | ||
| "--separator", type=str, default=None, help="Separator of different field in log") | ||
| parser.add_argument( | ||
| '--position', type=int, default=None, help='The position of data field') | ||
| parser.add_argument( | ||
| '--range', type=str, default="", help='The range of data field to intercept') | ||
| parser.add_argument( | ||
| '--base_batch_size', type=int, help='base_batch size on gpu') | ||
| parser.add_argument( | ||
| '--skip_steps', type=int, default=0, help='The number of steps to be skipped') | ||
| parser.add_argument( | ||
| '--model_mode', type=int, default=-1, help='Analysis mode, default value is -1') | ||
| parser.add_argument( | ||
| '--ips_unit', type=str, default=None, help='IPS unit') | ||
| parser.add_argument( | ||
| '--model_name', type=str, default=0, help='training model_name, transformer_base') | ||
| parser.add_argument( | ||
| '--mission_name', type=str, default=0, help='training mission name') | ||
| parser.add_argument( | ||
| '--direction_id', type=int, default=0, help='training direction_id') | ||
| parser.add_argument( | ||
| '--run_mode', type=str, default="sp", help='multi process or single process') | ||
| parser.add_argument( | ||
| '--index', type=int, default=1, help='{1: speed, 2:mem, 3:profiler, 6:max_batch_size}') | ||
| parser.add_argument( | ||
| '--gpu_num', type=int, default=1, help='nums of training gpus') | ||
| args = parser.parse_args() | ||
| args.separator = None if args.separator == "None" else args.separator | ||
| return args | ||
|
|
||
|
|
||
| def _is_number(num): | ||
| pattern = re.compile(r'^[-+]?[-0-9]\d*\.\d*|[-+]?\.?[0-9]\d*$') | ||
| result = pattern.match(num) | ||
| if result: | ||
| return True | ||
| else: | ||
| return False | ||
|
|
||
|
|
||
| class TimeAnalyzer(object): | ||
| def __init__(self, filename, keyword=None, separator=None, position=None, range="-1"): | ||
| if filename is None: | ||
| raise Exception("Please specify the filename!") | ||
|
|
||
| if keyword is None: | ||
| raise Exception("Please specify the keyword!") | ||
|
|
||
| self.filename = filename | ||
| self.keyword = keyword | ||
| self.separator = separator | ||
| self.position = position | ||
| self.range = range | ||
| self.records = None | ||
| self._distil() | ||
|
|
||
| def _distil(self): | ||
| self.records = [] | ||
| with open(self.filename, "r") as f_object: | ||
| lines = f_object.readlines() | ||
| for line in lines: | ||
| if self.keyword not in line: | ||
| continue | ||
| try: | ||
| result = None | ||
|
|
||
| # Distil the string from a line. | ||
| line = line.strip() | ||
| line_words = line.split(self.separator) if self.separator else line.split() | ||
| if args.position: | ||
| result = line_words[self.position] | ||
| else: | ||
| # Distil the string following the keyword. | ||
| for i in range(len(line_words) - 1): | ||
| if line_words[i] == self.keyword: | ||
| result = line_words[i + 1] | ||
| break | ||
|
|
||
| # Distil the result from the picked string. | ||
| if not self.range: | ||
| result = result[0:] | ||
| elif _is_number(self.range): | ||
| result = result[0: int(self.range)] | ||
| else: | ||
| result = result[int(self.range.split(":")[0]): int(self.range.split(":")[1])] | ||
| self.records.append(float(result)) | ||
| except Exception as exc: | ||
| print("line is: {}; separator={}; position={}".format(line, self.separator, self.position)) | ||
|
|
||
| print("Extract {} records: separator={}; position={}".format(len(self.records), self.separator, self.position)) | ||
|
|
||
| def _get_fps(self, mode, batch_size, gpu_num, avg_of_records, run_mode, unit=None): | ||
| if mode == -1 and run_mode == 'sp': | ||
| assert unit, "Please set the unit when mode is -1." | ||
| fps = gpu_num * avg_of_records | ||
| elif mode == -1 and run_mode == 'mp': | ||
| assert unit, "Please set the unit when mode is -1." | ||
| fps = gpu_num * avg_of_records #temporarily, not used now | ||
| print("------------this is mp") | ||
| elif mode == 0: | ||
| # s/step -> samples/s | ||
| fps = (batch_size * gpu_num) / avg_of_records | ||
| unit = "samples/s" | ||
| elif mode == 1: | ||
| # steps/s -> steps/s | ||
| fps = avg_of_records | ||
| unit = "steps/s" | ||
| elif mode == 2: | ||
| # s/step -> steps/s | ||
| fps = 1 / avg_of_records | ||
| unit = "steps/s" | ||
| elif mode == 3: | ||
| # steps/s -> samples/s | ||
| fps = batch_size * gpu_num * avg_of_records | ||
| unit = "samples/s" | ||
| elif mode == 4: | ||
| # s/epoch -> s/epoch | ||
| fps = avg_of_records | ||
| unit = "s/epoch" | ||
| else: | ||
| ValueError("Unsupported analysis mode.") | ||
|
|
||
| return fps, unit | ||
|
|
||
| def analysis(self, batch_size, gpu_num=1, skip_steps=0, mode=-1, run_mode='sp', unit=None): | ||
| if batch_size <= 0: | ||
| print("base_batch_size should larger than 0.") | ||
| return 0, '' | ||
|
|
||
| if len(self.records) <= skip_steps: # to address the condition which item of log equals to skip_steps | ||
| print("no records") | ||
| return 0, '' | ||
|
|
||
| sum_of_records = 0 | ||
| sum_of_records_skipped = 0 | ||
| skip_min = self.records[skip_steps] | ||
| skip_max = self.records[skip_steps] | ||
|
|
||
| count = len(self.records) | ||
| for i in range(count): | ||
| sum_of_records += self.records[i] | ||
| if i >= skip_steps: | ||
| sum_of_records_skipped += self.records[i] | ||
| if self.records[i] < skip_min: | ||
| skip_min = self.records[i] | ||
| if self.records[i] > skip_max: | ||
| skip_max = self.records[i] | ||
|
|
||
| avg_of_records = sum_of_records / float(count) | ||
| avg_of_records_skipped = sum_of_records_skipped / float(count - skip_steps) | ||
|
|
||
| fps, fps_unit = self._get_fps(mode, batch_size, gpu_num, avg_of_records, run_mode, unit) | ||
| fps_skipped, _ = self._get_fps(mode, batch_size, gpu_num, avg_of_records_skipped, run_mode, unit) | ||
| if mode == -1: | ||
| print("average ips of %d steps, skip 0 step:" % count) | ||
| print("\tAvg: %.3f %s" % (avg_of_records, fps_unit)) | ||
| print("\tFPS: %.3f %s" % (fps, fps_unit)) | ||
| if skip_steps > 0: | ||
| print("average ips of %d steps, skip %d steps:" % (count, skip_steps)) | ||
| print("\tAvg: %.3f %s" % (avg_of_records_skipped, fps_unit)) | ||
| print("\tMin: %.3f %s" % (skip_min, fps_unit)) | ||
| print("\tMax: %.3f %s" % (skip_max, fps_unit)) | ||
| print("\tFPS: %.3f %s" % (fps_skipped, fps_unit)) | ||
| elif mode == 1 or mode == 3: | ||
| print("average latency of %d steps, skip 0 step:" % count) | ||
| print("\tAvg: %.3f steps/s" % avg_of_records) | ||
| print("\tFPS: %.3f %s" % (fps, fps_unit)) | ||
| if skip_steps > 0: | ||
| print("average latency of %d steps, skip %d steps:" % (count, skip_steps)) | ||
| print("\tAvg: %.3f steps/s" % avg_of_records_skipped) | ||
| print("\tMin: %.3f steps/s" % skip_min) | ||
| print("\tMax: %.3f steps/s" % skip_max) | ||
| print("\tFPS: %.3f %s" % (fps_skipped, fps_unit)) | ||
| elif mode == 0 or mode == 2: | ||
| print("average latency of %d steps, skip 0 step:" % count) | ||
| print("\tAvg: %.3f s/step" % avg_of_records) | ||
| print("\tFPS: %.3f %s" % (fps, fps_unit)) | ||
| if skip_steps > 0: | ||
| print("average latency of %d steps, skip %d steps:" % (count, skip_steps)) | ||
| print("\tAvg: %.3f s/step" % avg_of_records_skipped) | ||
| print("\tMin: %.3f s/step" % skip_min) | ||
| print("\tMax: %.3f s/step" % skip_max) | ||
| print("\tFPS: %.3f %s" % (fps_skipped, fps_unit)) | ||
|
|
||
| return round(fps_skipped, 3), fps_unit | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| args = parse_args() | ||
| run_info = dict() | ||
| run_info["log_file"] = args.filename | ||
| run_info["model_name"] = args.model_name | ||
| run_info["mission_name"] = args.mission_name | ||
| run_info["direction_id"] = args.direction_id | ||
| run_info["run_mode"] = args.run_mode | ||
| run_info["index"] = args.index | ||
| run_info["gpu_num"] = args.gpu_num | ||
| run_info["FINAL_RESULT"] = 0 | ||
| run_info["JOB_FAIL_FLAG"] = 0 | ||
|
|
||
| try: | ||
| if args.index == 1: | ||
| if args.gpu_num == 1: | ||
| run_info["log_with_profiler"] = args.log_with_profiler | ||
| run_info["profiler_path"] = args.profiler_path | ||
| analyzer = TimeAnalyzer(args.filename, args.keyword, args.separator, args.position, args.range) | ||
| run_info["FINAL_RESULT"], run_info["UNIT"] = analyzer.analysis( | ||
| batch_size=args.base_batch_size, | ||
| gpu_num=args.gpu_num, | ||
| skip_steps=args.skip_steps, | ||
| mode=args.model_mode, | ||
| run_mode=args.run_mode, | ||
| unit=args.ips_unit) | ||
| try: | ||
| if int(os.getenv('job_fail_flag')) == 1 or int(run_info["FINAL_RESULT"]) == 0: | ||
| run_info["JOB_FAIL_FLAG"] = 1 | ||
| except: | ||
| pass | ||
| elif args.index == 3: | ||
| run_info["FINAL_RESULT"] = {} | ||
| records_fo_total = TimeAnalyzer(args.filename, 'Framework overhead', None, 3, '').records | ||
| records_fo_ratio = TimeAnalyzer(args.filename, 'Framework overhead', None, 5).records | ||
| records_ct_total = TimeAnalyzer(args.filename, 'Computation time', None, 3, '').records | ||
| records_gm_total = TimeAnalyzer(args.filename, 'GpuMemcpy Calls', None, 4, '').records | ||
| records_gm_ratio = TimeAnalyzer(args.filename, 'GpuMemcpy Calls', None, 6).records | ||
| records_gmas_total = TimeAnalyzer(args.filename, 'GpuMemcpyAsync Calls', None, 4, '').records | ||
| records_gms_total = TimeAnalyzer(args.filename, 'GpuMemcpySync Calls', None, 4, '').records | ||
| run_info["FINAL_RESULT"]["Framework_Total"] = records_fo_total[0] if records_fo_total else 0 | ||
| run_info["FINAL_RESULT"]["Framework_Ratio"] = records_fo_ratio[0] if records_fo_ratio else 0 | ||
| run_info["FINAL_RESULT"]["ComputationTime_Total"] = records_ct_total[0] if records_ct_total else 0 | ||
| run_info["FINAL_RESULT"]["GpuMemcpy_Total"] = records_gm_total[0] if records_gm_total else 0 | ||
| run_info["FINAL_RESULT"]["GpuMemcpy_Ratio"] = records_gm_ratio[0] if records_gm_ratio else 0 | ||
| run_info["FINAL_RESULT"]["GpuMemcpyAsync_Total"] = records_gmas_total[0] if records_gmas_total else 0 | ||
| run_info["FINAL_RESULT"]["GpuMemcpySync_Total"] = records_gms_total[0] if records_gms_total else 0 | ||
| else: | ||
| print("Not support!") | ||
| except Exception: | ||
| traceback.print_exc() | ||
| print("{}".format(json.dumps(run_info))) # it's required, for the log file path insert to the database | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
|
|
||
| # PaddleOCR DB/EAST 算法训练benchmark测试 | ||
|
|
||
| PaddleOCR/benchmark目录下的文件用于获取并分析训练日志。 | ||
| 训练采用icdar2015数据集，包括1000张训练图像和500张测试图像。模型配置采用resnet18_vd作为backbone，分别训练batch_size=8和batch_size=16的情况。 | ||
|
|
||
| ## 运行训练benchmark | ||
|
|
||
| benchmark/run_det.sh 中包含了三个过程： | ||
| - 安装依赖 | ||
| - 下载数据 | ||
| - 执行训练 | ||
| - 日志分析获取IPS | ||
|
|
||
| 在执行训练部分，会执行单机单卡（默认0号卡）单机多卡训练，并分别执行batch_size=8和batch_size=16的情况。所以执行完后，每种模型会得到4个日志文件。 | ||
|
|
||
| run_det.sh 执行方式如下: | ||
|
|
||
| ``` | ||
| # cd PaddleOCR/ | ||
| bash benchmark/run_det.sh | ||
| ``` | ||
|
|
||
| 以DB为例，将得到四个日志文件，如下： | ||
| ``` | ||
| det_res18_db_v2.0_sp_bs16_fp32_1 | ||
| det_res18_db_v2.0_sp_bs8_fp32_1 | ||
| det_res18_db_v2.0_mp_bs16_fp32_1 | ||
| det_res18_db_v2.0_mp_bs8_fp32_1 | ||
| ``` | ||
|
|
||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.