| |
| import logging |
| import numpy as np |
| import pprint |
| import sys |
| from collections.abc import Mapping |
|
|
|
|
| def print_csv_format(results): |
| """ |
| Print main metrics in a format similar to Detectron, |
| so that they are easy to copypaste into a spreadsheet. |
| |
| Args: |
| results (OrderedDict[dict]): task_name -> {metric -> score} |
| unordered dict can also be printed, but in arbitrary order |
| """ |
| assert isinstance(results, Mapping) or not len(results), results |
| logger = logging.getLogger(__name__) |
| for task, res in results.items(): |
| if isinstance(res, Mapping): |
| |
| important_res = [(k, v) for k, v in res.items() if "-" not in k] |
| logger.info("copypaste: Task: {}".format(task)) |
| logger.info("copypaste: " + ",".join([k[0] for k in important_res])) |
| logger.info("copypaste: " + ",".join(["{0:.4f}".format(k[1]) for k in important_res])) |
| else: |
| logger.info(f"copypaste: {task}={res}") |
|
|
|
|
| def verify_results(cfg, results): |
| """ |
| Args: |
| results (OrderedDict[dict]): task_name -> {metric -> score} |
| |
| Returns: |
| bool: whether the verification succeeds or not |
| """ |
| expected_results = cfg.TEST.EXPECTED_RESULTS |
| if not len(expected_results): |
| return True |
|
|
| ok = True |
| for task, metric, expected, tolerance in expected_results: |
| actual = results[task].get(metric, None) |
| if actual is None: |
| ok = False |
| continue |
| if not np.isfinite(actual): |
| ok = False |
| continue |
| diff = abs(actual - expected) |
| if diff > tolerance: |
| ok = False |
|
|
| logger = logging.getLogger(__name__) |
| if not ok: |
| logger.error("Result verification failed!") |
| logger.error("Expected Results: " + str(expected_results)) |
| logger.error("Actual Results: " + pprint.pformat(results)) |
|
|
| sys.exit(1) |
| else: |
| logger.info("Results verification passed.") |
| return ok |
|
|
|
|
| def flatten_results_dict(results): |
| """ |
| Expand a hierarchical dict of scalars into a flat dict of scalars. |
| If results[k1][k2][k3] = v, the returned dict will have the entry |
| {"k1/k2/k3": v}. |
| |
| Args: |
| results (dict): |
| """ |
| r = {} |
| for k, v in results.items(): |
| if isinstance(v, Mapping): |
| v = flatten_results_dict(v) |
| for kk, vv in v.items(): |
| r[k + "/" + kk] = vv |
| else: |
| r[k] = v |
| return r |
|
|
