#!/usr/bin/env python import numpy as np from shapely.geometry import Polygon """ reference from : https://github.com/MhLiao/DB/blob/3c32b808d4412680310d3d28eeb6a2d5bf1566c5/concern/icdar2015_eval/detection/iou.py#L8 """ class DetectionIoUEvaluator(object): def __init__(self, iou_constraint=0.5, area_precision_constraint=0.5): self.iou_constraint = iou_constraint self.area_precision_constraint = area_precision_constraint def evaluate_image(self, gt, pred): def get_union(pD, pG): return Polygon(pD).union(Polygon(pG)).area def get_intersection_over_union(pD, pG): return get_intersection(pD, pG) / get_union(pD, pG) def get_intersection(pD, pG): return Polygon(pD).intersection(Polygon(pG)).area def compute_ap(confList, matchList, numGtCare): correct = 0 AP = 0 if len(confList) > 0: confList = np.array(confList) matchList = np.array(matchList) sorted_ind = np.argsort(-confList) confList = confList[sorted_ind] matchList = matchList[sorted_ind] for n in range(len(confList)): match = matchList[n] if match: correct += 1 AP += float(correct) / (n + 1) if numGtCare > 0: AP /= numGtCare return AP perSampleMetrics = {} matchedSum = 0 numGlobalCareGt = 0 numGlobalCareDet = 0 precision = 0 detMatched = 0 iouMat = np.empty([1, 1]) gtPols = [] detPols = [] gtPolPoints = [] detPolPoints = [] # Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [] # Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [] pairs = [] detMatchedNums = [] evaluationLog = '' for n in range(len(gt)): points = gt[n]['points'] dontCare = gt[n]['ignore'] if not Polygon(points).is_valid: continue gtPol = points gtPols.append(gtPol) gtPolPoints.append(points) if dontCare: gtDontCarePolsNum.append(len(gtPols) - 1) evaluationLog += ( 'GT polygons: ' + str(len(gtPols)) + (' (' + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else '\n')) for n in range(len(pred)): points = pred[n]['points'] if not Polygon(points).is_valid: continue detPol = points detPols.append(detPol) detPolPoints.append(points) if len(gtDontCarePolsNum) > 0: for dontCarePol in gtDontCarePolsNum: dontCarePol = gtPols[dontCarePol] intersected_area = get_intersection(dontCarePol, detPol) pdDimensions = Polygon(detPol).area precision = (0 if pdDimensions == 0 else intersected_area / pdDimensions) if precision > self.area_precision_constraint: detDontCarePolsNum.append(len(detPols) - 1) break evaluationLog += ( 'DET polygons: ' + str(len(detPols)) + (' (' + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else '\n')) if len(gtPols) > 0 and len(detPols) > 0: # Calculate IoU and precision matrixs outputShape = [len(gtPols), len(detPols)] iouMat = np.empty(outputShape) gtRectMat = np.zeros(len(gtPols), np.int8) detRectMat = np.zeros(len(detPols), np.int8) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): pG = gtPols[gtNum] pD = detPols[detNum] iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG) for gtNum in range(len(gtPols)): for detNum in range(len(detPols)): if (gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum): if iouMat[gtNum, detNum] > self.iou_constraint: gtRectMat[gtNum] = 1 detRectMat[detNum] = 1 detMatched += 1 pairs.append({'gt': gtNum, 'det': detNum}) detMatchedNums.append(detNum) evaluationLog += ('Match GT #' + str(gtNum) + ' with Det #' + str(detNum) + '\n') numGtCare = len(gtPols) - len(gtDontCarePolsNum) numDetCare = len(detPols) - len(detDontCarePolsNum) if numGtCare == 0: precision = float(0) if numDetCare > 0 else float(1) else: precision = 0 if numDetCare == 0 else float( detMatched) / numDetCare matchedSum += detMatched numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare perSampleMetrics = { 'gtCare': numGtCare, 'detCare': numDetCare, 'detMatched': detMatched, } return perSampleMetrics def combine_results(self, results): numGlobalCareGt = 0 numGlobalCareDet = 0 matchedSum = 0 for result in results: numGlobalCareGt += result['gtCare'] numGlobalCareDet += result['detCare'] matchedSum += result['detMatched'] methodRecall = (0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt) methodPrecision = (0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet) methodHmean = (0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)) methodMetrics = { 'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, } return methodMetrics if __name__ == '__main__': evaluator = DetectionIoUEvaluator() gts = [[ { 'points': [(0, 0), (1, 0), (1, 1), (0, 1)], 'text': 1234, 'ignore': False, }, { 'points': [(2, 2), (3, 2), (3, 3), (2, 3)], 'text': 5678, 'ignore': False, }, ]] preds = [[{ 'points': [(0.1, 0.1), (1, 0), (1, 1), (0, 1)], 'text': 123, 'ignore': False, }]] results = [] for gt, pred in zip(gts, preds): results.append(evaluator.evaluate_image(gt, pred)) metrics = evaluator.combine_results(results) print(metrics)