a
    ¡˜ëj[y  ã                   @   s   d Z ddlZddlmZ ddlmZ ddlZddlZe 	g d¢¡Z
dZe ddd¡ZeG d	d
„ d
ƒƒZeG dd„ dƒƒZG dd„ dƒZedk�rddlmZ edƒedƒgZedƒedƒgZeeeƒD ]T\ZZe ee¡Ze ¡ \ZZedej› deeƒ› �ƒ edej› deeƒ› �ƒ qÆdS )zZ
Implements object detection metrics: average precision, precision, recall, and f1 score.
é    N)Ú	dataclass)ÚPath)
g      à?gš™™™™™á?g333333ã?gÍÌÌÌÌÌä?gffffffæ?g      è?gš™™™™™é?g333333ë?gÍÌÌÌÌÌì?gffffffî?gš™™™™™¹?g)\�Âõ(ð?g{®Gáz„?c                   @   s2   e Zd ZU dZeed< eed< eed< eed< dS )Ú#ObjectDetectionAggregatedEvaluationzGClass representing a gathered class-aggregated object detection metricsÚf1_scoreÚ	precisionÚrecallÚm_apN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚfloatÚ__annotations__© r   r   úd/mnt/bigstor/www/usecases/.venv/lib/python3.9/site-packages/unstructured/metrics/object_detection.pyr      s
   
r   c                   @   s^   e Zd ZU dZeeef ed< eeef ed< eeef ed< eeef ed< edd„ ƒZ	dS )	Ú!ObjectDetectionPerClassEvaluationz@Class representing a gathered object detection metrics per-classr   r   r   r   c                    s~   ‡‡fdd„t tˆƒƒD ƒ}‡‡fdd„t tˆƒƒD ƒ‰‡‡fdd„t tˆƒƒD ƒ‰‡ ‡fdd„t tˆƒƒD ƒ}| |ˆˆ|ƒS )Nc                    s   i | ]}ˆ | ˆ| “qS r   r   ©Ú.0Úi)Úclass_labelsÚf1r   r   Ú
<dictcomp>(   ó    zBObjectDetectionPerClassEvaluation.from_tensors.<locals>.<dictcomp>c                    s   i | ]}ˆ | ˆ| “qS r   r   r   )r   r   r   r   r   )   r   c                    s   i | ]}ˆ | ˆ| “qS r   r   r   )r   r   r   r   r   *   r   c                    s   i | ]}ˆ| ˆ | “qS r   r   r   )Úapr   r   r   r   +   r   )ÚrangeÚlen)Úclsr   r   r   r   r   r   r   r   )r   r   r   r   r   r   Úfrom_tensors&   s
    z.ObjectDetectionPerClassEvaluation.from_tensorsN)
r	   r
   r   r   ÚdictÚstrr   r   Úclassmethodr   r   r   r   r   r      s   
r   c                
   @   sœ  e Zd ZeZeZeZd*e	e
j e	e
j e	e e	e e	e edœdd„Zeeed dœdd„ƒZeeef dœd	d
„Zeeee	e	f dœdd„ƒZed+eee	e dœdd„ƒZee
je
jee
jdœdd„ƒZeejeeef ejdœdd„ƒZee
je
je
jdœdd„ƒZe
je
je
je
je
je
je
je
je
jdœ	dd„Z d,e
je
jeeeeed œd!d"„Z!e
je
je
je
je
jed#œd$d%„Z"e
je
je
jee
je#d&œd'd(„Z$d)S )-ÚObjectDetectionEvalProcessorÚcpu)Údocument_predsÚdocument_targetsÚpages_heightÚpages_widthr   Údevicec                    sD   ˆ | _ ‡ fdd„|D ƒ| _‡ fdd„|D ƒ| _|| _|| _|| _dS )a¾  
        Initializes the ObjectDetection prediction and ground truth.

        Args:
            document_preds (list):      list (of length pages of document) of
                                        Tensors of shape (num_predictions, 6)
                                        format: (x1, y1, x2, y2, confidence,class_label)
                                        where x1,y1,x2,y2 are according to image size
            document_targets (list):    list (of length pages of document) of
                                        Tensors of shape (num_targets, 6)
                                        format: (label, x1, y1, x2, y2)
                                        where x,y,w,h are according to image size
            pages_height (list):        list of height of each page in the document
            pages_width (list):         list of width of each page in the document
            class_labels (list):        list of class labels
        c                    s   g | ]}|  ˆ ¡‘qS r   ©Úto)r   Úpred©r'   r   r   Ú
<listcomp>O   r   z9ObjectDetectionEvalProcessor.__init__.<locals>.<listcomp>c                    s   g | ]}|  ˆ ¡‘qS r   r(   )r   Útargetr+   r   r   r,   P   r   N)r'   r#   r$   r%   r&   r   )Úselfr#   r$   r%   r&   r   r'   r   r+   r   Ú__init__5   s    z%ObjectDetectionEvalProcessor.__init__)Úprediction_file_pathÚground_truth_file_pathÚreturnc                 C   sÊ  t |ƒ�}t |¡}W d  ƒ n1 s(0    Y  t |ƒ�}t |¡}W d  ƒ n1 sZ0    Y  t|d ƒt|d ƒks„J dƒ‚t|d ƒt|d ƒks¤J dƒ‚tt|d dd„ d�t|d d	d„ d�ƒD ]¸\}}|d
 |d
 k�sJ d|j› d|d
 › d|j› d|d
 › d�	ƒ‚|d
 }|d |d ksÎJ d|j› d|d d › d|d d › d|j› d|d d › d|d d › d|› d�ƒ‚qÎ|d }	| j||	dd�}
|  ||	¡}|  |¡\}}| |
||||	ƒS )aI  
        Initializes the ObjectDetection prediction and ground truth,
        and converts the data to the required format.

        Args:
            prediction_file_path (Path): path to json file with predictions dump from OD model
            ground_truth_file_path (Path): path to json file with OD ground truth data
        NZobject_detection_classesz5Classes in predictions and ground truth do not match.Úpagesz:Pages number in predictions and ground truth do not match.c                 S   s   | d S ©NÚnumberr   ©Úpr   r   r   Ú<lambda>q   r   z>ObjectDetectionEvalProcessor.from_json_files.<locals>.<lambda>)Úkeyc                 S   s   | d S r4   r   r6   r   r   r   r8   r   r   r5   zPage numbers in predictions z (z) and ground truth z) do not match.ÚsizezPage sizes in predictions r   z x é   z) do not match for page Ú.T)Ú
prediction)	ÚopenÚjsonÚloadÚsortedr   ÚzipÚnameÚ_process_dataÚ_parse_page_dimensions)r   r0   r1   ÚfZpredictions_dataZground_truth_dataZ	pred_pageZgt_pageZpage_numr   r#   r$   r%   r&   r   r   r   Úfrom_json_filesU   s`    
(
(ÿþÿþþ
ÿÿþÿ

ÿ
ÿþ
þ
ýýÿz,ObjectDetectionEvalProcessor.from_json_files)r2   c                 C   s¤  g }t | j| j| j| jƒD ](\}}}}| j||||d�}| |¡ qd\}}}	}
t| jƒ}t	 
|t	j¡}t	 
|t	j¡}t	 
|t	j¡}t	 
|t	j¡}t|ƒ�rfdd„ tt |Ž ƒD ƒ}| j|Ž \}}}}}| ¡ | ¡ | ¡   }}	}
| ¡ }| d¡}| d¡}| d¡}| d¡}t|ƒD ]J\}}t|| ƒ||< t|| ƒ||< t|| ƒ||< t|| ƒ||< �qtj||||| jd�}tt|
ƒt|ƒt|	ƒt|ƒd�}||fS )z­Get per document OD metrics.

        Returns:
            tuple: Tuple of ObjectDetectionAggregatedEvaluation and
                ObjectDetectionPerClassEvaluation
        )ÚpredsÚtargetsÚheightÚwidth)ç      ð¿rL   rL   rL   c                 S   s   g | ]}t  |d ¡‘qS )r   )ÚtorchÚcat)r   Úxr   r   r   r,   ¯   r   z<ObjectDetectionEvalProcessor.get_metrics.<locals>.<listcomp>r;   )r   r   r   r   r   )r   r   r   r   )rB   r#   r$   r%   r&   Ú _compute_page_detection_matchingÚappendr   r   ÚnpÚfullÚnanÚlistÚ_compute_detection_metricsÚmeanÚ	enumerater   r   r   r   )r.   Zdocument_matchingsrH   rI   rJ   rK   Zpage_matching_tensorsZmean_apZmean_precisionZmean_recallZmean_f1Znum_clsZmean_ap_per_classZmean_precision_per_classZmean_recall_per_classZmean_f1_per_classZmatching_info_tensorsZap_per_present_classesZprecision_per_present_classesZrecall_per_present_classesZf1_per_present_classesZpresent_classesZap_per_classZprecision_per_classZrecall_per_classZf1_per_classr   Zclass_indexZod_per_class_evaluationZod_evaluationr   r   r   Úget_metricsŠ   sp    	ÿü

	ÿú	ý




ûüz(ObjectDetectionEvalProcessor.get_metrics)Údatar2   c                 C   sB   g }g }| d D ](}|  |d d ¡ |  |d d ¡ q||fS )zX
        Process the page dimensions from the json file to the required format.
        r3   r:   rJ   rK   )rQ   )rZ   r%   r&   Úpager   r   r   rE   æ   s    z3ObjectDetectionEvalProcessor._parse_page_dimensionsF)rZ   r=   r2   c              
   C   s–   g }| d D ]„}g }|d D ]^}|d }|  |¡}|d \}	}
}}|rf|d }| |	|
||||g¡ q| ||	|
||g¡ qt |¡}| |¡ q|S )zQ
        Process the elements from the json file to the required format.
        r3   ÚelementsÚtypeZbboxZprob)ÚindexrQ   rM   Útensor)rZ   r   r=   Z
pages_listr[   Zpage_elementsÚelementZclass_labelZ	class_idxÚx1Úy1Zx2Úy2Ú
confidenceZpage_tensorr   r   r   rD   ò   s    

z*ObjectDetectionEvalProcessor._process_data)Úpreds_scoresÚ	preds_clsÚtop_kr2   c           
      C   sŒ   t  |¡}| dd¡t j|d | jd� dd¡k}|  dd¡| }|jddd�\}}|d|…dd…f jdd	�}||jddd
� }	|	 d¡S )aÂ  
        Get the indexes of all the top k predictions for every class

        Args:
            preds_scores:   The confidence scores, vector of shape (n_pred)
            preds_cls:      The predicted class, vector of shape (n_pred)
            top_k:          Number of predictions to keep per class, ordered by confidence score

        Returns:
            top_k_idx:     Indexes of the top k predictions. length <= (k * n_unique_class)
        éÿÿÿÿr;   r+   r   T©Ú
descendingNF©Úas_tuple©Údim)rM   ÚmaxÚviewÚaranger'   ÚsortÚnonzeroÚsplit)
re   rf   rg   Zn_unique_clsÚmaskZpreds_scores_per_clsZsorted_scores_per_clsZsorting_idxZidx_with_satisfying_scoresZ	top_k_idxr   r   r   Ú_get_top_k_idx_per_cls	  s    

ÿþz3ObjectDetectionEvalProcessor._get_top_k_idx_per_cls)ÚboxesÚ	img_shaper2   c                 C   sX   | dddgf j d|d d�| dddgf< | dddgf j d|d d�| dddgf< | S )a*  
        Clips bboxes to image boundaries.

        Args:
            bboxes:         Input bounding boxes in XYXY format of [..., 4] shape
            img_shape:      Image shape (height, width).
        Returns:
            clipped_boxes:  Clipped bboxes in XYXY format of [..., 4] shape
        .r   é   r;   )Úminro   é   )Zclip)rw   rx   r   r   r   Ú"_change_bbox_bounds_for_image_size$  s    **z?ObjectDetectionEvalProcessor._change_bbox_bounds_for_image_size)Úbox1Úbox2r2   c                 C   s    dd„ }|| j ƒ}||j ƒ}t | dd…ddd…f |dd…dd…f ¡t | dd…ddd…f |dd…dd…f ¡  d¡ d¡}||dd…df | |  S )a   
        Return intersection-over-union (Jaccard index) of boxes.
        Both sets of boxes are expected to be in (x1, y1, x2, y2) format.

        Args:
            box1: Tensor of shape [N, 4]
            box2: Tensor of shape [M, 4]

        Returns:
            iou:    Tensor of shape [N, M]: the NxM matrix containing the pairwise IoU values
                    for every element in boxes1 and boxes2
        c                 S   s    | d | d  | d | d   S )Nry   r   r{   r;   r   )Zboxr   r   r   Úbox_areaF  s    z7ObjectDetectionEvalProcessor._box_iou.<locals>.box_areaNry   r   )ÚTrM   rz   ro   ÚclampÚprod)r}   r~   r   Zarea1Zarea2Úinterr   r   r   Ú_box_iou6  s    

\ÿþÿz%ObjectDetectionEvalProcessor._box_iou)	Úpreds_box_xyxyrf   Útargets_box_xyxyÚtargets_clsÚpreds_matchedÚtargets_matchedÚpreds_idx_to_useÚiou_thresholdsr2   c	                 C   sä   |   || |¡}	||  dd¡| dd¡k}
d|	|
< |	jddd�\}}||d kjdd�D ]„\}}|| }|||f }|||f |k}t ||dd…f  ||dd…f  ¡}t ||¡}d|||f< d|||f< | ¡ rZ qàqZ|S )	a¼  
        Computes the matching targets based on IoU for regular scenarios.

        Args:
            preds_box_xyxy: (torch.Tensor) Predicted bounding boxes in XYXY format.
            preds_cls: (torch.Tensor) Predicted classes.
            targets_box_xyxy: (torch.Tensor) Target bounding boxes in XYXY format.
            targets_cls: (torch.Tensor) Target classes.
            preds_matched: (torch.Tensor) Tensor indicating which predictions are matched.
            targets_matched: (torch.Tensor) Tensor indicating which targets are matched.
            preds_idx_to_use: (torch.Tensor) Indices of predictions to use.

        Returns:
            targets: Computed matching targets.
        rh   r;   r   T)rj   ZstableFrk   N)r„   rp   rr   rs   rM   Úlogical_andÚall)r.   r…   rf   r†   r‡   rˆ   r‰   rŠ   r‹   ZiouZcls_mismatchZ
sorted_iouZtarget_sortedZpred_selected_iZtarget_sorted_iZpred_iZtarget_iZis_iou_above_thresholdZare_candidates_freeZare_candidates_goodr   r   r   Ú_compute_targetsU  s&    ÿ ÿz-ObjectDetectionEvalProcessor._compute_targetséd   T)rH   rI   rJ   rK   rg   Úreturn_on_cpur2   c              
   C   sÆ  | j j| jd�}t|ƒ}|du s,t|ƒdkr¬tjd|ftj| jd�}	tjd|ftj| jd�}
tjg tj| jd�}tjg tj| jd�}|dd…df j| jd�}|	|
|||fS tjt|ƒ|tj| jd�}	tjt|ƒ|tj| jd�}tjt|ƒ|tj| jd�}
|dd…df |dd…dd…f |dd…df   }}}|dd…df |dd…dd…f  }}|  	|||¡}d	|
dd…dd…f< d
|
|< t|ƒdk�r¸|  
|||f¡ |  |||||	|||¡}	|	|
|||fS )aC  
        Match predictions (NMS output) and the targets (ground truth) with respect to metric
        and confidence score for a given image.

        Args:
            preds:          Tensor of shape (num_img_predictions, 6)
                            format: (x1, y1, x2, y2, confidence, class_label)
                            where x1,y1,x2,y2 are according to image size
            targets:        targets for this image of shape (num_img_targets, 5)
                            format:     (label, x1, y1, x2, y2)
                            where x1,y1,x2,y2 are according to image size
            height:         dimensions of the image
            width:          dimensions of the image
            top_k:          Number of predictions to keep per class, ordered by confidence score
            return_on_cpu:  If True, the output will be returned on "CPU", otherwise it will be
                            returned on "device"

        Returns:
            preds_matched:      Tensor of shape (num_img_predictions, n_thresholds)
                                True when prediction (i) is matched with a target with respect to
                                the (j)th threshold
            preds_to_ignore:    Tensor of shape (num_img_predictions, n_thresholds)
                                True when prediction (i) is matched with a crowd target with
                                respect to the (j)th threshold
            preds_scores:       Tensor of shape (num_img_predictions),
                                confidence score for every prediction
            preds_cls:          Tensor of shape (num_img_predictions),
                                predicted class for every prediction
            targets_cls:        Tensor of shape (num_img_targets),
                                ground truth class for every target
        r+   Nr   )Údtyper'   rh   é   r;   é   TF)r‹   r)   r'   r   rM   ÚzerosÚboolr_   Úfloat32rv   r|   rŽ   )r.   rH   rI   rJ   rK   rg   r�   Z
thresholdsZnum_thresholdsrˆ   Úpreds_to_ignorere   rf   r‡   r‰   Z	preds_boxZtargets_boxrŠ   r   r   r   rP      sH    )ÿÿÿ8&øz=ObjectDetectionEvalProcessor._compute_page_detection_matching)rˆ   r—   re   rf   r‡   r2   c              	   C   sZ  |  | j¡|  | j¡ }}|  | j¡|  | j¡|  | j¡  }}}| j  | j¡}| j}t |¡ ¡ }t|ƒ|jd  }	}
tj	|	|
f| jd�}tj	|	|
f| jd�}tj	|	|
f| jd�}t
|ƒD ]v\}}||k||k }}| j|| || || | ¡ ||d�\}}}|||dd…f< |||dd…f< |||dd…f< q¼d| | || d  }|||||fS )aœ  
        Compute the list of precision, recall, MaP and f1 for every class.

        Args:
            preds_matched:      Tensor of shape (num_predictions, n_iou_thresholds)
                                True when prediction (i) is matched with a target with respect
                                to the (j)th IoU threshold
            preds_to_ignore     Tensor of shape (num_predictions, n_iou_thresholds)
                                True when prediction (i) is matched with a crowd target with
                                respect to the (j)th IoU threshold
            preds_scores:       Tensor of shape (num_predictions),
                                confidence score for every prediction
            preds_cls:          Tensor of shape (num_predictions),
                                predicted class for every prediction
            targets_cls:        Tensor of shape (num_targets),
                                ground truth class for every target box to be detected

        Returns:
            ap, precision, recall, f1:  Tensors of shape (n_class, nb_iou_thrs)
            unique_classes:             Vector with all unique target classes
        rh   r+   ©rˆ   r—   re   Ú	n_targetsÚrecall_thresholdsÚscore_thresholdNry   g¼‰Ø—²Òœ<)r)   r'   rš   r›   rM   ÚuniqueÚlongr   Úshaper”   rX   Ú"_compute_detection_metrics_per_clsÚsum)r.   rˆ   r—   re   rf   r‡   rš   r›   Zunique_classesZn_classÚnb_iou_thrsr   r   r   Zcls_iZclass_valueZcls_preds_idxZcls_targets_idxZcls_apZcls_precisionZ
cls_recallr   r   r   r   rV   ö  sF    ÿ


ý
ÿúü	z7ObjectDetectionEvalProcessor._compute_detection_metricsr˜   c                 C   sä  |j d }|}t t |¡t |¡¡}	t|ƒdkr`tj|| jd�tj|| jd�tj|| jd�fS |jrx|jtj	u rxtj
n|j}
tj| |
¡dd�}||dd…f }|	|dd…f }	||  ¡ }tj|dtjd�}tj|	dtjd�}|| }||| t tj¡j  }| d¡ d¡j d¡}tj| | dd�}|dk�r\tj|| jd�}tj|| jd�}n||d	  }||d	  }| d	d¡ |d	¡}tj|j ¡ |d
d�j}tj|tjd	|| jd�fdd�}tj||dd�}| d¡}|||fS )a§  
        Compute the list of precision, recall and MaP of a given class for every recall threshold.

        Args:
            preds_matched:      Tensor of shape (num_predictions, n_thresholds)
                                True when prediction (i) is matched with a target
                                with respect to the(j)th threshold
            preds_to_ignore     Tensor of shape (num_predictions, n_thresholds)
                                True when prediction (i) is matched with a crowd target
                                with respect to the (j)th threshold
            preds_scores:       Tensor of shape (num_predictions),
                                confidence score for every prediction
            n_targets:          Number of target boxes of this class
            recall_thresholds:  Tensor of shape (max_n_rec_thresh)
                                list of recall thresholds used to compute MaP
            score_threshold:    Minimum confidence score to consider a prediction
                                for the computation of precision and recall (not MaP)

        Returns:
            ap, precision, recall:     Tensors of shape (nb_thrs)
        rh   r   r+   Tri   N)Zaxisr‘   )Úrightr;   Frm   )Úinputr^   rn   )rž   rM   rŒ   Zlogical_notr   r”   r'   Zis_cudar‘   r•   Zuint8Zargsortr)   Ú
contiguousZcumsumr   ZfinfoÚfloat64ZepsZflipZcummaxÚvaluesZsearchsortedrp   Úrepeatr€   rN   ZgatherrW   )r.   rˆ   r—   re   r™   rš   r›   r¡   ZtpsZfpsr‘   Zsort_indZrolling_tpsZrolling_fpsZrolling_recallsZrolling_precisionsZlowest_score_above_thresholdr   r   Zrecall_threshold_idxZsampled_precision_pointsr   r   r   r   rŸ   @  sd     
ÿý	ÿ
ÿýÿ

ÿÿÿÿÿÿ
z?ObjectDetectionEvalProcessor._compute_detection_metrics_per_clsN)r"   )F)r�   T)%r	   r
   r   ÚIOU_THRESHOLDSr‹   ÚSCORE_THRESHOLDr›   ÚRECALL_THRESHOLDSrš   rU   rM   ZTensorÚintr   r/   r    r   rG   Útupler   r   rY   Ústaticmethodr   rE   r•   rD   rv   rR   Zndarrayr|   r„   rŽ   rP   rV   r   rŸ   r   r   r   r   r!   0   sˆ   	 ùù ü6
þ\
þþ öQ  ùøXùLùr!   Ú__main__)Úasdictzpths/to/predictions.jsonzpths/to/predictions2.jsonzpths/to/ground_truth.jsonzpths/to/ground_truth2.jsonzMetrics for z:
zPer class Metrics for )r   r?   Zdataclassesr   Úpathlibr   ÚnumpyrR   rM   r_   r¨   r©   rq   rª   r   r   r!   r	   r¯   Zprediction_file_pathsZground_truth_file_pathsrB   r0   r1   rG   Zeval_processorrY   ZmetricsZper_class_metricsÚprintrC   r   r   r   r   Ú<module>   sF   ÿ	     
þÿÿ