U
    `]'                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZ d dlmZmZ d dlmZ d dlm  mZ d dlZG dd deZdS )    N)SummaryWriter)TimerAverageMeter)Loggervalidate_gradient)tqdmc                   @   s`   e Zd Zdd ZdddZdd Zdd	d
Zdd Zdd Zdd Z	dd Z
dd Zdd ZdS )Trainerc              	   C   st  || _ d| _|j| _|j| _|j| _|j| _|j| _|j| j| _|j	| _	|j
| _
|j| _|j| _|j| _|j| _|j| _|j| _|j| _|j| _|j| _|j| _d| _d| _t|jd| _t|j| _| jdtdd | j D d  d	 |jd
kr|  |j t! | _"|j#| j"d< |j$| j"d< |j%| j"d< t&|j dd}|t'| j W 5 Q R X |(  d S )N   g     j@g     j)log_dirz#parameters c                 S   s   g | ]}|  qS  )nelement).0xr   r   )/workspace/OverlapPredator/lib/trainer.py
<listcomp>+   s     z$Trainer.__init__.<locals>.<listcomp>g    .Az M
 trainvaltestz/modelw))configstart_epoch	max_epochsave_dirdeviceverbose
max_pointsmodelto	optimizer	schedulerZscheduler_freqZsnapshot_freqsnapshot_dir	benchmark	iter_sizeverbose_freqw_circle_lossw_overlap_lossw_saliency_loss	desc_loss	best_lossbest_recallr   
tboard_dirwriterr   loggerwritesum
parametersZpretrain_load_pretraindictloadertrain_loader
val_loadertest_loaderopenstrclose)selfargsfr   r   r   __init__   sD    ,zTrainer.__init__Nc                 C   s   || j  | j | j | j| jd}|d krJtj| j	d| d}ntj| j	d| d}| j
d| d t|| d S )N)epoch
state_dictr   r    r)   r*   Zmodel_z.pthzSave model to 
)r   r?   r   r    r)   r*   ospathjoinr   r-   r.   torchsave)r:   r>   namestatefilenamer   r   r   	_snapshot:   s    zTrainer._snapshotc                 C   s   t j|rt|}| j|d  |d | _| j|d  | j	|d  |d | _
|d | _| jd| d | jd	| j
 d
 | jd| j d
 ntd| dd S )Nr?   r>   r    r   r)   r*   z(Successfully load pretrained model from z!
zCurrent best loss r@   zCurrent best recall z=> no checkpoint found at '')rA   rB   isfilerD   loadr   load_state_dictr   r    r   r)   r*   r-   r.   
ValueError)r:   resumerG   r   r   r   r1   J   s    



zTrainer._load_pretrainr   c                 C   s   | j j| d S )Nlr)r   param_groups)r:   groupr   r   r   _get_lrZ   s    zTrainer._get_lrc                 C   sJ   t  }d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d	< |S )
N        circle_lossrecallsaliency_lossZsaliency_recallZsaliency_precisionoverlap_lossZoverlap_recallZoverlap_precision)r2   )r:   statsr   r   r   
stats_dict]   s    zTrainer.stats_dictc                 C   s.   t  }|  }| D ]\}}t ||< q|S )N)r2   rZ   itemsr   )r:   ZmetersrY   key_r   r   r   stats_meteri   s
    zTrainer.stats_meterc                 C   s  |dkst |dkr| j  | |\}}}|d d }|d d d }|d |d  }}	|d }
|d	 |d
  }}|d | ||d   }}| |||||
||	||	}|d | j |d | j  |d | j  }|  n| j  t	
  | |\}}}|d d }|d d d }|d |d  }}	|d }
|d	 |d
  }}|d | ||d   }}| |||||
||	||	}W 5 Q R X t|d  |d< t|d  |d< t|d  |d< |S )Nr   r   r   r   pointsr   stack_lengthsrottranscorrespondencessrc_pcd_rawtgt_pcd_rawrU   rX   rW   )AssertionErrorr   r   r(   r%   r&   r'   backwardevalrD   no_gradfloatdetach)r:   inputsphasefeatsscores_overlapscores_saliencypcdlen_srcc_rotc_transcorrespondencesrc_pcdtgt_pcd	src_feats	tgt_featsrY   Zc_lossr   r   r   inference_one_batchq   s6    
*


$zTrainer.inference_one_batchc                    sr  t   |dkst  }tt j| j j| j } j| 	 } j
  tt|D ]}| }| D ]<\}}	t|	tkr fdd|	D ||< qv|	 j||< qvz ||}
|d  j dkr|dkrt j}|r j
  n jd  j
  |
 D ]\}}|| | qW n, tk
r` } zt| W 5 d }~X Y nX tj   |d  j! dkr` j"r`||d  | }| D ]&\}} j#$| d| |j%| q| d	| d
|d dd| d}| D ]"\}}|| d|j%dd7 }q j|d  q`| d	| }| D ]"\}}|| d|j%dd7 }q: j|d  |S )Nr_   c                    s   g | ]}|  jqS r   )r   r   )r   itemr:   r   r   r      s     z/Trainer.inference_one_epoch.<locals>.<listcomp>r	   r   r   zgradient not valid
/z Epoch: z [Z4d]z: z.2f	r@   )&gccollectrg   r^   intlenr3   dataset
batch_size__iter__r   	zero_gradr   rangenextr[   typelistr   r   r{   r#   r   r   stepr-   r.   update	ExceptionprintrD   cudaempty_cacher$   r   r,   
add_scalaravg)r:   r>   rn   r^   num_iterc_loader_iterZc_iterrm   kvrY   Zgradient_validr\   valueinstZ	curr_itermessager   r}   r   inference_one_epoch   sL     



 "zTrainer.inference_one_epochc                 C   s   t d t| j| jD ]}| |d | j  | |d}|d j| jk rd|d j| _| 	|d |d j| j
kr|d j| _
| 	|d |d jdkrd	| _qd
| _qt d d S )Nzstart training...r   r   rU   r)   rV   r*   g333333?g      ?rT   zTraining finish!)r   r   r   r   r   r    r   r   r)   rI   r*   r'   )r:   r>   r^   r   r   r   r      s    
zTrainer.trainc                 C   s6   t d | dd}| D ]\}}t ||j qd S )Nz+Start to evaluate on validation datasets...r   r   )r   r   r[   r   )r:   r^   r\   r   r   r   r   ri      s    zTrainer.eval)N)r   )__name__
__module____qualname__r=   rI   r1   rS   rZ   r^   r{   r   r   ri   r   r   r   r   r      s   ,

5@r   )timerA   rD   copynumpynptorch.nnnnZtensorboardXr   	lib.timerr   r   	lib.utilsr   r   r   Ztorch.nn.functional
functionalFr   objectr   r   r   r   r   <module>   s    