forked from stevenwudi/3DCNN_HMM
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Step2_3DCNN_Pred_Precompute.py
232 lines (193 loc) · 9.62 KB
/
Step2_3DCNN_Pred_Precompute.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
#-------------------------------------------------------------------------------
# Name: Starting Kit for ChaLearn LAP 2014 Track3
# Purpose: Show basic functionality of provided code
#
# Author: Xavier Baro
# Author: Di Wu: stevenwudi@gmail.com
# Created: 24/03/2014
# Copyright: (c) Chalearn LAP 2014
# Licence: GPL3
#-------------------------------------------------------------------------------
import sys, os,random,numpy,zipfile
from shutil import copyfile
import matplotlib.pyplot as plt
import cv2
import time
import cPickle
import numpy
import pickle
import scipy.io as sio
from scipy.misc import imresize
from numpy import log
####################################
##################load CNN here######
sys.path.append(r'I:\Kaggle_multimodal\StartingKit_track3\Final_project\ConvNet_3DCNN')
from utils import viterbi_path, viterbi_path_log
from utils import viterbi_colab_clean
from utils import createSubmisionFile
from utils import imdisplay
from ChalearnLAPEvaluation import evalGesture,exportGT_Gesture
from ChalearnLAPSample import GestureSample
from shownet_codalab import ShowConvNet
""" Main script. Show how to perform all competition steps
Access the sample information to learn a model. """
# Data folder (Training data)
STATE_NO = 10
##################################
### load the pre-store normalization constant
## Load Prior and transitional Matrix
dic=sio.loadmat('Transition_matrix.mat')
Transition_matrix = dic['Transition_matrix']
Prior = dic['Prior']
####################################
##################load CNN here######
import getopt as opt
from gpumodel import IGPUModel
from options import *
#################################### PER CNN customize
Flag_multiview = 0
CNN_NAME = 'ConvNet__2014-05-28_01.59.00_150'
outPred='./ConvNet_3DCNN/training/Valid_3DCNN_' + CNN_NAME
if not os.path.exists(outPred):
os.makedirs(outPred)
####################################
op = ShowConvNet.get_options_parser()
op.options['load_file'].value=r'.\ConvNet_3DCNN\tmp\ConvNet__2014-05-28_01.59.00'
op.options['write_features'].value ='probs'
load_dic = IGPUModel.load_checkpoint(op.options["load_file"].value)
old_op = load_dic["op"]
old_op.merge_from(op)
op = old_op
op.eval_expr_defaults()
op.options['train_batch_range'].value=[1]
op.options['test_batch_range'].value=[1]
op.options['data_path'].value=r'.\ConvNet_3DCNN\storage_sk_final'
model = ShowConvNet(op, load_dic)
model.crop_border = 0
meta = pickle.load(open(r'.\ConvNet_3DCNN\storage_sk_final\batches.meta'))
data_mean = meta['data_mean']
# pre-allocating the memory
IM_SZ = 90
debug_show = True
print("Extracting the training files")
data_path=os.path.join("I:\Kaggle_multimodal\Training\\")
cnn_path=os.path.join(".\ConvNet_3DCNN\Depth_Precompute\\")
# Get the list of training samples
samples=os.listdir(data_path)
samples_cnn = os.listdir(cnn_path)
for file_count, file in enumerate(samples):
if not file_count<650:
if file == samples_cnn[file_count-650]:
time_tic = time.time()
print("\t Processing file " + file)
# Create the object to access the sample
smp=GestureSample(os.path.join(data_path,file))
# ###############################################
# USE Ground Truth information to learn the model
# ###############################################
load_path_cnn= os.path.join(cnn_path,file)
dic_cnn = cPickle.load( open(load_path_cnn, "rb" ) )
cuboid = dic_cnn['cuboid']
Feature_all = numpy.zeros(( IM_SZ, IM_SZ, 4, cuboid.shape[-1]-3), numpy.uint8)
for frame in range(cuboid.shape[-1]-3):
Feature_all[:,:,:,frame] = cuboid[:,:, frame:frame+4]
### now the crucial part to write feature here###########
data = numpy.array(Feature_all, dtype=numpy.single)
#(data dimensionality)x(number of cases). !!!!!!!!!!!!!!
data = numpy.transpose(data, (2,0,1,3))
data = numpy.reshape(data, (4*90*90, data.shape[-1]))
data = numpy.array(data - data_mean[:,numpy.newaxis], dtype=numpy.single)
if Flag_multiview:#self.multiview:
data = data.reshape((4,90,90, data.shape[-1]))
target = numpy.zeros((4*82*82, data.shape[-1]*5), dtype=numpy.single)
border_size = 4
start_positions = [(0,0), (0, border_size*2),
(border_size, border_size),
(border_size*2, 0), (border_size*2, border_size*2)]
end_positions = [(sy+82, sx+82) for (sy,sx) in start_positions]
for i in xrange(5):
pic = data[:,start_positions[i][0]:end_positions[i][0],start_positions[i][1]:end_positions[i][1],:]
target[:,i * data.shape[-1]:(i+1)* data.shape[-1]] = pic.reshape((4*82*82, data.shape[-1]))
data = numpy.require(target, dtype=numpy.single)
ftrs = model.do_write_features(data)
##########################
# viterbi path decoding
#####################
if Flag_multiview:
accumulate_prob = numpy.zeros((ftrs.shape[0]/5, ftrs.shape[1]))
for i in range(5):
accumulate_prob += ftrs[i*ftrs.shape[0]/5:(i+1)*ftrs.shape[0]/5,:]
accumulate_prob = accumulate_prob/5.0
log_observ_likelihood = log(accumulate_prob.T + numpy.finfo(numpy.float32).eps)
else:
log_observ_likelihood = log(ftrs.T + numpy.finfo(numpy.float32).eps)
log_observ_likelihood[-1, 0:5] = 0
log_observ_likelihood[-1, -5:] = 0
print("\t Viterbi path decoding " )
# do it in log space avoid numeric underflow
[path, predecessor_state_index, global_score] = viterbi_path_log(log(Prior), log(Transition_matrix), log_observ_likelihood)
#[path, predecessor_state_index, global_score] = viterbi_path(Prior, Transition_matrix, observ_likelihood)
# Some gestures are not within the vocabulary
[pred_label, begin_frame, end_frame, Individual_score, frame_length] = viterbi_colab_clean(path, global_score, threshold=-100, mini_frame=19)
#begin_frame = begin_frame + 1
end_frame = end_frame + 5 # note here 3DCNN should add 5 frames because we used 4 frames
### plot the path and prediction
if True:
im = imdisplay(global_score)
plt.clf()
plt.imshow(im, cmap='gray')
plt.plot(range(global_score.shape[-1]), path, color='c',linewidth=2.0)
plt.xlim((0, global_score.shape[-1]))
# plot ground truth
gesturesList=smp.getGestures()
for gesture in gesturesList:
# Get the gesture ID, and start and end frames for the gesture
gestureID,startFrame,endFrame=gesture
frames_count = numpy.array(range(startFrame, endFrame+1))
pred_label_temp = ((gestureID-1) *10 +5) * numpy.ones(len(frames_count))
plt.plot(frames_count, pred_label_temp, color='r', linewidth=5.0)
# plot clean path
for i in range(len(begin_frame)):
frames_count = numpy.array(range(begin_frame[i], end_frame[i]+1))
pred_label_temp = ((pred_label[i]-1) *10 +5) * numpy.ones(len(frames_count))
plt.plot(frames_count, pred_label_temp, color='#ffff00', linewidth=2.0)
from pylab import *
if True:
save_dir=r'.\ConvNet_3DCNN\training\Depth_path_'+ CNN_NAME
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path= os.path.join(save_dir,file)
savefig(save_path, bbox_inches='tight')
#plt.show()
else:
plt.show()
### save the result
print "Elapsed time %d sec" % int(time.time() - time_tic)
save_dir=r'.\ConvNet_3DCNN\training\Depth_'+ CNN_NAME
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path= os.path.join(save_dir,file)
out_file = open(save_path, 'wb')
dic = {'log_observ_likelihood':log_observ_likelihood}
cPickle.dump(dic, out_file, protocol=cPickle.HIGHEST_PROTOCOL)
out_file.close()
pred=[]
for i in range(len(begin_frame)):
pred.append([ pred_label[i], begin_frame[i], end_frame[i]] )
smp.exportPredictions(pred,outPred)
# ###############################################
## delete the sample
del smp
TruthDir='./training/gt/'
final_score = evalGesture(outPred,TruthDir)
print("The score for this prediction is " + "{:.12f}".format(final_score))
# Submision folder (output)
outSubmision='./training/submision/'
# Prepare submision file (only for validation and final evaluation data sets)
createSubmisionFile(outPred, outSubmision)
#Cropped: \ConvNet_3DCNN\tmp\ConvNet__2014-05-28_01.59.00'
#The score for this prediction is 0.514078928486
#The score for this prediction is 0.686585827374 ConvNet__2014-06-25_18.21.33_100
#The score for this prediction is 0.692423435630 ConvNet__2014-06-25_18.21.33_250
#The score for this prediction is 0.729009426871 ConvNet__2014-05-28_01.59.00_150!!! --old!!!
#The score for this prediction is 0.696395444797 ConvNet__2014-06-25_18.56.59_162