I am trying to adapt the code to achieve 1-D convolution using 1-D input. The model is compilable so you can see the layers and shapes in .summary(), but it throws the error when .fit() the model. it seems to occur in loss computation. Below is my code:
import numpy as np from scipy.stats import norm from keras.layers import Input, Dense, Lambda, Flatten, Reshape from keras.layers import Conv1D, UpSampling1D from keras.models import Model from keras import backend as K from keras import metrics num_conv = 6 batch_size = 100 latent_dim = 2 intermediate_dim = 128 epochs = 50 epsilon_std = 1.0 x = Input(batch_shape=(batch_size, 310, 1)) conv_1 = Conv1D(1, kernel_size=num_conv, padding='same', activation='relu')(x) conv_2 = Conv1D(64, kernel_size=num_conv, padding='same', strides=2, activation='relu')(conv_1) conv_3 = Conv1D(64, kernel_size=num_conv, padding='same', activation='relu')(conv_2) flatten = Flatten()(conv_3) hidden = Dense(intermediate_dim, activation='relu')(flatten) z_mean = Dense(latent_dim)(hidden) z_log_var = Dense(latent_dim)(hidden) def sampling(args): z_mean, z_log_var = args epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0., stddev=epsilon_std) return(z_mean + K.exp(z_log_var/2) * epsilon) z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var]) decoder_h = Dense(256, activation='relu')(z) decoder = Dense(155, activation='relu')(decoder_h) decoder = Reshape((155, 1))(decoder) de_conv_1 = Conv1D(64, kernel_size=num_conv, padding='same', activation='relu')(decoder) de_conv_2 = Conv1D(64, kernel_size=num_conv, padding='same', activation='relu')(de_conv_1) upsamp = UpSampling1D(2)(de_conv_2) x_decoded_mean = Conv1D(1, kernel_size=num_conv, padding='same', activation='relu')(upsamp) x_decoded_mean = Reshape([310, 1])(x_decoded_mean) def vae_loss(x, x_decoded_mean): x_ = x[:, 150:160, :] x_decoded_mean_ = x_decoded_mean[:, 150:160, :] xent_loss = 10 * metrics.mean_squared_error(x_, x_decoded_mean_) kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) -K.exp(z_log_var), axis=-1) return(xent_loss + kl_loss) vae = Model(x, x_decoded_mean) vae.summary() vae.compile(optimizer='rmsprop', loss=vae_loss) The input data shape is (n_sample, 310, 1). It is an one-D time-series but I include prior and posterior 150 frames to predict the middle 10 frames, resulting in 310 frames as input.
In vae_loss(), the reason that x and x_decoded_mean are sliced is that the purpose is to reconstruct the middle 10 frames with additional information of prior and posterior 150 frames. Therefore I want to force the model to focus on loss computed only from the middle 10 frames.
I got the following error when I .fit() the model:
# X.shape == (n_samples, 310, 1) # n_samples % batch_size == 0 vae.fit(x=X, y=X, batch_size=batch_size, epochs=epochs, shuffle=True) The long error below:
Epoch 1/50 --------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last) /Users/yjluo/WORK/pitchPerfect/vae/model2.py in <module>() 77 vae.fit(x=X, y=X, batch_size=batch_size, 78 epochs=epochs, ---> 79 shuffle=True) /usr/local/lib/python2.7/site-packages/keras/engine/training.pyc in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs) 1496 val_f=val_f, val_ins=val_ins, shuffle=shuffle, 1497 callback_metrics=callback_metrics, -> 1498 initial_epoch=initial_epoch) 1499 1500 def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): /usr/local/lib/python2.7/site-packages/keras/engine/training.pyc in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch) 1150 batch_logs['size'] = len(batch_ids) 1151 callbacks.on_batch_begin(batch_index, batch_logs) -> 1152 outs = f(ins_batch) 1153 if not isinstance(outs, list): 1154 outs = [outs] /usr/local/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc in __call__(self, inputs) 2227 session = get_session() 2228 updated = session.run(self.outputs + [self.updates_op], -> 2229 feed_dict=feed_dict) 2230 return updated[:len(self.outputs)] 2231 /usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata) 776 try: 777 result = self._run(None, fetches, feed_dict, options_ptr, --> 778 run_metadata_ptr) 779 if run_metadata: 780 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr) /usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata) 980 if final_fetches or final_targets: 981 results = self._do_run(handle, final_targets, final_fetches, --> 982 feed_dict_string, options, run_metadata) 983 else: 984 results = [] /usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata) 1030 if handle is None: 1031 return self._do_call(_run_fn, self._session, feed_dict, fetch_list, -> 1032 target_list, options, run_metadata) 1033 else: 1034 return self._do_call(_prun_fn, self._session, handle, feed_dict, /usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args) 1050 except KeyError: 1051 pass -> 1052 raise type(e)(node_def, op, message) 1053 1054 def _extend_graph(self): InvalidArgumentError: Incompatible shapes: [100,10] vs. [100] [[Node: gradients_4/add_121_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:@add_121"], _device="/job:localhost/replica:0/task:0/cpu:0"](gradients_4/add_121_grad/Shape, gradients_4/add_121_grad/Shape_1)]] Caused by op u'gradients_4/add_121_grad/BroadcastGradientArgs', defined at: File "/usr/local/bin/ipython", line 11, in <module> sys.exit(start_ipython()) File "/usr/local/lib/python2.7/site-packages/IPython/__init__.py", line 119, in start_ipython return launch_new_instance(argv=argv, **kwargs) File "/usr/local/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance app.start() File "/usr/local/lib/python2.7/site-packages/IPython/terminal/ipapp.py", line 355, in start self.shell.mainloop() File "/usr/local/lib/python2.7/site-packages/IPython/terminal/interactiveshell.py", line 493, in mainloop self.interact() File "/usr/local/lib/python2.7/site-packages/IPython/terminal/interactiveshell.py", line 484, in interact self.run_cell(code, store_history=True) File "/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2718, in run_cell interactivity=interactivity, compiler=compiler, result=result) File "/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2828, in run_ast_nodes if self.run_code(code, result): File "/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2882, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "<ipython-input-5-475083cdc0be>", line 1, in <module> get_ipython().magic(u'run model2.py') File "/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2160, in magic return self.run_line_magic(magic_name, magic_arg_s) File "/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2081, in run_line_magic result = fn(*args,**kwargs) File "<decorator-gen-58>", line 2, in run File "/usr/local/lib/python2.7/site-packages/IPython/core/magic.py", line 188, in <lambda> call = lambda f, *a, **k: f(*a, **k) File "/usr/local/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 742, in run run() File "/usr/local/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 728, in run exit_ignore=exit_ignore) File "/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2483, in safe_execfile self.compile if kw['shell_futures'] else None) File "/usr/local/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 289, in execfile builtin_mod.execfile(filename, *where) File "/Users/yjluo/WORK/pitchPerfect/vae/model2.py", line 79, in <module> shuffle=True) File "/usr/local/lib/python2.7/site-packages/keras/engine/training.py", line 1481, in fit self._make_train_function() File "/usr/local/lib/python2.7/site-packages/keras/engine/training.py", line 1013, in _make_train_function self.total_loss) File "/usr/local/lib/python2.7/site-packages/keras/optimizers.py", line 197, in get_updates grads = self.get_gradients(loss, params) File "/usr/local/lib/python2.7/site-packages/keras/optimizers.py", line 47, in get_gradients grads = K.gradients(loss, params) File "/usr/local/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 2264, in gradients return tf.gradients(loss, variables, colocate_gradients_with_ops=True) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gradients_impl.py", line 560, in gradients grad_scope, op, func_call, lambda: grad_fn(op, *out_grads)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gradients_impl.py", line 368, in _MaybeCompile return grad_fn() # Exit early File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gradients_impl.py", line 560, in <lambda> grad_scope, op, func_call, lambda: grad_fn(op, *out_grads)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/math_grad.py", line 598, in _AddGrad rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 411, in _broadcast_gradient_args name=name) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 768, in apply_op op_def=op_def) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2336, in create_op original_op=self._default_original_op, op_def=op_def) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1228, in __init__ self._traceback = _extract_stack() ...which was originally created as op u'add_121', defined at: File "/usr/local/bin/ipython", line 11, in <module> sys.exit(start_ipython()) [elided 16 identical lines from previous traceback] File "/usr/local/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 289, in execfile builtin_mod.execfile(filename, *where) File "/Users/yjluo/WORK/pitchPerfect/vae/model2.py", line 68, in <module> vae.compile(optimizer='rmsprop', loss=vae_loss) File "/usr/local/lib/python2.7/site-packages/keras/engine/training.py", line 910, in compile sample_weight, mask) File "/usr/local/lib/python2.7/site-packages/keras/engine/training.py", line 436, in weighted score_array = fn(y_true, y_pred) File "/Users/yjluo/WORK/pitchPerfect/vae/model2.py", line 64, in vae_loss return(xent_loss + kl_loss) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 821, in binary_op_wrapper return func(x, y, name=name) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 73, in add result = _op_def_lib.apply_op("Add", x=x, y=y, name=name) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 768, in apply_op op_def=op_def) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2336, in create_op original_op=self._default_original_op, op_def=op_def) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1228, in __init__ self._traceback = _extract_stack() InvalidArgumentError (see above for traceback): Incompatible shapes: [100,10] vs. [100] [[Node: gradients_4/add_121_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _class=["loc:@add_121"], _device="/job:localhost/replica:0/task:0/cpu:0"](gradients_4/add_121_grad/Shape, gradients_4/add_121_grad/Shape_1)]] Based on the line Incompatible shapes: [100,10] vs. [100], I believe it happens in loss computation but I can't figure out the solution. Moreover, even I don't do the slicing in vae_loss(), the error still show as Incompatible shapes: [100,310] vs. [100]. Could anyone please give me some suggestion?