问题
I'm working on a segmentation problem in Keras and I want to display segmentation results at the end of every training epoch.
I want something similar to Tensorflow: How to Display Custom Images in Tensorboard (e.g. Matplotlib Plots), but using Keras. I know that Keras has the TensorBoard callback but it seems limited for this purpose.
I know this would break the Keras backend abstraction, but I'm interested in using TensorFlow backend anyway.
Is it possible to achieve that with Keras + TensorFlow?
回答1:
So, the following solution works well for me:
import tensorflow as tf
def make_image(tensor):
"""
Convert an numpy representation image to Image protobuf.
Copied from https://github.com/lanpa/tensorboard-pytorch/
"""
from PIL import Image
height, width, channel = tensor.shape
image = Image.fromarray(tensor)
import io
output = io.BytesIO()
image.save(output, format='PNG')
image_string = output.getvalue()
output.close()
return tf.Summary.Image(height=height,
width=width,
colorspace=channel,
encoded_image_string=image_string)
class TensorBoardImage(keras.callbacks.Callback):
def __init__(self, tag):
super().__init__()
self.tag = tag
def on_epoch_end(self, epoch, logs={}):
# Load image
img = data.astronaut()
# Do something to the image
img = (255 * skimage.util.random_noise(img)).astype('uint8')
image = make_image(img)
summary = tf.Summary(value=[tf.Summary.Value(tag=self.tag, image=image)])
writer = tf.summary.FileWriter('./logs')
writer.add_summary(summary, epoch)
writer.close()
return
tbi_callback = TensorBoardImage('Image Example')
Just pass the callback to fit or fit_generator.
Note that you can also run some operations using the model inside the callback. For example, you may run the model on some images to check its performance.
回答2:
Based on the above answers and my own searching, I provide the following code to finish the following things using TensorBoard in Keras:
- problem setup: to predict the disparity map in binocular stereo matching;
- to feeds the model with input left image
xand ground truth disparity mapgt; - to display the input
xand ground truth 'gt', at some iteration time; - to display the output
yof your model, at some iteration time.
First of all, you have to make your costumed callback class with
Callback.Notethat a callback has access to its associated model through the class propertyself.model. AlsoNote: you have to feed the input to the model with feed_dict, if you want to get and display the output of your model.from keras.callbacks import Callback import numpy as np from keras import backend as K import tensorflow as tf import cv2 # make the 1 channel input image or disparity map look good within this color map. This function is not necessary for this Tensorboard problem shown as above. Just a function used in my own research project. def colormap_jet(img): return cv2.cvtColor(cv2.applyColorMap(np.uint8(img), 2), cv2.COLOR_BGR2RGB) class customModelCheckpoint(Callback): def __init__(self, log_dir='./logs/tmp/', feed_inputs_display=None): super(customModelCheckpoint, self).__init__() self.seen = 0 self.feed_inputs_display = feed_inputs_display self.writer = tf.summary.FileWriter(log_dir) # this function will return the feeding data for TensorBoard visualization; # arguments: # * feed_input_display : [(input_yourModelNeed, left_image, disparity_gt ), ..., (input_yourModelNeed, left_image, disparity_gt), ...], i.e., the list of tuples of Numpy Arrays what your model needs as input and what you want to display using TensorBoard. Note: you have to feed the input to the model with feed_dict, if you want to get and display the output of your model. def custom_set_feed_input_to_display(self, feed_inputs_display): self.feed_inputs_display = feed_inputs_display # copied from the above answers; def make_image(self, numpy_img): from PIL import Image height, width, channel = numpy_img.shape image = Image.fromarray(numpy_img) import io output = io.BytesIO() image.save(output, format='PNG') image_string = output.getvalue() output.close() return tf.Summary.Image(height=height, width=width, colorspace= channel, encoded_image_string=image_string) # A callback has access to its associated model through the class property self.model. def on_batch_end(self, batch, logs = None): logs = logs or {} self.seen += 1 if self.seen % 200 == 0: # every 200 iterations or batches, plot the costumed images using TensorBorad; summary_str = [] for i in range(len(self.feed_inputs_display)): feature, disp_gt, imgl = self.feed_inputs_display[i] disp_pred = np.squeeze(K.get_session().run(self.model.output, feed_dict = {self.model.input : feature}), axis = 0) #disp_pred = np.squeeze(self.model.predict_on_batch(feature), axis = 0) summary_str.append(tf.Summary.Value(tag= 'plot/img0/{}'.format(i), image= self.make_image( colormap_jet(imgl)))) # function colormap_jet(), defined above; summary_str.append(tf.Summary.Value(tag= 'plot/disp_gt/{}'.format(i), image= self.make_image( colormap_jet(disp_gt)))) summary_str.append(tf.Summary.Value(tag= 'plot/disp/{}'.format(i), image= self.make_image( colormap_jet(disp_pred)))) self.writer.add_summary(tf.Summary(value = summary_str), global_step =self.seen)Next, pass this callback object to
fit_generator()for your model, like:feed_inputs_4_display = some_function_you_wrote() callback_mc = customModelCheckpoint( log_dir = log_save_path, feed_inputd_display = feed_inputs_4_display) # or callback_mc.custom_set_feed_input_to_display(feed_inputs_4_display) yourModel.fit_generator(... callbacks = callback_mc) ...Now your can run the code, and go the TensorBoard host to see the costumed image display. For example, this is what I got using the aforementioned code:
Done! Enjoy!
回答3:
I'm trying to display matplotlib plots to the tensorboard (useful incases of plotting statistics, heatmaps, etc). It can be used for the general case also.
class AttentionLogger(keras.callbacks.Callback):
def __init__(self, val_data, logsdir):
super(AttentionLogger, self).__init__()
self.logsdir = logsdir # where the event files will be written
self.validation_data = val_data # validation data generator
self.writer = tf.summary.FileWriter(self.logsdir) # creating the summary writer
@tfmpl.figure_tensor
def attention_matplotlib(self, gen_images):
'''
Creates a matplotlib figure and writes it to tensorboard using tf-matplotlib
gen_images: The image tensor of shape (batchsize,width,height,channels) you want to write to tensorboard
'''
r, c = 5,5 # want to write 25 images as a 5x5 matplotlib subplot in TBD (tensorboard)
figs = tfmpl.create_figures(1, figsize=(15,15))
cnt = 0
for idx, f in enumerate(figs):
for i in range(r):
for j in range(c):
ax = f.add_subplot(r,c,cnt+1)
ax.set_yticklabels([])
ax.set_xticklabels([])
ax.imshow(gen_images[cnt]) # writes the image at index cnt to the 5x5 grid
cnt+=1
f.tight_layout()
return figs
def on_train_begin(self, logs=None): # when the training begins (run only once)
image_summary = [] # creating a list of summaries needed (can be scalar, images, histograms etc)
for index in range(len(self.model.output)): # self.model is accessible within callback
img_sum = tf.summary.image('img{}'.format(index), self.attention_matplotlib(self.model.output[index]))
image_summary.append(img_sum)
self.total_summary = tf.summary.merge(image_summary)
def on_epoch_end(self, epoch, logs = None): # at the end of each epoch run this
logs = logs or {}
x,y = next(self.validation_data) # get data from the generator
# get the backend session and sun the merged summary with appropriate feed_dict
sess_run_summary = K.get_session().run(self.total_summary, feed_dict = {self.model.input: x['encoder_input']})
self.writer.add_summary(sess_run_summary, global_step =epoch) #finally write the summary!
Then you will have to give it as an argument to fit/fit_generator
#val_generator is the validation data generator
callback_image = AttentionLogger(logsdir='./tensorboard', val_data=val_generator)
... # define the model and generators
# autoencoder is the model, note how callback is suppiled to fit_generator
autoencoder.fit_generator(generator=train_generator,
validation_data=val_generator,
callbacks=callback_image)
In my case where I'm displaying attention maps (as heatmaps) to tensorboard, this is the output.
回答4:
Similarily, you might want to try tf-matplotlib. Here's a scatter plot
import tensorflow as tf
import numpy as np
import tfmpl
@tfmpl.figure_tensor
def draw_scatter(scaled, colors):
'''Draw scatter plots. One for each color.'''
figs = tfmpl.create_figures(len(colors), figsize=(4,4))
for idx, f in enumerate(figs):
ax = f.add_subplot(111)
ax.axis('off')
ax.scatter(scaled[:, 0], scaled[:, 1], c=colors[idx])
f.tight_layout()
return figs
with tf.Session(graph=tf.Graph()) as sess:
# A point cloud that can be scaled by the user
points = tf.constant(
np.random.normal(loc=0.0, scale=1.0, size=(100, 2)).astype(np.float32)
)
scale = tf.placeholder(tf.float32)
scaled = points*scale
# Note, `scaled` above is a tensor. Its being passed `draw_scatter` below.
# However, when `draw_scatter` is invoked, the tensor will be evaluated and a
# numpy array representing its content is provided.
image_tensor = draw_scatter(scaled, ['r', 'g'])
image_summary = tf.summary.image('scatter', image_tensor)
all_summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter('log', sess.graph)
summary = sess.run(all_summaries, feed_dict={scale: 2.})
writer.add_summary(summary, global_step=0)
When executed, this results in the following plot inside Tensorboard
Note that tf-matplotlib takes care about evaluating any tensor inputs, avoids pyplot threading issues and supports blitting for runtime critical plotting.
回答5:
I believe I found a better way to log such custom images to tensorboard making use of the tf-matplotlib. Here is how...
class TensorBoardDTW(tf.keras.callbacks.TensorBoard):
def __init__(self, **kwargs):
super(TensorBoardDTW, self).__init__(**kwargs)
self.dtw_image_summary = None
def _make_histogram_ops(self, model):
super(TensorBoardDTW, self)._make_histogram_ops(model)
tf.summary.image('dtw-cost', create_dtw_image(model.output))
One just need to overwrite the _make_histogram_ops method from the TensorBoard callback class to add the custom summary. In my case, the create_dtw_image is a function that creates an image using the tf-matplotlib.
Regards,.
回答6:
Here is example how to draw landmarks on image:
class CustomCallback(keras.callbacks.Callback):
def __init__(self, model, generator):
self.generator = generator
self.model = model
def tf_summary_image(self, tensor):
import io
from PIL import Image
tensor = tensor.astype(np.uint8)
height, width, channel = tensor.shape
image = Image.fromarray(tensor)
output = io.BytesIO()
image.save(output, format='PNG')
image_string = output.getvalue()
output.close()
return tf.Summary.Image(height=height,
width=width,
colorspace=channel,
encoded_image_string=image_string)
def on_epoch_end(self, epoch, logs={}):
frames_arr, landmarks = next(self.generator)
# Take just 1st sample from batch
frames_arr = frames_arr[0:1,...]
y_pred = self.model.predict(frames_arr)
# Get last frame for which we have done predictions
img = frames_arr[0,-1,:,:]
img = img * 255
img = img[:, :, ::-1]
img = np.copy(img)
landmarks_gt = landmarks[-1].reshape(-1,2)
landmarks_pred = y_pred.reshape(-1,2)
img = draw_landmarks(img, landmarks_gt, (0,255,0))
img = draw_landmarks(img, landmarks_pred, (0,0,255))
image = self.tf_summary_image(img)
summary = tf.Summary(value=[tf.Summary.Value(image=image)])
writer = tf.summary.FileWriter('./logs')
writer.add_summary(summary, epoch)
writer.close()
return
回答7:
class customModelCheckpoint(Callback):
def __init__(self, log_dir='../logs/', feed_inputs_display=None):
super(customModelCheckpoint, self).__init__()
self.seen = 0
self.feed_inputs_display = feed_inputs_display
self.writer = tf.summary.FileWriter(log_dir)
def custom_set_feed_input_to_display(self, feed_inputs_display):
self.feed_inputs_display = feed_inputs_display
# A callback has access to its associated model through the class property self.model.
def on_batch_end(self, batch, logs = None):
logs = logs or {}
self.seen += 1
if self.seen % 8 == 0: # every 200 iterations or batches, plot the costumed images using TensorBorad;
summary_str = []
feature = self.feed_inputs_display[0][0]
disp_gt = self.feed_inputs_display[0][1]
disp_pred = self.model.predict_on_batch(feature)
summary_str.append(tf.summary.image('disp_input/{}'.format(self.seen), feature, max_outputs=4))
summary_str.append(tf.summary.image('disp_gt/{}'.format(self.seen), disp_gt, max_outputs=4))
summary_str.append(tf.summary.image('disp_pred/{}'.format(self.seen), disp_pred, max_outputs=4))
summary_st = tf.summary.merge(summary_str)
summary_s = K.get_session().run(summary_st)
self.writer.add_summary(summary_s, global_step=self.seen)
self.writer.flush()
callback_mc = customModelCheckpoint(log_dir='../logs/', feed_inputs_display=[(a, b)])
callback_tb = TensorBoard(log_dir='../logs/', histogram_freq=0, write_graph=True, write_images=True)
callback = []
def data_gen(fr1, fr2):
while True:
hdr_arr = []
ldr_arr = []
for i in range(args['batch_size']):
try:
ldr = pickle.load(fr2)
hdr = pickle.load(fr1)
except EOFError:
fr1 = open(args['data_h_hdr'], 'rb')
fr2 = open(args['data_h_ldr'], 'rb')
hdr_arr.append(hdr)
ldr_arr.append(ldr)
hdr_h = np.array(hdr_arr)
ldr_h = np.array(ldr_arr)
gen = aug.flow(hdr_h, ldr_h, batch_size=args['batch_size'])
out = gen.next()
a = out[0]
b = out[1]
callback_mc.custom_set_feed_input_to_display(feed_inputs_display=[(a, b)])
yield [a, b]
callback.append(callback_tb)
callback.append(callback_mc)
H = model.fit_generator(data_gen(fr1, fr2), steps_per_epoch=100, epochs=args['epoch'], callbacks=callback)
picture
来源：https://stackoverflow.com/questions/43784921/how-to-display-custom-images-in-tensorboard-using-keras