Issue with AdamOptimizer

问题

I'm using a simple network and i'm trying to use AdamOptimizer to minimize the loss in a Q learning contexte.

Here the code :

### DATASET IMPORT
from DataSet import *

### NETWORK
state_size      = STATE_SIZE
stack_size      = STACK_SIZE
action_size     = ACTION_SIZE
learning_rate   = LEARNING_RATE
hidden_tensors  = HIDDEN_TENSORS
gamma           = GAMMA

import tensorflow as tf
import numpy as np

class NNetwork:       
    def __init__(self, name='NNetwork'):

        # Initialisations
        self.state_size     = state_size
        self.action_size    = action_size
        self.model          = tf.keras.models.Sequential()
        self.optimizer      = tf.keras.optimizers.Adam(learning_rate)

        # Network shaping
        self.model.add(tf.keras.layers.Dense(self.state_size,   activation='relu',      kernel_initializer='glorot_uniform'))
        self.model.add(tf.keras.layers.Dense(hidden_tensors,    activation='relu',      kernel_initializer='glorot_uniform'))
        self.model.add(tf.keras.layers.Dense(action_size,       activation='linear',    kernel_initializer='glorot_uniform'))

    # Prediction function (return Q_values)
    def get_outputs(self, inputs):
        inputs = tf.convert_to_tensor(inputs, dtype=tf.float32)
        return self.model.predict(inputs)

    # Optimization of the network
    def optimize(self, state, action, reward, next_state):
        next_Q_values   = self.get_outputs(next_state)
        target_Q        = reward + gamma * np.max(next_Q_values)
        curent_Q        = tf.reduce_sum(tf.multiply(self.get_outputs(state), action))
        loss           = tf.square(target_Q - curent_Q)
        self.optimizer.minimize(tf.convert_to_tensor(loss), self.model.trainable_variables)



B = NNetwork('b')
print(B.get_outputs([[0.12, 0.59]]))

B.optimize([[0.12, 0.59]], [1, 0, 0, 0, 0, 0, 0], 100000000, [[0.13, 0.58]])
print(B.get_outputs([[0.12, 0.59]]))

So my problem is :

When i execute this code i got this :

[[-0.00105272 0.02356465 -0.01908724 -0.03868931 0.01585
0.02427034 0.00203115]] Traceback (most recent call last): File ".\DQNet.py", line 69, in B.optimize([[0.12, 0.59]], [1, 0, 0, 0, 0, 0, 0], 100000000, [[0.13, 0.58]]) File ".\DQNet.py", line 62, in optimize tf.keras.optimizers.Adam(learning_rate).minimize(tf.convert_to_tensor(10), self.model.trainable_variables) File "C:\Users\Odeven poste 1\Documents[Python-3.6.8\python-3.6.8.amd64\lib\site-packages\tensorflow\python\keras\optimizer_v2\optimizer_v2.py", line 296, in minimize loss, var_list=var_list, grad_loss=grad_loss) File "C:\Users\Odeven poste 1\Documents[Python-3.6.8\python-3.6.8.amd64\lib\site-packages\tensorflow\python\keras\optimizer_v2\optimizer_v2.py", line 328, in _compute_gradients loss_value = loss() TypeError: 'tensorflow.python.framework.ops.EagerTensor' object is not callable

That mean that my network is functional because i got Q values but when i try to call my 'optimize' function i got an error on the line :

self.optimizer.minimize(tf.convert_to_tensor(loss), self.model.trainable_variables)

And i really don't understand why i got this error :

'tensorflow.python.framework.ops.EagerTensor' object is not callable

Because i'm pretty sure that the 'loss' parameter i have to give to the minimize function should be a Tensor...

回答1:

In TF2 the loss parameter of the minimize method must be a Python callable.

Thus, you can change your loss definition to:

def loss():
    return tf.square(target_Q - curent_Q)

and use it without converting it to a Tensor:

self.optimizer.minimize(loss, self.model.trainable_variables)

来源：https://stackoverflow.com/questions/55953344/issue-with-adamoptimizer