TF: how to solve ValueError: Variable ...weight already exists, disallowed. Did you mean to set reuse=True

Question

I build an Inverse Compositional CNN but it reports errors as bellows:

ValueError: Variable left_src_tgt_warp/ICSTN/icnv1/weight already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:

I find that using tf.reset_default_graph() can solve this problem. But I do not know where I should add it.

for l in range(opt.warpN):
    with tf.variable_scope("ICSTN", reuse=l > 0) as sc:
    end_points_collection = sc.original_name_scope + '_end_points'
    with slim.arg_scope([slim.conv2d, slim.conv2d_transpose],
                       normalizer_fn=slim.batch_norm,
                      weights_regularizer=slim.l2_regularizer(0.05),
                            normalizer_params=batch_norm_params,
                            activation_fn=tf.nn.relu,
                          outputs_collections=end_points_collection):
            imageWarp = inverse_warp(
                inputImage,
                depth,
                pM,
                intrinsics,
                intrinsics_inv)
            imageWarpAll.append(imageWarp)
            feat = tf.reshape(imageWarp, [batch_size, H, W, C])
            print('feat shape:', feat.get_shape())
            print('pM_ini:', pM.get_shape())
            with tf.variable_scope("icnv1"):
                feat = conv2Layer(opt, feat, 4)
                feat = tf.nn.relu(feat)
            with tf.variable_scope("icnv2"):
                feat = conv2Layer(opt, feat, 8)
                feat = tf.nn.relu(feat)
                feat = tf.nn.max_pool(feat, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
            feat = tf.reshape(feat, [opt.batch_size, -1])
            with tf.variable_scope("fc3"):
                feat = linearLayer(opt, feat, 48)
                feat = tf.nn.relu(feat)
            with tf.variable_scope("fc4"):
                feat = linearLayer(opt, feat, 6, final=True)
            dp = tf.reshape(feat, [-1, 6])
            print('dp: ', dp.get_shape())
        dpM = pose_vec2mat(dp)
        pM = tf.matmul(dpM, pM)
    imageWarp = inverse_warp(
        inputImage,
        depth,
        pM,
        intrinsics,
        intrinsics_inv)
    imageWarpAll.append(imageWarp)
    return imageWarpAll, pM
def build_train_graph():
    with tf.name_scope("cnn1"):...
    with tf.name_scope("cnn2"):...
    with tf.name_scope("Inverse Compositional CNN"):...
def train(self, opt):
    with tf.variable_scope(tf.get_variable_scope()):
            for i in range(opt.num_gpus):
                print('gpu:', i)
                with tf.device('/gpu:%d' % i):
                    self.build_train_graph(L_img_splits[i], R_img_splits[i], L_cam2pix_splits[i], L_pix2cam_splits[i],
                                       R_cam2pix_splits[i], R_pix2cam_splits[i], L_sca_splits[i], R_sca_splits[i],
                                       reuse_variables)
                    self.collect_summaries(i)
                    tower_losses.append(self.total_loss)
                    reuse_variables = True
                    grads = opt_step.compute_gradients(self.total_loss)
                    tower_grads.append(grads)
        grads = average_gradients(tower_grads)
        apply_gradient_op = opt_step.apply_gr`enter code here`adients(grads, global_step=global_step)
        incr_global_step = tf.assign(global_step, global_step + 1)
        total_loss = tf.reduce_mean(tower_losses)

        tf.summary.scalar('learning_rate', learning_rate, ['model_0'])
        tf.summary.scalar('total_loss', total_loss, ['model_0'])
        summary_op = tf.summary.merge_all('model_0')
        # self.collect_summaries()
        # SESSION
        config = tf.ConfigProto(allow_soft_placement=True)
        config.gpu_options.allow_growth = True
        sess = tf.Session(config=config)

        # SAVER
        summary_writer = tf.summary.FileWriter(
            opt.checkpoint_dir + '/s%.1d_%.3d/' % (opt.seq_length, opt.img_height) + opt.model_name, sess.graph)
        self.saver = tf.train.Saver()
        # COUNT PARAM
        total_num_parameters = 0
        for variable in tf.trainable_variables():
            total_num_parameters += np.array(variable.get_shape().as_list()).prod()
         print("number of trainable parameters: {}".format(total_num_parameters))
        # INIT
        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())
        coordinator = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coordinator)
        # LOAD CHECKPOINT IF SET
        if opt.continue_train:
             print("Resume training from previous checkpoint")
             checkpoint = tf.train.latest_checkpoint(
                 os.path.join(opt.checkpoint_dir, 's%.1d_%.3d' % (opt.seq_length, opt.img_height), opt.model_name))
             self.saver.restore(sess, checkpoint)
        if opt.re_train:
            sess.run(global_step.assign(0))

Solution 1

It's because of the for loop in the first part of the code that's possibly missing function name.

The loop tries to create the left_src_tgt_warp/ICSTN/icnv1/weight (same for icnv2 and so on):

def foo(num_layers):
    opt = tf.placeholder(tf.float32, [None, 64])
    for i in range(num_layers):
        with tf.variable_scope("icnv1"):
            feat = tf.layers.dense(opt, units=1, activation=tf.nn.relu)

foo(5)

ValueError: Variable icnv1/dense/kernel already exists, disallowed. Did you mean to set reuse=True

You need distinct names for Variables. One way to achieve this is like this:

def foo(num_layers):
    opt = tf.placeholder(tf.float32, [None, 64])
    for i in range(num_layers):
        with tf.variable_scope("icnv1_layer_{}".format(i)):
            feat = tf.layers.dense(opt, units=1, activation=tf.nn.relu)

we now have distinct names for each, icnv1_layer_1, icnv1_layer_2, etc. Depending on the depth.

Unless of course, you want a shared weights (e.g. it's the same layer, updates as one). In that case just set:

with tf.variable_scope("icnv1", reuse=tf.AUTO_REUSE):

Solution 2

I'm using Jupyter Notebooks to run my models and I recently realize that this error was occuring due to the fact that variables of my model was been saved on an "outter context". So when I restart kernel (and thus clean all my workspace variable) and run all cells the error gone away.

Comments

[Chaine]

Have you solved this problem?

[June Wang]

Do you might know what ValueError: Trying to share variable shortcut/weights, but specified shape (1, 1, 32, 64) and found shape (1, 1, 16, 32). is suggesting when set reuse=tf.AUTO_REUSE?

[Aechlys]

@JuneWang Hi, when you try to reuse the layer one of your dimensions changed from 16 to 32. Check the inputs to the shared layer and make sure the dimensions are the same.