diff --git a/lib/images/style/west-elm-brightcolors.jpg b/lib/images/style/west-elm-brightcolors.jpg new file mode 100644 index 0000000..086f40c Binary files /dev/null and b/lib/images/style/west-elm-brightcolors.jpg differ diff --git a/src/custom_vgg19V2.py b/src/custom_vgg19V2.py new file mode 100644 index 0000000..09f5c7f --- /dev/null +++ b/src/custom_vgg19V2.py @@ -0,0 +1,140 @@ +import os +import tensorflow as tf +import numpy as np +import inspect +import urllib.request + +VGG_MEAN = [103.939, 116.779, 123.68] +data = None +dir_path = os.path.dirname(os.path.realpath(__file__)) +weights_name = dir_path + "/../lib/weights/vgg19.npy" +weights_url = "https://www.dropbox.com/s/68opci8420g7bcl/vgg19.npy?dl=1" + + +class Vgg19: + def __init__(self, vgg19_npy_path=None): + global data + + if vgg19_npy_path is None: + path = inspect.getfile(Vgg19) + path = os.path.abspath(os.path.join(path, os.pardir)) + path = os.path.join(path, weights_name) + + if os.path.exists(path): + vgg19_npy_path = path + else: + print("VGG19 weights were not found in the project directory") + + answer = 0 + while answer is not 'y' and answer is not 'N': + answer = input("Would you like to download the 548 MB file? [y/N] ").replace(" ", "") + + # Download weights if yes, else exit the program + if answer == 'y': + print("Downloading. Please be patient...") + urllib.request.urlretrieve(weights_url, weights_name) + vgg19_npy_path = path + elif answer == 'N': + print("Exiting the program..") + exit(0) + + if data is None: + data = np.load(vgg19_npy_path, encoding='latin1', allow_pickle=True) + self.data_dict = data.item() + print("VGG19 weights loaded") + + else: + self.data_dict = data.item() + + def build(self, rgb, shape): + rgb_scaled = rgb * 255.0 + num_channels = shape[2] + channel_shape = shape + channel_shape[2] = 1 + + # Convert RGB to BGR + red, green, blue = tf.split(axis=3, num_or_size_splits=3, value=rgb_scaled) + + assert red.get_shape().as_list()[1:] == channel_shape + assert green.get_shape().as_list()[1:] == channel_shape + assert blue.get_shape().as_list()[1:] == channel_shape + + bgr = tf.concat(axis=3, values=[ + blue - VGG_MEAN[0], + green - VGG_MEAN[1], + red - VGG_MEAN[2], + ]) + + shape[2] = num_channels + assert bgr.get_shape().as_list()[1:] == shape + + self.conv1_1 = self.conv_layer(bgr, "conv1_1") + self.conv1_2 = self.conv_layer(self.conv1_1, "conv1_2") + self.pool1 = self.avg_pool(self.conv1_2, 'pool1') + + self.conv2_1 = self.conv_layer(self.pool1, "conv2_1") + self.conv2_2 = self.conv_layer(self.conv2_1, "conv2_2") + self.pool2 = self.avg_pool(self.conv2_2, 'pool2') + + self.conv3_1 = self.conv_layer(self.pool2, "conv3_1") + self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2") + self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3") + self.conv3_4 = self.conv_layer(self.conv3_3, "conv3_4") + self.pool3 = self.avg_pool(self.conv3_4, 'pool3') + + self.conv4_1 = self.conv_layer(self.pool3, "conv4_1") + self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2") + self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3") + self.conv4_4 = self.conv_layer(self.conv4_3, "conv4_4") + self.pool4 = self.avg_pool(self.conv4_4, 'pool4') + + self.conv5_1 = self.conv_layer(self.pool4, "conv5_1") + self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2") + self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3") + self.conv5_4 = self.conv_layer(self.conv5_3, "conv5_4") + + self.data_dict = None + + def avg_pool(self, bottom, name): + return tf.nn.avg_pool2d(input=bottom, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name=name) + + def max_pool(self, bottom, name): + return tf.nn.max_pool2d(input=bottom, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name=name) + + def conv_layer(self, bottom, name): + with tf.compat.v1.variable_scope(name): + filt = self.get_conv_filter(name) + + conv = tf.nn.conv2d(input=bottom, filters=filt, strides=[1, 1, 1, 1], padding='SAME') + + conv_biases = self.get_bias(name) + bias = tf.nn.bias_add(conv, conv_biases) + + relu = tf.nn.relu(bias) + return relu + + def fc_layer(self, bottom, name): + with tf.compat.v1.variable_scope(name): + shape = bottom.get_shape().as_list() + dim = 1 + for d in shape[1:]: + dim *= d + x = tf.reshape(bottom, [-1, dim]) + + weights = self.get_fc_weight(name) + biases = self.get_bias(name) + + # Fully connected layer. Note that the '+' operation automatically + # broadcasts the biases. + fc = tf.nn.bias_add(tf.matmul(x, weights), biases) + + return fc + + def get_conv_filter(self, name): + return tf.constant(self.data_dict[name][0], name="filter") + + def get_bias(self, name): + return tf.constant(self.data_dict[name][1], name="biases") + + def get_fc_weight(self, name): + return tf.constant(self.data_dict[name][0], name="weights") diff --git a/src/report.txt b/src/report.txt new file mode 100644 index 0000000..4f071d5 --- /dev/null +++ b/src/report.txt @@ -0,0 +1,26 @@ +TensorFlow 2.0 Upgrade Script +----------------------------- +Converted 1 files +Detected 0 issues that require attention +-------------------------------------------------------------------------------- +================================================================================ +Detailed log follows: + +================================================================================ +-------------------------------------------------------------------------------- +Processing file 'custom_vgg19.py' + outputting to 'custom_vgg19V2.py' +-------------------------------------------------------------------------------- + +99:15: INFO: Added keywords to args of function 'tf.nn.avg_pool' +99:15: INFO: Renamed keyword argument for tf.nn.avg_pool from value to input +99:15: INFO: Renamed 'tf.nn.avg_pool' to 'tf.nn.avg_pool2d' +102:15: INFO: Added keywords to args of function 'tf.nn.max_pool' +102:15: INFO: Renamed keyword argument for tf.nn.max_pool from value to input +102:15: INFO: Renamed 'tf.nn.max_pool' to 'tf.nn.max_pool2d' +105:13: INFO: Renamed 'tf.variable_scope' to 'tf.compat.v1.variable_scope' +108:19: INFO: Added keywords to args of function 'tf.nn.conv2d' +108:19: INFO: Renamed keyword argument for tf.nn.conv2d from filter to filters +117:13: INFO: Renamed 'tf.variable_scope' to 'tf.compat.v1.variable_scope' +-------------------------------------------------------------------------------- + diff --git a/src/style_transfer.py b/src/style_transfer.py index b1c9067..b973628 100644 --- a/src/style_transfer.py +++ b/src/style_transfer.py @@ -8,6 +8,9 @@ import numpy as np import os import tensorflow as tf + +# import tensorflow.compat.v1 as tf + import time import utils from functools import reduce @@ -134,8 +137,8 @@ def parse_args(): STYLE_PATH = os.path.realpath(args.style) OUT_PATH = os.path.realpath(args.out) - -with tf.Session() as sess: +with tf.compat.v1.Session() as sess: +# with tf.Session() as sess: parse_args() # Initialize and process photo image to be used for our content diff --git a/src/style_transferV2.py b/src/style_transferV2.py new file mode 100644 index 0000000..9a212ac --- /dev/null +++ b/src/style_transferV2.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 +# Mohamed K. Eid (mohamedkeid@gmail.com) +# Description: TensorFlow implementation of "A Neural Algorithm of Artistic Style" using TV denoising as a regularizer. + +import argparse +import custom_vgg19V2 as vgg19 +import logging +import numpy as np +import os +import tensorflow as tf + +# import tensorflow.compat.v1 as tf + +import time +import utilsV2 +from functools import reduce + +# Model hyperparams +CONTENT_LAYER = 'conv4_2' +STYLE_LAYERS = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1'] +EPOCHS = 300 +LEARNING_RATE = .02 +TOTAL_VARIATION_SMOOTHING = 1.5 +NORM_TERM = 6. + +# Loss term weights +CONTENT_WEIGHT = 1. +STYLE_WEIGHT = 3. +NORM_WEIGHT = .1 +TV_WEIGHT = .1 + +# Default image paths +DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +OUT_PATH = DIR_PATH + '/../output/out_%.0f.jpg' % time.time() +INPUT_PATH, STYLE_PATH = None, None + +# Logging params +PRINT_TRAINING_STATUS = True +PRINT_N = 100 + +# Logging config +log_dir = DIR_PATH + '/../log/' +if not os.path.isdir(log_dir): + os.makedirs(log_dir) + print('Directory "%s" was created for logging.' % log_dir) +log_path = ''.join([log_dir, str(time.time()), '.log']) +logging.basicConfig(filename=log_path, level=logging.INFO) +print("Printing log to %s" % log_path) + + +# Given an activated filter maps of any particular layer, return its respected gram matrix +def convert_to_gram(filter_maps): + # Get the dimensions of the filter maps to reshape them into two dimenions + dimension = filter_maps.get_shape().as_list() + reshaped_maps = tf.reshape(filter_maps, [dimension[1] * dimension[2], dimension[3]]) + + # Compute the inner product to get the gram matrix + if dimension[1] * dimension[2] > dimension[3]: + return tf.matmul(reshaped_maps, reshaped_maps, transpose_a=True) + else: + return tf.matmul(reshaped_maps, reshaped_maps, transpose_b=True) + + +# Compute the content loss given a variable image (x) and a content image (c) +def get_content_loss(x, c): + with tf.compat.v1.name_scope('get_content_loss'): + # Get the activated VGG feature maps and return the normalized euclidean distance + noise_representation = getattr(x, CONTENT_LAYER) + photo_representation = getattr(c, CONTENT_LAYER) + return get_l2_norm_loss(noise_representation - photo_representation) + + +# Compute the L2-norm divided by squared number of dimensions +def get_l2_norm_loss(diffs): + shape = diffs.get_shape().as_list() + size = reduce(lambda x, y: x * y, shape) ** 2 + sum_of_squared_diffs = tf.reduce_sum(input_tensor=tf.square(diffs)) + return sum_of_squared_diffs / size + + +# Compute style loss given a variable image (x) and a style image (s) +def get_style_loss(x, s): + with tf.compat.v1.name_scope('get_style_loss'): + style_layer_losses = [get_style_loss_for_layer(x, s, l) for l in STYLE_LAYERS] + style_weights = tf.constant([1. / len(style_layer_losses)] * len(style_layer_losses), tf.float32) + weighted_layer_losses = tf.multiply(style_weights, tf.convert_to_tensor(value=style_layer_losses)) + return tf.reduce_sum(input_tensor=weighted_layer_losses) + + +# Compute style loss for a layer (l) given the variable image (x) and the style image (s) +def get_style_loss_for_layer(x, s, l): + with tf.compat.v1.name_scope('get_style_loss_for_layer'): + # Compute gram matrices using the activated filter maps of the art and generated images + x_layer_maps = getattr(x, l) + s_layer_maps = getattr(s, l) + x_layer_gram = convert_to_gram(x_layer_maps) + s_layer_gram = convert_to_gram(s_layer_maps) + + # Make sure the feature map dimensions are the same + assert_equal_shapes = tf.compat.v1.assert_equal(x_layer_maps.get_shape(), s_layer_maps.get_shape()) + with tf.control_dependencies([assert_equal_shapes]): + # Compute and return the normalized gram loss using the gram matrices + shape = x_layer_maps.get_shape().as_list() + size = reduce(lambda a, b: a * b, shape) ** 2 + gram_loss = get_l2_norm_loss(x_layer_gram - s_layer_gram) + return gram_loss / size + + +# Compute total variation regularization loss term given a variable image (x) and its shape +def get_total_variation(x, shape): + with tf.compat.v1.name_scope('get_total_variation'): + # Get the dimensions of the variable image + height = shape[1] + width = shape[2] + size = reduce(lambda a, b: a * b, shape) ** 2 + + # Disjoin the variable image and evaluate the total variation + x_cropped = x[:, :height - 1, :width - 1, :] + left_term = tf.square(x[:, 1:, :width - 1, :] - x_cropped) + right_term = tf.square(x[:, :height - 1, 1:, :] - x_cropped) + smoothed_terms = tf.pow(left_term + right_term, TOTAL_VARIATION_SMOOTHING / 2.) + return tf.reduce_sum(input_tensor=smoothed_terms) / size + + +# Parse arguments and assign them to their respective global variables +def parse_args(): + global INPUT_PATH, STYLE_PATH, OUT_PATH + + parser = argparse.ArgumentParser() + parser.add_argument("input", help="path to the input image you'd like to apply the style to") + parser.add_argument("style", help="path to the image you'd like to reference the style from") + parser.add_argument("--out", default=OUT_PATH, help="path to where the styled image will be created") + args = parser.parse_args() + + # Assign image paths from the arg parsing + INPUT_PATH = os.path.realpath(args.input) + STYLE_PATH = os.path.realpath(args.style) + OUT_PATH = os.path.realpath(args.out) + +with tf.compat.v1.Session() as sess: +# with tf.Session() as sess: + parse_args() + + # Initialize and process photo image to be used for our content + photo, image_shape = utilsV2.load_image(INPUT_PATH) + image_shape = [1] + image_shape + photo = photo.reshape(image_shape).astype(np.float32) + + # Initialize and process art image to be used for our style + art = utilsV2.load_image2(STYLE_PATH, height=image_shape[1], width=image_shape[2]) + art = art.reshape(image_shape).astype(np.float32) + + # Initialize the variable image that will become our final output as random noise + noise = tf.Variable(tf.random.truncated_normal(image_shape, mean=.5, stddev=.1)) + + # VGG Networks Init + with tf.compat.v1.name_scope('vgg_content'): + content_model = vgg19.Vgg19() + content_model.build(photo, image_shape[1:]) + + with tf.compat.v1.name_scope('vgg_style'): + style_model = vgg19.Vgg19() + style_model.build(art, image_shape[1:]) + + with tf.compat.v1.name_scope('vgg_x'): + x_model = vgg19.Vgg19() + x_model.build(noise, image_shape[1:]) + + # Loss functions + with tf.compat.v1.name_scope('loss'): + # Content + if CONTENT_WEIGHT is 0: + content_loss = tf.constant(0.) + else: + content_loss = get_content_loss(x_model, content_model) * CONTENT_WEIGHT + + # Style + if STYLE_WEIGHT is 0: + style_loss = tf.constant(0.) + else: + style_loss = get_style_loss(x_model, style_model) * STYLE_WEIGHT + + # Norm regularization + if NORM_WEIGHT is 0: + norm_loss = tf.constant(0.) + else: + norm_loss = (get_l2_norm_loss(noise) ** NORM_TERM) * NORM_WEIGHT + + # Total variation denoising + if TV_WEIGHT is 0: + tv_loss = tf.constant(0.) + else: + tv_loss = get_total_variation(noise, image_shape) * TV_WEIGHT + + # Total loss + total_loss = content_loss + style_loss + norm_loss + tv_loss + + # Update image + with tf.compat.v1.name_scope('update_image'): + optimizer = tf.compat.v1.train.AdamOptimizer(LEARNING_RATE) + grads = optimizer.compute_gradients(total_loss, [noise]) + clipped_grads = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in grads] + update_image = optimizer.apply_gradients(clipped_grads) + + # Train + logging.info("Initializing variables and beginning training..") + sess.run(tf.compat.v1.global_variables_initializer()) + start_time = time.time() + for i in range(EPOCHS): + _, loss = sess.run([update_image, total_loss]) + if PRINT_TRAINING_STATUS and i % PRINT_N == 0: + logging.info("Epoch %04d | Loss %.03f" % (i, loss)) + + # FIN + elapsed = time.time() - start_time + logging.info("Training complete. The session took %.2f seconds to complete." % elapsed) + logging.info("Rendering final image and closing TensorFlow session..") + + # Render the image after making sure the repo's dedicated output dir exists + out_dir = os.path.dirname(os.path.realpath(__file__)) + '/../output/' + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + utilsV2.render_img(sess, noise, save=True, out_path=OUT_PATH) diff --git a/src/utils.py b/src/utils.py index a14789b..ce23875 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,7 +3,11 @@ import skimage import skimage.io import skimage.transform -from scipy.misc import toimage +import PIL +import scipy +# from scipy.misc import toimage +from PIL import Image + # Return a numpy array of an image specified by its path @@ -48,6 +52,8 @@ def render_img(session, x, save=False, out_path=None): img = np.clip(session.run(x), 0, 1) if save: - toimage(np.reshape(img, shape[1:])).save(out_path) + Image.fromarray(np.reshape(img, shape[1:])).save(out_path) + # toimage(np.reshape(img, shape[1:])).save(out_path) else: - toimage(np.reshape(img, shape[1:])).show() \ No newline at end of file + Image.fromarray(np.reshape(img, shape[1:])).show() + diff --git a/src/utilsV2.py b/src/utilsV2.py new file mode 100644 index 0000000..31f5b27 --- /dev/null +++ b/src/utilsV2.py @@ -0,0 +1,59 @@ +import numpy as np +import os + +import skimage +import skimage.io +import skimage.transform +import PIL +import scipy +# from scipy.misc import toimage +from PIL import Image + + +# Return a numpy array of an image specified by its path +def load_image(path): + # Load image [height, width, depth] + img = skimage.io.imread(path) / 255.0 + assert (0 <= img).all() and (img <= 1.0).all() + + # Crop image from center + short_edge = min(img.shape[:2]) + yy = int((img.shape[0] - short_edge) / 2) + xx = int((img.shape[1] - short_edge) / 2) + shape = list(img.shape) + + crop_img = img[yy: yy + short_edge, xx: xx + short_edge] + resized_img = skimage.transform.resize(crop_img, (shape[0], shape[1])) + return resized_img, shape + + +# Return a resized numpy array of an image specified by its path +def load_image2(path, height=None, width=None): + # Load image + img = skimage.io.imread(path) / 255.0 + if height is not None and width is not None: + ny = height + nx = width + elif height is not None: + ny = height + nx = img.shape[1] * ny / img.shape[0] + elif width is not None: + nx = width + ny = img.shape[0] * nx / img.shape[1] + else: + ny = img.shape[0] + nx = img.shape[1] + return skimage.transform.resize(img, (ny, nx)) + + +# Render the generated image given a tensorflow session and a variable image (x) +def render_img(session, x, save=False, out_path=None): + shape = x.get_shape().as_list() + img = (np.clip(session.run(x), 0, 1) * 255).astype(np.uint8) + + if save: + Image.fromarray(np.reshape(img, shape[1:])).resize((1200, 1200)).save(out_path) + # toimage(np.reshape(img, shape[1:])).save(out_path) + else: + Image.fromarray(np.reshape(img, shape[1:])).show() +