diff --git a/dnn_marathon.ipynb b/dnn_marathon.ipynb index f217f06..b35cec0 100644 --- a/dnn_marathon.ipynb +++ b/dnn_marathon.ipynb @@ -2269,37 +2269,6 @@ " print(labels.size())\n" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "training" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# training\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "learning curve" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "prediction" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -2486,26 +2455,444 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Keras 3 examples" + "### Keras 3\n", + "The full Keras API, available for JAX, TensorFlow, and PyTorch. \n", + "https://keras.io/examples/" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "3.0.0\n" + "3.5.0\n" ] } ], "source": [ + "import os\n", + "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n", + "\n", "import keras\n", + "\n", "print(keras.__version__)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "MNIST convnet" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "x_train shape: (60000, 28, 28, 1)\n", + "y_train shape: (60000,)\n", + "60000 train samples\n", + "10000 test samples\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "# Load the data and split it between train and test sets\n", + "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n", + "\n", + "# Scale images to the [0, 1] range\n", + "x_train = x_train.astype(\"float32\") / 255\n", + "x_test = x_test.astype(\"float32\") / 255\n", + "# Make sure images have shape (28, 28, 1)\n", + "x_train = np.expand_dims(x_train, -1)\n", + "x_test = np.expand_dims(x_test, -1)\n", + "print(\"x_train shape:\", x_train.shape)\n", + "print(\"y_train shape:\", y_train.shape)\n", + "print(x_train.shape[0], \"train samples\")\n", + "print(x_test.shape[0], \"test samples\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Model: \"sequential_1\"\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1mModel: \"sequential_1\"\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
+       "┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
+       "│ conv2d_4 (Conv2D)               │ (None, 26, 26, 64)     │           640 │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ conv2d_5 (Conv2D)               │ (None, 24, 24, 64)     │        36,928 │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ max_pooling2d_1 (MaxPooling2D)  │ (None, 12, 12, 64)     │             0 │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ conv2d_6 (Conv2D)               │ (None, 10, 10, 128)    │        73,856 │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ conv2d_7 (Conv2D)               │ (None, 8, 8, 128)      │       147,584 │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ global_average_pooling2d_1      │ (None, 128)            │             0 │\n",
+       "│ (GlobalAveragePooling2D)        │                        │               │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ dropout_1 (Dropout)             │ (None, 128)            │             0 │\n",
+       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
+       "│ dense_1 (Dense)                 │ (None, 10)             │         1,290 │\n",
+       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n", + "│ conv2d_4 (\u001b[38;5;33mConv2D\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m26\u001b[0m, \u001b[38;5;34m26\u001b[0m, \u001b[38;5;34m64\u001b[0m) │ \u001b[38;5;34m640\u001b[0m │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ conv2d_5 (\u001b[38;5;33mConv2D\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m24\u001b[0m, \u001b[38;5;34m24\u001b[0m, \u001b[38;5;34m64\u001b[0m) │ \u001b[38;5;34m36,928\u001b[0m │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ max_pooling2d_1 (\u001b[38;5;33mMaxPooling2D\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m12\u001b[0m, \u001b[38;5;34m12\u001b[0m, \u001b[38;5;34m64\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ conv2d_6 (\u001b[38;5;33mConv2D\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m10\u001b[0m, \u001b[38;5;34m10\u001b[0m, \u001b[38;5;34m128\u001b[0m) │ \u001b[38;5;34m73,856\u001b[0m │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ conv2d_7 (\u001b[38;5;33mConv2D\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m, \u001b[38;5;34m8\u001b[0m, \u001b[38;5;34m128\u001b[0m) │ \u001b[38;5;34m147,584\u001b[0m │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ global_average_pooling2d_1 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", + "│ (\u001b[38;5;33mGlobalAveragePooling2D\u001b[0m) │ │ │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ dropout_1 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n", + "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", + "│ dense_1 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m10\u001b[0m) │ \u001b[38;5;34m1,290\u001b[0m │\n", + "└─────────────────────────────────┴────────────────────────┴───────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
 Total params: 260,298 (1016.79 KB)\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m260,298\u001b[0m (1016.79 KB)\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
 Trainable params: 260,298 (1016.79 KB)\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m260,298\u001b[0m (1016.79 KB)\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
 Non-trainable params: 0 (0.00 B)\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Model parameters\n", + "num_classes = 10\n", + "input_shape = (28, 28, 1)\n", + "\n", + "model = keras.Sequential(\n", + " [\n", + " keras.layers.Input(shape=input_shape),\n", + " keras.layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\"),\n", + " keras.layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\"),\n", + " keras.layers.MaxPooling2D(pool_size=(2, 2)),\n", + " keras.layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\"),\n", + " keras.layers.Conv2D(128, kernel_size=(3, 3), activation=\"relu\"),\n", + " keras.layers.GlobalAveragePooling2D(),\n", + " keras.layers.Dropout(0.5),\n", + " keras.layers.Dense(num_classes, activation=\"softmax\"),\n", + " ]\n", + ")\n", + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "\u001b[1m399/399\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 36ms/step - acc: 0.9882 - loss: 0.0414 - val_acc: 0.9932 - val_loss: 0.0273\n", + "Epoch 2/10\n", + "\u001b[1m399/399\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 35ms/step - acc: 0.9881 - loss: 0.0381 - val_acc: 0.9933 - val_loss: 0.0240\n", + "Epoch 3/10\n", + "\u001b[1m399/399\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 35ms/step - acc: 0.9883 - loss: 0.0382 - val_acc: 0.9941 - val_loss: 0.0288\n", + "Epoch 4/10\n", + "\u001b[1m399/399\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 35ms/step - acc: 0.9902 - loss: 0.0331 - val_acc: 0.9927 - val_loss: 0.0280\n", + "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 4ms/step\n" + ] + } + ], + "source": [ + "model.compile(\n", + " loss=keras.losses.SparseCategoricalCrossentropy(),\n", + " optimizer=keras.optimizers.Adam(learning_rate=1e-3),\n", + " metrics=[\n", + " keras.metrics.SparseCategoricalAccuracy(name=\"acc\"),\n", + " ],\n", + ")\n", + "\n", + "batch_size = 128\n", + "epochs = 10\n", + "\n", + "callbacks = [\n", + " #keras.callbacks.ModelCheckpoint(filepath=\"model_at_epoch_{epoch}.keras\"),\n", + " keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=2),\n", + "]\n", + "\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " batch_size=batch_size,\n", + " epochs=epochs,\n", + " validation_split=0.15,\n", + " callbacks=callbacks,\n", + ")\n", + "score = model.evaluate(x_test, y_test, verbose=0)\n", + "model.save(\"final_model.keras\")\n", + "model = keras.saving.load_model(\"final_model.keras\")\n", + "predictions = model.predict(x_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An complete customized example" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\zhang\\AppData\\Local\\Temp\\ipykernel_48840\\834262769.py:79: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " real_images = torch.tensor(real_images, device=device)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1m1094/1094\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m56s\u001b[0m 51ms/step - d_loss: 0.4536 - g_loss: 1.2440\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "# This guide can only be run with the torch backend.\n", + "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n", + "\n", + "import torch\n", + "import keras\n", + "from keras import layers\n", + "import numpy as np\n", + "\n", + "# Create the discriminator\n", + "discriminator = keras.Sequential(\n", + " [\n", + " keras.Input(shape=(28, 28, 1)),\n", + " layers.Conv2D(64, (3, 3), strides=(2, 2), padding=\"same\"),\n", + " layers.LeakyReLU(negative_slope=0.2),\n", + " layers.Conv2D(128, (3, 3), strides=(2, 2), padding=\"same\"),\n", + " layers.LeakyReLU(negative_slope=0.2),\n", + " layers.GlobalMaxPooling2D(),\n", + " layers.Dense(1),\n", + " ],\n", + " name=\"discriminator\",\n", + ")\n", + "\n", + "# Create the generator\n", + "latent_dim = 128\n", + "generator = keras.Sequential(\n", + " [\n", + " keras.Input(shape=(latent_dim,)),\n", + " # We want to generate 128 coefficients to reshape into a 7x7x128 map\n", + " layers.Dense(7 * 7 * 128),\n", + " layers.LeakyReLU(negative_slope=0.2),\n", + " layers.Reshape((7, 7, 128)),\n", + " layers.Conv2DTranspose(128, (4, 4), strides=(2, 2), padding=\"same\"),\n", + " layers.LeakyReLU(negative_slope=0.2),\n", + " layers.Conv2DTranspose(128, (4, 4), strides=(2, 2), padding=\"same\"),\n", + " layers.LeakyReLU(negative_slope=0.2),\n", + " layers.Conv2D(1, (7, 7), padding=\"same\", activation=\"sigmoid\"),\n", + " ],\n", + " name=\"generator\",\n", + ")\n", + "\n", + "\n", + "class GAN(keras.Model):\n", + " def __init__(self, discriminator, generator, latent_dim):\n", + " super().__init__()\n", + " self.discriminator = discriminator\n", + " self.generator = generator\n", + " self.latent_dim = latent_dim\n", + " self.d_loss_tracker = keras.metrics.Mean(name=\"d_loss\")\n", + " self.g_loss_tracker = keras.metrics.Mean(name=\"g_loss\")\n", + " self.seed_generator = keras.random.SeedGenerator(1337)\n", + " self.built = True\n", + "\n", + " @property\n", + " def metrics(self):\n", + " return [self.d_loss_tracker, self.g_loss_tracker]\n", + "\n", + " def compile(self, d_optimizer, g_optimizer, loss_fn):\n", + " super().compile()\n", + " self.d_optimizer = d_optimizer\n", + " self.g_optimizer = g_optimizer\n", + " self.loss_fn = loss_fn\n", + "\n", + " def train_step(self, real_images):\n", + " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + " if isinstance(real_images, tuple) or isinstance(real_images, list):\n", + " real_images = real_images[0]\n", + " # Sample random points in the latent space\n", + " batch_size = real_images.shape[0]\n", + " random_latent_vectors = keras.random.normal(\n", + " shape=(batch_size, self.latent_dim), seed=self.seed_generator\n", + " )\n", + "\n", + " # Decode them to fake images\n", + " generated_images = self.generator(random_latent_vectors)\n", + "\n", + " # Combine them with real images\n", + " real_images = torch.tensor(real_images, device=device)\n", + " combined_images = torch.concat([generated_images, real_images], axis=0)\n", + "\n", + " # Assemble labels discriminating real from fake images\n", + " labels = torch.concat(\n", + " [\n", + " torch.ones((batch_size, 1), device=device),\n", + " torch.zeros((batch_size, 1), device=device),\n", + " ],\n", + " axis=0,\n", + " )\n", + " # Add random noise to the labels - important trick!\n", + " labels += 0.05 * keras.random.uniform(labels.shape, seed=self.seed_generator)\n", + "\n", + " # Train the discriminator\n", + " self.zero_grad()\n", + " predictions = self.discriminator(combined_images)\n", + " d_loss = self.loss_fn(labels, predictions)\n", + " d_loss.backward()\n", + " grads = [v.value.grad for v in self.discriminator.trainable_weights]\n", + " with torch.no_grad():\n", + " self.d_optimizer.apply(grads, self.discriminator.trainable_weights)\n", + "\n", + " # Sample random points in the latent space\n", + " random_latent_vectors = keras.random.normal(\n", + " shape=(batch_size, self.latent_dim), seed=self.seed_generator\n", + " )\n", + "\n", + " # Assemble labels that say \"all real images\"\n", + " misleading_labels = torch.zeros((batch_size, 1), device=device)\n", + "\n", + " # Train the generator (note that we should *not* update the weights\n", + " # of the discriminator)!\n", + " self.zero_grad()\n", + " predictions = self.discriminator(self.generator(random_latent_vectors))\n", + " g_loss = self.loss_fn(misleading_labels, predictions)\n", + " grads = g_loss.backward()\n", + " grads = [v.value.grad for v in self.generator.trainable_weights]\n", + " with torch.no_grad():\n", + " self.g_optimizer.apply(grads, self.generator.trainable_weights)\n", + "\n", + " # Update metrics and return their value.\n", + " self.d_loss_tracker.update_state(d_loss)\n", + " self.g_loss_tracker.update_state(g_loss)\n", + " return {\n", + " \"d_loss\": self.d_loss_tracker.result(),\n", + " \"g_loss\": self.g_loss_tracker.result(),\n", + " }\n", + "\n", + "# Prepare the dataset. We use both the training & test MNIST digits.\n", + "batch_size = 64\n", + "(x_train, _), (x_test, _) = keras.datasets.mnist.load_data()\n", + "all_digits = np.concatenate([x_train, x_test])\n", + "all_digits = all_digits.astype(\"float32\") / 255.0\n", + "all_digits = np.reshape(all_digits, (-1, 28, 28, 1))\n", + "\n", + "# Create a TensorDataset\n", + "dataset = torch.utils.data.TensorDataset(\n", + " torch.from_numpy(all_digits), torch.from_numpy(all_digits)\n", + ")\n", + "# Create a DataLoader\n", + "dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)\n", + "\n", + "gan = GAN(discriminator=discriminator, generator=generator, latent_dim=latent_dim)\n", + "gan.compile(\n", + " d_optimizer=keras.optimizers.Adam(learning_rate=0.0003),\n", + " g_optimizer=keras.optimizers.Adam(learning_rate=0.0003),\n", + " loss_fn=keras.losses.BinaryCrossentropy(from_logits=True),\n", + ")\n", + "\n", + "gan.fit(dataloader, epochs=1)" + ] } ], "metadata": { diff --git a/final_model.keras b/final_model.keras new file mode 100644 index 0000000..d5a4c57 Binary files /dev/null and b/final_model.keras differ