ml-labs-02-linear-regression-from-scratch.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/sacrum/ml-labs-02-linear-regression-from-scratch?scriptVersionId=178243428\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","id":"425a33dc","metadata":{"papermill":{"duration":0.010592,"end_time":"2024-05-17T16:28:37.838036","exception":false,"start_time":"2024-05-17T16:28:37.827444","status":"completed"},"tags":[]},"source":["# 1. Linear Regression\n","\n","In this section we will implement basic functions that are used for training a linear regression model"]},{"cell_type":"markdown","id":"0a45d16e","metadata":{"papermill":{"duration":0.010123,"end_time":"2024-05-17T16:28:37.858941","exception":false,"start_time":"2024-05-17T16:28:37.848818","status":"completed"},"tags":[]},"source":["### Setting Up"]},{"cell_type":"code","execution_count":1,"id":"1c7669fb","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:37.881334Z","iopub.status.busy":"2024-05-17T16:28:37.880386Z","iopub.status.idle":"2024-05-17T16:28:37.893915Z","shell.execute_reply":"2024-05-17T16:28:37.892924Z"},"papermill":{"duration":0.027735,"end_time":"2024-05-17T16:28:37.896614","exception":false,"start_time":"2024-05-17T16:28:37.868879","status":"completed"},"tags":[]},"outputs":[],"source":["import numpy as np"]},{"cell_type":"code","execution_count":2,"id":"61325705","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:37.918737Z","iopub.status.busy":"2024-05-17T16:28:37.918309Z","iopub.status.idle":"2024-05-17T16:28:37.923658Z","shell.execute_reply":"2024-05-17T16:28:37.922515Z"},"papermill":{"duration":0.019273,"end_time":"2024-05-17T16:28:37.926038","exception":false,"start_time":"2024-05-17T16:28:37.906765","status":"completed"},"tags":[]},"outputs":[],"source":["# this function prints the vector with its shape\n","\n","def print_vector(vector, vector_name):\n","\tprint(f\"\"\">> {vector_name}.shape\\n{vector.shape}\\n\\n>> {vector_name}\\n{vector}\\n\"\"\")"]},{"cell_type":"code","execution_count":3,"id":"d0949417","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:37.948668Z","iopub.status.busy":"2024-05-17T16:28:37.947847Z","iopub.status.idle":"2024-05-17T16:28:37.954545Z","shell.execute_reply":"2024-05-17T16:28:37.953133Z"},"papermill":{"duration":0.021706,"end_time":"2024-05-17T16:28:37.957476","exception":false,"start_time":"2024-05-17T16:28:37.93577","status":"completed"},"tags":[]},"outputs":[],"source":["# Initialize Dummy Data\n","\n","n_dims = 3\n","n_samples = 10\n","\n","weight = np.random.rand(n_dims)\n","feature_vector = np.random.rand(n_samples, n_dims)\n","y_true = np.random.rand(n_samples)"]},{"cell_type":"code","execution_count":4,"id":"cd58bc7e","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:37.979657Z","iopub.status.busy":"2024-05-17T16:28:37.978862Z","iopub.status.idle":"2024-05-17T16:28:37.988981Z","shell.execute_reply":"2024-05-17T16:28:37.987429Z"},"papermill":{"duration":0.024521,"end_time":"2024-05-17T16:28:37.991873","exception":false,"start_time":"2024-05-17T16:28:37.967352","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":[">> weight.shape\n","(3,)\n","\n",">> weight\n","[0.27390397 0.58489484 0.03643782]\n","\n",">> feature_vector.shape\n","(10, 3)\n","\n",">> feature_vector\n","[[0.01059579 0.6124475  0.29862076]\n"," [0.56801277 0.196372   0.7172178 ]\n"," [0.55751561 0.24821628 0.41610273]\n"," [0.30215765 0.32051184 0.35208816]\n"," [0.19319418 0.07730115 0.71617865]\n"," [0.39898015 0.25610437 0.43126924]\n"," [0.78716473 0.54406537 0.79419203]\n"," [0.92592103 0.07131411 0.76749022]\n"," [0.02865615 0.35722214 0.93128407]\n"," [0.06399781 0.58180371 0.57440601]]\n","\n",">> y_true.shape\n","(10,)\n","\n",">> y_true\n","[0.79012808 0.6416438  0.06313733 0.97930555 0.07768066 0.09060499\n"," 0.00745351 0.4453443  0.61780283 0.52368578]\n","\n"]}],"source":["print_vector(weight, \"weight\")\n","print_vector(feature_vector, \"feature_vector\")\n","print_vector(y_true, \"y_true\")"]},{"cell_type":"markdown","id":"8cc7309d","metadata":{"papermill":{"duration":0.011994,"end_time":"2024-05-17T16:28:38.015906","exception":false,"start_time":"2024-05-17T16:28:38.003912","status":"completed"},"tags":[]},"source":["### Linear regression single prediction\n","Create a function that takes two vectors as input (weight and feature vector) and returns the linear regression prediction for that input."]},{"cell_type":"code","execution_count":5,"id":"a4b7791a","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.040571Z","iopub.status.busy":"2024-05-17T16:28:38.039998Z","iopub.status.idle":"2024-05-17T16:28:38.047806Z","shell.execute_reply":"2024-05-17T16:28:38.045686Z"},"papermill":{"duration":0.023706,"end_time":"2024-05-17T16:28:38.050511","exception":false,"start_time":"2024-05-17T16:28:38.026805","status":"completed"},"tags":[]},"outputs":[],"source":["def linear_regression_prediction(weight, feature_vector):\n","    return np.dot(weight, feature_vector)"]},{"cell_type":"code","execution_count":6,"id":"67cdc486","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.073231Z","iopub.status.busy":"2024-05-17T16:28:38.072849Z","iopub.status.idle":"2024-05-17T16:28:38.078738Z","shell.execute_reply":"2024-05-17T16:28:38.077649Z"},"papermill":{"duration":0.020454,"end_time":"2024-05-17T16:28:38.08128","exception":false,"start_time":"2024-05-17T16:28:38.060826","status":"completed"},"tags":[]},"outputs":[],"source":["single_predictions = linear_regression_prediction(weight, feature_vector[0])"]},{"cell_type":"code","execution_count":7,"id":"ef771c11","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.105489Z","iopub.status.busy":"2024-05-17T16:28:38.104771Z","iopub.status.idle":"2024-05-17T16:28:38.110879Z","shell.execute_reply":"2024-05-17T16:28:38.109203Z"},"papermill":{"duration":0.020357,"end_time":"2024-05-17T16:28:38.113182","exception":false,"start_time":"2024-05-17T16:28:38.092825","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":[">> single_predictions.shape\n","()\n","\n",">> single_predictions\n","0.37200069924579143\n","\n"]}],"source":["print_vector(single_predictions, \"single_predictions\")"]},{"cell_type":"markdown","id":"de7f563b","metadata":{"papermill":{"duration":0.010519,"end_time":"2024-05-17T16:28:38.134287","exception":false,"start_time":"2024-05-17T16:28:38.123768","status":"completed"},"tags":[]},"source":["### Linear regression vector prediction\n","Create a function that takes a matrix and a vector as input (weight vector and feature matrix) and returns the linear regression prediction vector for the whole training set."]},{"cell_type":"code","execution_count":8,"id":"0614d624","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.158875Z","iopub.status.busy":"2024-05-17T16:28:38.157956Z","iopub.status.idle":"2024-05-17T16:28:38.165183Z","shell.execute_reply":"2024-05-17T16:28:38.163908Z"},"papermill":{"duration":0.022912,"end_time":"2024-05-17T16:28:38.168888","exception":false,"start_time":"2024-05-17T16:28:38.145976","status":"completed"},"tags":[]},"outputs":[],"source":["def linear_regression_vector_prediction(weight_vector, feature_matrix):\n","    return np.dot(feature_matrix, weight_vector)"]},{"cell_type":"code","execution_count":9,"id":"2d88bfee","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.193272Z","iopub.status.busy":"2024-05-17T16:28:38.192778Z","iopub.status.idle":"2024-05-17T16:28:38.198612Z","shell.execute_reply":"2024-05-17T16:28:38.197241Z"},"papermill":{"duration":0.021463,"end_time":"2024-05-17T16:28:38.201833","exception":false,"start_time":"2024-05-17T16:28:38.18037","status":"completed"},"tags":[]},"outputs":[],"source":["predictions = linear_regression_vector_prediction(weight, feature_vector)"]},{"cell_type":"code","execution_count":10,"id":"97be14d3","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.229304Z","iopub.status.busy":"2024-05-17T16:28:38.228807Z","iopub.status.idle":"2024-05-17T16:28:38.23655Z","shell.execute_reply":"2024-05-17T16:28:38.234663Z"},"papermill":{"duration":0.026507,"end_time":"2024-05-17T16:28:38.239457","exception":false,"start_time":"2024-05-17T16:28:38.21295","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":[">> predictions.shape\n","(10,)\n","\n",">> predictions\n","[0.3720007  0.29657177 0.31304803 0.28305723 0.12422569 0.27479088\n"," 0.56276719 0.32329037 0.25072038 0.37875335]\n","\n"]}],"source":["print_vector(predictions, \"predictions\")"]},{"cell_type":"markdown","id":"f0ec7536","metadata":{"papermill":{"duration":0.012209,"end_time":"2024-05-17T16:28:38.262983","exception":false,"start_time":"2024-05-17T16:28:38.250774","status":"completed"},"tags":[]},"source":["### Mean Squared Error\n","Now create a function that takes a vector of predictions and a vector of actual values as input and returns the Mean Squared Error."]},{"cell_type":"code","execution_count":11,"id":"8404ee0b","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.288692Z","iopub.status.busy":"2024-05-17T16:28:38.28829Z","iopub.status.idle":"2024-05-17T16:28:38.293774Z","shell.execute_reply":"2024-05-17T16:28:38.292527Z"},"papermill":{"duration":0.021706,"end_time":"2024-05-17T16:28:38.296286","exception":false,"start_time":"2024-05-17T16:28:38.27458","status":"completed"},"tags":[]},"outputs":[],"source":["def mean_squared_error(predictions, actual_values):\n","    return np.mean((predictions - actual_values) ** 2)"]},{"cell_type":"code","execution_count":12,"id":"17f0e8af","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.323734Z","iopub.status.busy":"2024-05-17T16:28:38.323208Z","iopub.status.idle":"2024-05-17T16:28:38.329837Z","shell.execute_reply":"2024-05-17T16:28:38.32839Z"},"papermill":{"duration":0.024784,"end_time":"2024-05-17T16:28:38.333178","exception":false,"start_time":"2024-05-17T16:28:38.308394","status":"completed"},"tags":[]},"outputs":[],"source":["loss = mean_squared_error(predictions, y_true)"]},{"cell_type":"code","execution_count":13,"id":"6e7dd10c","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.36044Z","iopub.status.busy":"2024-05-17T16:28:38.360058Z","iopub.status.idle":"2024-05-17T16:28:38.366486Z","shell.execute_reply":"2024-05-17T16:28:38.364962Z"},"papermill":{"duration":0.024707,"end_time":"2024-05-17T16:28:38.369495","exception":false,"start_time":"2024-05-17T16:28:38.344788","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":[">> loss.shape\n","()\n","\n",">> loss\n","0.13562385653777925\n","\n"]}],"source":["print_vector(loss, \"loss\")"]},{"cell_type":"markdown","id":"17cabfb0","metadata":{"papermill":{"duration":0.011692,"end_time":"2024-05-17T16:28:38.39383","exception":false,"start_time":"2024-05-17T16:28:38.382138","status":"completed"},"tags":[]},"source":["### MSE Gradient\n","Now create a function that takes a vector of predictions and a vector of actual values as input and returns the Gradient of Mean Squared Error."]},{"cell_type":"code","execution_count":14,"id":"8f66a1c4","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.420092Z","iopub.status.busy":"2024-05-17T16:28:38.419701Z","iopub.status.idle":"2024-05-17T16:28:38.425997Z","shell.execute_reply":"2024-05-17T16:28:38.424712Z"},"papermill":{"duration":0.02396,"end_time":"2024-05-17T16:28:38.42906","exception":false,"start_time":"2024-05-17T16:28:38.4051","status":"completed"},"tags":[]},"outputs":[],"source":["def mse_gradient(feature_vector, predictions, actual_values):\n","    # grad = (2/n) * X * (P-Y)\n","\n","    # (P-Y)\n","    diff = predictions - actual_values\n","\n","    # X * diff\n","    dot = np.dot(feature_vector.T, diff)\n","\n","    n = len(feature_vector)\n","    grad = (2/n) * dot\n","\n","    return grad"]},{"cell_type":"code","execution_count":15,"id":"5a0ca1b7","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.455384Z","iopub.status.busy":"2024-05-17T16:28:38.454987Z","iopub.status.idle":"2024-05-17T16:28:38.460084Z","shell.execute_reply":"2024-05-17T16:28:38.458752Z"},"papermill":{"duration":0.021542,"end_time":"2024-05-17T16:28:38.462665","exception":false,"start_time":"2024-05-17T16:28:38.441123","status":"completed"},"tags":[]},"outputs":[],"source":["grad = mse_gradient(feature_vector, predictions, y_true)"]},{"cell_type":"code","execution_count":16,"id":"e188bc75","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.489679Z","iopub.status.busy":"2024-05-17T16:28:38.489262Z","iopub.status.idle":"2024-05-17T16:28:38.496985Z","shell.execute_reply":"2024-05-17T16:28:38.495485Z"},"papermill":{"duration":0.023981,"end_time":"2024-05-17T16:28:38.499827","exception":false,"start_time":"2024-05-17T16:28:38.475846","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":[">> grad.shape\n","(3,)\n","\n",">> grad\n","[ 0.02306238 -0.07124562 -0.09569898]\n","\n"]}],"source":["print_vector(grad, \"grad\")"]},{"cell_type":"markdown","id":"27185fa7","metadata":{"papermill":{"duration":0.012623,"end_time":"2024-05-17T16:28:38.525549","exception":false,"start_time":"2024-05-17T16:28:38.512926","status":"completed"},"tags":[]},"source":["### Gradient Descent Algorithm\n","This function performs batch gradient descent to optimize a linear regression model."]},{"cell_type":"code","execution_count":17,"id":"82913b54","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.554109Z","iopub.status.busy":"2024-05-17T16:28:38.553566Z","iopub.status.idle":"2024-05-17T16:28:38.565472Z","shell.execute_reply":"2024-05-17T16:28:38.563746Z"},"papermill":{"duration":0.029156,"end_time":"2024-05-17T16:28:38.568148","exception":false,"start_time":"2024-05-17T16:28:38.538992","status":"completed"},"tags":[]},"outputs":[],"source":["\n","\"\"\"\n","Batch gradient descent: batch_size=len(X)\n","Stochastic gradient descent: batch_size=1\n","Mini-batch gradient descent: batch_size=32\n","\"\"\"\n","\n","def gradient_descent(X, y, batch_size=32, epochs=10, learning_rate=0.01):\n","\n","\t# n_samples, n_dims\n","\tn, m = X.shape\n","\n","\t# initialize random weights\n","\tweights = np.random.rand(m)\n","\n","\tlosses = []\n","\tfor epoch in range(epochs):\n","\n","\t\tloss = 0\n","\t\tfor iteration in range(0, n, batch_size):\n","\n","\t\t\tbatch_start = iteration\n","\t\t\tbatch_end = iteration + batch_size\n","\n","\t\t\tx_batch = X[batch_start:batch_end]\n","\t\t\ty_batch = y[batch_start:batch_end]\n","\n","\t\t\tpredictions = linear_regression_vector_prediction(weights, x_batch)\n","\n","\t\t\tgradient = mse_gradient(x_batch, predictions, y_batch)\n","\t\t\tweights -= learning_rate * gradient\n","\t\t\t\n","\t\t\tbatch_loss = mean_squared_error(predictions, y_batch)\n","\t\t\tloss += batch_loss\n","\n","\t\tlosses.append(loss)\n","\t\tprint(f\"epoch {epoch+1}/{epochs} | loss {loss}\")\n","\n","\treturn weights, losses"]},{"cell_type":"code","execution_count":18,"id":"9a87d843","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.594657Z","iopub.status.busy":"2024-05-17T16:28:38.594256Z","iopub.status.idle":"2024-05-17T16:28:38.601128Z","shell.execute_reply":"2024-05-17T16:28:38.599661Z"},"papermill":{"duration":0.023992,"end_time":"2024-05-17T16:28:38.603898","exception":false,"start_time":"2024-05-17T16:28:38.579906","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["epoch 1/10 | loss 0.33818799746331574\n","epoch 2/10 | loss 0.28736461232730826\n","epoch 3/10 | loss 0.24897924512567643\n","epoch 4/10 | loss 0.21995183951632158\n","epoch 5/10 | loss 0.1979662194841185\n","epoch 6/10 | loss 0.18128060240383342\n","epoch 7/10 | loss 0.16858511719998276\n","epoch 8/10 | loss 0.15889466723676263\n","epoch 9/10 | loss 0.15146837011329442\n","epoch 10/10 | loss 0.14574898153977262\n"]}],"source":["weights = gradient_descent(feature_vector, y_true, learning_rate=0.1)"]},{"cell_type":"markdown","id":"25c08c4e","metadata":{"papermill":{"duration":0.011445,"end_time":"2024-05-17T16:28:38.628002","exception":false,"start_time":"2024-05-17T16:28:38.616557","status":"completed"},"tags":[]},"source":["# 2. California Housing\n","In this section we will apply Linear Regression functions implemented from scracth above, on California Housing Dataset to predict house prices"]},{"cell_type":"markdown","id":"305f2a7c","metadata":{"papermill":{"duration":0.011507,"end_time":"2024-05-17T16:28:38.651086","exception":false,"start_time":"2024-05-17T16:28:38.639579","status":"completed"},"tags":[]},"source":["### Loading and Scaling Data\n","- Data Source\n","\t- [Kaggle](https://www.kaggle.com/datasets/camnugent/california-housing-prices)\n","\t- [Google Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)"]},{"cell_type":"code","execution_count":19,"id":"49a692da","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:38.680492Z","iopub.status.busy":"2024-05-17T16:28:38.680006Z","iopub.status.idle":"2024-05-17T16:28:42.365391Z","shell.execute_reply":"2024-05-17T16:28:42.363721Z"},"papermill":{"duration":3.701226,"end_time":"2024-05-17T16:28:42.367992","exception":false,"start_time":"2024-05-17T16:28:38.666766","status":"completed"},"tags":[]},"outputs":[{"data":{"text/plain":["((20640, 8), (20640,))"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["# Dataset can be loaded using scikitlearn datasets\n","\n","from sklearn import datasets\n","\n","california_housing = datasets.fetch_california_housing()\n","\n","X = california_housing.data  # Feature matrix\n","y = california_housing.target  # Target values (median house values)\n","\n","# Scale The Data\n","X = (X - X.mean(axis=0)) / X.std(axis=0)\n","y = (y - y.mean()) / y.std()\n","\n","X.shape, y.shape"]},{"cell_type":"markdown","id":"73585732","metadata":{"papermill":{"duration":0.013249,"end_time":"2024-05-17T16:28:42.396052","exception":false,"start_time":"2024-05-17T16:28:42.382803","status":"completed"},"tags":[]},"source":["### Applying Gradient Descents"]},{"cell_type":"code","execution_count":20,"id":"c05b10f9","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:42.425688Z","iopub.status.busy":"2024-05-17T16:28:42.423883Z","iopub.status.idle":"2024-05-17T16:28:42.471925Z","shell.execute_reply":"2024-05-17T16:28:42.470697Z"},"papermill":{"duration":0.073727,"end_time":"2024-05-17T16:28:42.481867","exception":false,"start_time":"2024-05-17T16:28:42.40814","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["epoch 1/100 | loss 2.7378412203269975\n","epoch 2/100 | loss 1.0363077675021954\n","epoch 3/100 | loss 0.671534991760565\n","epoch 4/100 | loss 0.555747856043864\n","epoch 5/100 | loss 0.5111275941682364\n","epoch 6/100 | loss 0.48953331587963117\n","epoch 7/100 | loss 0.47623067953369225\n","epoch 8/100 | loss 0.4664110866545011\n","epoch 9/100 | loss 0.45838965163542933\n","epoch 10/100 | loss 0.4515148358425929\n","epoch 11/100 | loss 0.4454955903489219\n","epoch 12/100 | loss 0.4401749881710063\n","epoch 13/100 | loss 0.43545073861582617\n","epoch 14/100 | loss 0.4312460019306337\n","epoch 15/100 | loss 0.427498058706507\n","epoch 16/100 | loss 0.4241535302888601\n","epoch 17/100 | loss 0.4211660897559262\n","epoch 18/100 | loss 0.41849516685767146\n","epoch 19/100 | loss 0.41610508183949463\n","epoch 20/100 | loss 0.4139643885975785\n","epoch 21/100 | loss 0.4120453380167856\n","epoch 22/100 | loss 0.4103234224386257\n","epoch 23/100 | loss 0.40877698192813544\n","epoch 24/100 | loss 0.4073868611052471\n","epoch 25/100 | loss 0.4061361088918584\n","epoch 26/100 | loss 0.40500971532133084\n","epoch 27/100 | loss 0.4039943806121556\n","epoch 28/100 | loss 0.4030783124317402\n","epoch 29/100 | loss 0.4022510478331279\n","epoch 30/100 | loss 0.40150329680493885\n","epoch 31/100 | loss 0.4008268047635396\n","epoch 32/100 | loss 0.4002142316520251\n","epoch 33/100 | loss 0.3996590456024235\n","epoch 34/100 | loss 0.39915542937214643\n","epoch 35/100 | loss 0.39869819798819495\n","epoch 36/100 | loss 0.39828272622717037\n","epoch 37/100 | loss 0.39790488472932106\n","epoch 38/100 | loss 0.39756098369374937\n","epoch 39/100 | loss 0.3972477232321866\n","epoch 40/100 | loss 0.39696214957277676\n","epoch 41/100 | loss 0.39670161640510515\n","epoch 42/100 | loss 0.3964637507450714\n","epoch 43/100 | loss 0.3962464227746916\n","epoch 44/100 | loss 0.3960477191788763\n","epoch 45/100 | loss 0.39586591955987616\n","epoch 46/100 | loss 0.39569947556144036\n","epoch 47/100 | loss 0.39554699237972485\n","epoch 48/100 | loss 0.3954072123773938\n","epoch 49/100 | loss 0.39527900055188975\n","epoch 50/100 | loss 0.3951613316391156\n","epoch 51/100 | loss 0.395053278660289\n","epoch 52/100 | loss 0.39495400274298964\n","epoch 53/100 | loss 0.39486274406781036\n","epoch 54/100 | loss 0.394778813809902\n","epoch 55/100 | loss 0.39470158696039503\n","epoch 56/100 | loss 0.39463049592644395\n","epoch 57/100 | loss 0.3945650248207213\n","epoch 58/100 | loss 0.39450470436179536\n","epoch 59/100 | loss 0.3944491073161388\n","epoch 60/100 | loss 0.39439784442069603\n","epoch 61/100 | loss 0.3943505607321253\n","epoch 62/100 | loss 0.3943069323551498\n","epoch 63/100 | loss 0.3942666635080066\n","epoch 64/100 | loss 0.3942294838878675\n","epoch 65/100 | loss 0.3941951463034055\n","epoch 66/100 | loss 0.3941634245454637\n","epoch 67/100 | loss 0.39413411147011473\n","epoch 68/100 | loss 0.39410701727133624\n","epoch 69/100 | loss 0.3940819679231103\n","epoch 70/100 | loss 0.39405880377304103\n","epoch 71/100 | loss 0.394037378271591\n","epoch 72/100 | loss 0.3940175568228177\n","epoch 73/100 | loss 0.3939992157440552\n","epoch 74/100 | loss 0.3939822413233759\n","epoch 75/100 | loss 0.3939665289648873\n","epoch 76/100 | loss 0.39395198241300666\n","epoch 77/100 | loss 0.39393851304781224\n","epoch 78/100 | loss 0.39392603924442066\n","epoch 79/100 | loss 0.39391448579009114\n","epoch 80/100 | loss 0.39390378335342663\n","epoch 81/100 | loss 0.3938938680006332\n","epoch 82/100 | loss 0.3938846807543238\n","epoch 83/100 | loss 0.3938761671908233\n","epoch 84/100 | loss 0.3938682770723437\n","epoch 85/100 | loss 0.39386096401077125\n","epoch 86/100 | loss 0.3938541851601358\n","epoch 87/100 | loss 0.39384790093512617\n","epoch 88/100 | loss 0.3938420747532784\n","epoch 89/100 | loss 0.3938366727986979\n","epoch 90/100 | loss 0.3938316638053852\n","epoch 91/100 | loss 0.3938270188584216\n","epoch 92/100 | loss 0.39382271121144197\n","epoch 93/100 | loss 0.39381871611896624\n","epoch 94/100 | loss 0.39381501068230274\n","epoch 95/100 | loss 0.3938115737078524\n","epoch 96/100 | loss 0.3938083855767523\n","epoch 97/100 | loss 0.39380542812489805\n","epoch 98/100 | loss 0.39380268453246847\n","epoch 99/100 | loss 0.39380013922215684\n","epoch 100/100 | loss 0.3937977777653866\n"]}],"source":["# Apply Batch (Vanilla) Gradient Descent\n","\n","batch_gd_weights, batch_gd_losses = gradient_descent(\n","\tX, y,\n","\tbatch_size=len(X),\n","\tepochs=100,\n","\tlearning_rate=0.4,\n",")"]},{"cell_type":"code","execution_count":21,"id":"e91ffa79","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:42.540658Z","iopub.status.busy":"2024-05-17T16:28:42.539959Z","iopub.status.idle":"2024-05-17T16:28:51.253948Z","shell.execute_reply":"2024-05-17T16:28:51.252459Z"},"papermill":{"duration":8.745306,"end_time":"2024-05-17T16:28:51.256848","exception":false,"start_time":"2024-05-17T16:28:42.511542","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["epoch 1/15 | loss 16437.312375524853\n","epoch 2/15 | loss 9370.945585916368\n","epoch 3/15 | loss 8538.964737954915\n","epoch 4/15 | loss 8222.168607181642\n","epoch 5/15 | loss 8083.914501931886\n","epoch 6/15 | loss 8015.746799781206\n","epoch 7/15 | loss 7978.504369434242\n","epoch 8/15 | loss 7956.586074310669\n","epoch 9/15 | loss 7943.00539590955\n","epoch 10/15 | loss 7934.285151695873\n","epoch 11/15 | loss 7928.54395166849\n","epoch 12/15 | loss 7924.696994318178\n","epoch 13/15 | loss 7922.087492679448\n","epoch 14/15 | loss 7920.302441537773\n","epoch 15/15 | loss 7919.074436194764\n"]}],"source":["# Apply Stochastic Gradient Descent\n","\n","stochastic_gd_weights, stochastic_gd_losses = gradient_descent(\n","\tX, y,\n","\tbatch_size=1,\n","\tepochs=15,\n","\tlearning_rate=0.0002,\n",")"]},{"cell_type":"code","execution_count":22,"id":"06612446","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:51.294877Z","iopub.status.busy":"2024-05-17T16:28:51.293699Z","iopub.status.idle":"2024-05-17T16:28:51.474059Z","shell.execute_reply":"2024-05-17T16:28:51.472502Z"},"papermill":{"duration":0.199695,"end_time":"2024-05-17T16:28:51.477505","exception":false,"start_time":"2024-05-17T16:28:51.27781","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["epoch 1/15 | loss 256.77767811135914\n","epoch 2/15 | loss 151.49911960179924\n","epoch 3/15 | loss 143.31823009347804\n","epoch 4/15 | loss 138.36544539165948\n","epoch 5/15 | loss 135.4234512648322\n","epoch 6/15 | loss 133.58684238072743\n","epoch 7/15 | loss 132.39577460187846\n","epoch 8/15 | loss 131.59758017590138\n","epoch 9/15 | loss 131.04798112436706\n","epoch 10/15 | loss 130.66088432022943\n","epoch 11/15 | loss 130.38299378329182\n","epoch 12/15 | loss 130.18025686264042\n","epoch 13/15 | loss 130.03031858414033\n","epoch 14/15 | loss 129.91815088286893\n","epoch 15/15 | loss 129.83343459931228\n"]}],"source":["# Apply Mini-Batch Gradient Descent\n","\n","mini_batch_gd_weights, mini_batch_gd_losses = gradient_descent(\n","\tX, y,\n","\tbatch_size=64,\n","\tepochs=15,\n","\tlearning_rate=0.008\n",")"]},{"cell_type":"markdown","id":"661c24e3","metadata":{"papermill":{"duration":0.014781,"end_time":"2024-05-17T16:28:51.507394","exception":false,"start_time":"2024-05-17T16:28:51.492613","status":"completed"},"tags":[]},"source":["### Comparing Mean Squared Error on 3 Gradient Descents"]},{"cell_type":"code","execution_count":23,"id":"05868304","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:51.537792Z","iopub.status.busy":"2024-05-17T16:28:51.537308Z","iopub.status.idle":"2024-05-17T16:28:51.551117Z","shell.execute_reply":"2024-05-17T16:28:51.549281Z"},"papermill":{"duration":0.037819,"end_time":"2024-05-17T16:28:51.559268","exception":false,"start_time":"2024-05-17T16:28:51.521449","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Mean Squared Error on Batch Gradient Descent\n","0.39379558679584864\n"]}],"source":["preds = linear_regression_vector_prediction(batch_gd_weights, X)\n","mse = mean_squared_error(preds, y)\n","\n","print(\"Mean Squared Error on Batch Gradient Descent\")\n","print(mse)"]},{"cell_type":"code","execution_count":24,"id":"562cfbf0","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:51.628541Z","iopub.status.busy":"2024-05-17T16:28:51.627796Z","iopub.status.idle":"2024-05-17T16:28:51.643182Z","shell.execute_reply":"2024-05-17T16:28:51.641556Z"},"papermill":{"duration":0.061324,"end_time":"2024-05-17T16:28:51.652316","exception":false,"start_time":"2024-05-17T16:28:51.590992","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Mean Squared Error on Stochastic Gradient Descent\n","0.40076754475946436\n"]}],"source":["preds = linear_regression_vector_prediction(stochastic_gd_weights, X)\n","mse = mean_squared_error(preds, y)\n","\n","print(\"Mean Squared Error on Stochastic Gradient Descent\")\n","print(mse)"]},{"cell_type":"code","execution_count":25,"id":"b638d5e3","metadata":{"execution":{"iopub.execute_input":"2024-05-17T16:28:51.720962Z","iopub.status.busy":"2024-05-17T16:28:51.720232Z","iopub.status.idle":"2024-05-17T16:28:51.731914Z","shell.execute_reply":"2024-05-17T16:28:51.729924Z"},"papermill":{"duration":0.054774,"end_time":"2024-05-17T16:28:51.741088","exception":false,"start_time":"2024-05-17T16:28:51.686314","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Mean Squared Error on Mini Batch Gradient Descent\n","0.39718631227489\n"]}],"source":["preds = linear_regression_vector_prediction(mini_batch_gd_weights, X)\n","mse = mean_squared_error(preds, y)\n","\n","print(\"Mean Squared Error on Mini Batch Gradient Descent\")\n","print(mse)"]},{"cell_type":"code","execution_count":null,"id":"b2815ce8","metadata":{"papermill":{"duration":0.03414,"end_time":"2024-05-17T16:28:51.81096","exception":false,"start_time":"2024-05-17T16:28:51.77682","status":"completed"},"tags":[]},"outputs":[],"source":[]},{"cell_type":"markdown","id":"ced20796","metadata":{"papermill":{"duration":0.017055,"end_time":"2024-05-17T16:28:51.86217","exception":false,"start_time":"2024-05-17T16:28:51.845115","status":"completed"},"tags":[]},"source":["**Find More Labs**\n","\n","This lab is from my Machine Learning Course, that is a part of my [Software Engineering](https://seecs.nust.edu.pk/program/bachelor-of-software-engineering-for-fall-2021-onward) Degree at [NUST](https://nust.edu.pk).\n","\n","The content in the provided list of notebooks covers a range of topics in **machine learning** and **data analysis** implemented from scratch or using popular libraries like **NumPy**, **pandas**, **scikit-learn**, **seaborn**, and **matplotlib**. It includes introductory materials on NumPy showcasing its efficiency for mathematical operations, **linear regression**, **logistic regression**, **decision trees**, **K-nearest neighbors (KNN)**, **support vector machines (SVM)**, **Naive Bayes**, **K-means** clustering, principle component analysis (**PCA**), and **neural networks** with **backpropagation**. Each notebook demonstrates practical implementation and application of these algorithms on various datasets such as the **California Housing** Dataset, **MNIST** dataset, **Iris** dataset, **Auto-MPG** dataset, and the **UCI Adult Census Income** dataset. Additionally, it covers topics like **gradient descent optimization**, model evaluation metrics (e.g., **accuracy, precision, recall, f1 score**), **regularization** techniques (e.g., **Lasso**, **Ridge**), and **data visualization**.\n","\n","| Title                                                                                                                   | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n","| ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n","| [01 - Intro to Numpy](https://www.kaggle.com/code/sacrum/ml-labs-01-intro-to-numpy)                                     | The notebook demonstrates NumPy's efficiency for mathematical operations like array `reshaping`, `sigmoid`, `softmax`, `dot` and `outer products`, `L1 and L2 losses`, and matrix operations. It highlights NumPy's superiority over standard Python lists in speed and convenience for scientific computing and machine learning tasks.                                                                                                                                                                                              |\n","| [02 - Linear Regression From Scratch](https://www.kaggle.com/code/sacrum/ml-labs-02-linear-regression-from-scratch)     | This notebook implements `linear regression` and `gradient descent` from scratch in Python using `NumPy`, focusing on predicting house prices with the `California Housing Dataset`. It defines functions for prediction, `MSE` calculation, and gradient computation. Batch gradient descent is used for optimization. The dataset is loaded, scaled, and split. `Batch, stochastic, and mini-batch gradient descents` are applied with varying hyperparameters. Finally, the MSEs of the predictions from each method are compared. |\n","| [03 - Logistic Regression from Scratch](https://www.kaggle.com/code/sacrum/ml-labs-03-logistic-regression-from-scratch) | This notebook outlines the implementation of `logistic regression` from scratch in Python using `NumPy`, including functions for prediction, loss calculation, gradient computation, and batch `gradient descent` optimization, applied to the `MNIST` dataset for handwritten digit recognition and `Iris` data. And also inclues metrics like `accuracy`, `precision`, `recall`, `f1 score`                                                                                                                                         |\n","| [04 - Auto-MPG Regression](https://www.kaggle.com/code/sacrum/ml-labs-04-auto-mpg-regression)                           | The notebook uses `pandas` for data manipulation, `seaborn` and `matplotlib` for visualization, and `sklearn` for `linear regression` and `regularization` techniques (`Lasso` and `Ridge`). It includes data loading, processing, visualization, model training, and evaluation on the `Auto-MPG dataset`.                                                                                                                                                                                                                           |\n","| [05 - Desicion Trees from Scratch](https://www.kaggle.com/code/sacrum/ml-labs-05-desicion-trees-from-scratch)           | In this notebook, `DecisionTree` algorithm has been implmented from scratch and applied on dummy dataset                                                                                                                                                                                                                                                                                                                                                                                                                              |\n","| [06 - KNN from Scratch](https://www.kaggle.com/code/sacrum/ml-labs-06-knn-from-scratch)                                 | In this notebook, `K-Nearest Neighbour` algorithm has been implemented from scratch and compared with KNN provided in scikit-learn package                                                                                                                                                                                                                                                                                                                                                                                            |\n","| [07 - SVM](https://www.kaggle.com/code/sacrum/ml-labs-07-svm)                                                           | This notebook implements `SVM classifier` on `Iris Dataset`                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n","| [08 - Naive Bayes](https://www.kaggle.com/code/sacrum/ml-labs-08-naive-bayes)                                           | This notebook trains `Naive Bayes` and compares it with other algorithms `Decision Trees`, `SVM` and `Logistic Regression`                                                                                                                                                                                                                                                                                                                                                                                                            |\n","| [09 - K-means](https://www.kaggle.com/code/sacrum/ml-labs-09-k-means)                                                   | In this notebook `K-means` algorithm has been implemented using `scikit-learn` and different values of `k` are compared to understand the `elbow method` in `Calinski Harabasz Scores`                                                                                                                                                                                                                                                                                                                                                |\n","| [10 - UCI Adult Census Income](https://www.kaggle.com/code/sacrum/ml-labs-10-uci-adult-census-income)                   | Here I have used the UCI Adult Income dataset and applied different machine learning algorithms to find the best model configuration for predicting salary from the given information                                                                                                                                                                                                                                                                                                                                                 |\n","| [11 - PCA](https://www.kaggle.com/code/sacrum/ml-labs-11-pca)                                                           | `Principle Component Analysis` implemented from scratch                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n","| [12 - Neural Networks](https://www.kaggle.com/code/sacrum/ml-labs-12-neural-networks)                                   | This code implements neural networks with back propagation from scratch                                                                                                                                                                                                                                                                                                                                                                                                                                                               |"]},{"cell_type":"code","execution_count":null,"id":"48e610d7","metadata":{"papermill":{"duration":0.016777,"end_time":"2024-05-17T16:28:51.894457","exception":false,"start_time":"2024-05-17T16:28:51.87768","status":"completed"},"tags":[]},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"datasetId":5227,"sourceId":7876,"sourceType":"datasetVersion"}],"dockerImageVersionId":30646,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":18.347831,"end_time":"2024-05-17T16:28:52.537187","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-05-17T16:28:34.189356","version":"2.5.0"}},"nbformat":4,"nbformat_minor":5}