{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.439091Z",
     "start_time": "2025-11-05T13:38:02.434601Z"
    }
   },
   "source": [
    "from time import sleep\n",
    "\n",
    "import numpy as np\n",
    "from time import sleep"
   ],
   "outputs": [],
   "execution_count": 122
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.497177Z",
     "start_time": "2025-11-05T13:38:02.491954Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# nn_architecture = [\n",
    "#     {\"input_dim\": 2, \"output_dim\": 4, \"activation\": \"relu\"},\n",
    "#     {\"input_dim\": 4, \"output_dim\": 6, \"activation\": \"relu\"},\n",
    "#     {\"input_dim\": 6, \"output_dim\": 6, \"activation\": \"relu\"},\n",
    "#     {\"input_dim\": 6, \"output_dim\": 4, \"activation\": \"relu\"},\n",
    "#     {\"input_dim\": 4, \"output_dim\": 1, \"activation\": \"sigmoid\"},\n",
    "# ]\n",
    "nn_architecture = [\n",
    "    {\"input_dim\": 2, \"output_dim\": 25, \"activation\": \"relu\"},\n",
    "    {\"input_dim\": 25, \"output_dim\": 50, \"activation\": \"relu\"},\n",
    "    {\"input_dim\": 50, \"output_dim\": 50, \"activation\": \"relu\"},\n",
    "    {\"input_dim\": 50, \"output_dim\": 25, \"activation\": \"relu\"},\n",
    "    {\"input_dim\": 25, \"output_dim\": 1, \"activation\": \"sigmoid\"},\n",
    "]"
   ],
   "id": "48cafaf4b64967bb",
   "outputs": [],
   "execution_count": 123
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.517026Z",
     "start_time": "2025-11-05T13:38:02.511404Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def init_layers(nn_architecture, seed = 99):\n",
    "    np.random.seed(seed)\n",
    "    number_of_layers = len(nn_architecture)\n",
    "    params_values = {}\n",
    "\n",
    "    for idx, layer in enumerate(nn_architecture):\n",
    "        layer_idx = idx + 1\n",
    "        layer_input_size = layer[\"input_dim\"]\n",
    "        layer_output_size = layer[\"output_dim\"]\n",
    "\n",
    "        params_values['W' + str(layer_idx)] = np.random.randn(\n",
    "            layer_output_size, layer_input_size) * 0.1\n",
    "        params_values['b' + str(layer_idx)] = np.random.randn(\n",
    "            layer_output_size, 1) * 0.1\n",
    "\n",
    "    return params_values\n"
   ],
   "id": "d13137630b41b756",
   "outputs": [],
   "execution_count": 124
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.534993Z",
     "start_time": "2025-11-05T13:38:02.531123Z"
    }
   },
   "cell_type": "code",
   "source": [
    "params = init_layers(nn_architecture)\n",
    "# params"
   ],
   "id": "31f205147667dea6",
   "outputs": [],
   "execution_count": 125
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.551697Z",
     "start_time": "2025-11-05T13:38:02.546622Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def sigmoid(Z):\n",
    "    return 1/(1+np.exp(-Z))\n",
    "\n",
    "def relu(Z):\n",
    "    return np.maximum(0,Z)\n",
    "\n",
    "def sigmoid_backward(dA, Z):\n",
    "    sig = sigmoid(Z)\n",
    "    return dA * sig * (1 - sig)\n",
    "\n",
    "def relu_backward(dA, Z):\n",
    "    dZ = np.array(dA, copy = True)\n",
    "    dZ[Z <= 0] = 0;\n",
    "    return dZ;"
   ],
   "id": "c1b960e7dcf09d91",
   "outputs": [],
   "execution_count": 126
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.569859Z",
     "start_time": "2025-11-05T13:38:02.565321Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def single_layer_forward_propagation(A_prev, W_curr, b_curr, activation=\"relu\"):\n",
    "    Z_curr = np.dot(W_curr, A_prev) + b_curr\n",
    "\n",
    "    if activation == \"relu\":\n",
    "        activation_func = relu\n",
    "    elif activation == \"sigmoid\":\n",
    "        activation_func = sigmoid\n",
    "    else:\n",
    "        raise Exception('Non-supported activation function')\n",
    "\n",
    "    return activation_func(Z_curr), Z_curr"
   ],
   "id": "efae2e184daf2fce",
   "outputs": [],
   "execution_count": 127
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.587354Z",
     "start_time": "2025-11-05T13:38:02.582340Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def full_forward_propagation(X, params_values, nn_architecture):\n",
    "    memory = {}\n",
    "    A_curr = X\n",
    "\n",
    "    for idx, layer in enumerate(nn_architecture):\n",
    "        layer_idx = idx + 1\n",
    "        A_prev = A_curr\n",
    "\n",
    "        activ_function_curr = layer[\"activation\"]\n",
    "        W_curr = params_values[\"W\" + str(layer_idx)]\n",
    "        b_curr = params_values[\"b\" + str(layer_idx)]\n",
    "        A_curr, Z_curr = single_layer_forward_propagation(A_prev, W_curr, b_curr, activ_function_curr)\n",
    "\n",
    "        memory[\"A\" + str(idx)] = A_prev\n",
    "        memory[\"Z\" + str(layer_idx)] = Z_curr\n",
    "\n",
    "    return A_curr, memory"
   ],
   "id": "c3cd9e8f51dbe967",
   "outputs": [],
   "execution_count": 128
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.606905Z",
     "start_time": "2025-11-05T13:38:02.601463Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def get_cost_value(Y_hat, Y):\n",
    "    m = Y_hat.shape[1]\n",
    "    cost = -1 / m * (np.dot(Y, np.log(Y_hat).T) + np.dot(1 - Y, np.log(1 - Y_hat).T))\n",
    "    return np.squeeze(cost)\n",
    "\n",
    "# an auxiliary function that converts probability into class\n",
    "def convert_prob_into_class(probs):\n",
    "    probs_ = np.copy(probs)\n",
    "    probs_[probs_ > 0.5] = 1\n",
    "    probs_[probs_ <= 0.5] = 0\n",
    "    return probs_\n",
    "\n",
    "def get_accuracy_value(Y_hat, Y):\n",
    "    Y_hat_ = convert_prob_into_class(Y_hat)\n",
    "    return (Y_hat_ == Y).all(axis=0).mean()"
   ],
   "id": "121416e7bbab57bb",
   "outputs": [],
   "execution_count": 129
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.624438Z",
     "start_time": "2025-11-05T13:38:02.619376Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def single_layer_backward_propagation(dA_curr, W_curr, b_curr, Z_curr, A_prev, activation=\"relu\"):\n",
    "    m = A_prev.shape[1]\n",
    "\n",
    "    if activation == \"relu\":\n",
    "        backward_activation_func = relu_backward\n",
    "    elif activation == \"sigmoid\":\n",
    "        backward_activation_func = sigmoid_backward\n",
    "    else:\n",
    "        raise Exception('Non-supported activation function')\n",
    "\n",
    "    dZ_curr = backward_activation_func(dA_curr, Z_curr)\n",
    "    dW_curr = np.dot(dZ_curr, A_prev.T) / m\n",
    "    db_curr = np.sum(dZ_curr, axis=1, keepdims=True) / m\n",
    "    dA_prev = np.dot(W_curr.T, dZ_curr)\n",
    "\n",
    "    return dA_prev, dW_curr, db_curr"
   ],
   "id": "92e4b87664f18a63",
   "outputs": [],
   "execution_count": 130
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.642647Z",
     "start_time": "2025-11-05T13:38:02.635731Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def full_backward_propagation(Y_hat, Y, memory, params_values, nn_architecture):\n",
    "    grads_values = {}\n",
    "    m = Y.shape[1]\n",
    "    Y = Y.reshape(Y_hat.shape)\n",
    "\n",
    "    dA_prev = - (np.divide(Y, Y_hat) - np.divide(1 - Y, 1 - Y_hat));\n",
    "\n",
    "    for layer_idx_prev, layer in reversed(list(enumerate(nn_architecture))):\n",
    "        layer_idx_curr = layer_idx_prev + 1\n",
    "        activ_function_curr = layer[\"activation\"]\n",
    "\n",
    "        dA_curr = dA_prev\n",
    "\n",
    "        A_prev = memory[\"A\" + str(layer_idx_prev)]\n",
    "        Z_curr = memory[\"Z\" + str(layer_idx_curr)]\n",
    "        W_curr = params_values[\"W\" + str(layer_idx_curr)]\n",
    "        b_curr = params_values[\"b\" + str(layer_idx_curr)]\n",
    "\n",
    "        dA_prev, dW_curr, db_curr = single_layer_backward_propagation(\n",
    "            dA_curr, W_curr, b_curr, Z_curr, A_prev, activ_function_curr)\n",
    "\n",
    "        grads_values[\"dW\" + str(layer_idx_curr)] = dW_curr\n",
    "        grads_values[\"db\" + str(layer_idx_curr)] = db_curr\n",
    "\n",
    "    return grads_values"
   ],
   "id": "2c8e4eed1846f003",
   "outputs": [],
   "execution_count": 131
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.658802Z",
     "start_time": "2025-11-05T13:38:02.654594Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def update(params_values, grads_values, nn_architecture, learning_rate):\n",
    "    for layer_idx, layer in enumerate(nn_architecture, 1):\n",
    "        params_values[\"W\" + str(layer_idx)] -= learning_rate * grads_values[\"dW\" + str(layer_idx)]\n",
    "        params_values[\"b\" + str(layer_idx)] -= learning_rate * grads_values[\"db\" + str(layer_idx)]\n",
    "\n",
    "    return params_values;"
   ],
   "id": "16320b953a183511",
   "outputs": [],
   "execution_count": 132
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:39:17.682538Z",
     "start_time": "2025-11-05T13:39:17.675416Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def train(X, Y, nn_architecture, epochs, learning_rate, verbose=False, callback=None):\n",
    "    # initiation of neural net parameters\n",
    "    params_values = init_layers(nn_architecture, 2)\n",
    "    # initiation of lists storing the history\n",
    "    # of metrics calculated during the learning process\n",
    "    cost_history = []\n",
    "    accuracy_history = []\n",
    "\n",
    "    # performing calculations for subsequent iterations\n",
    "    for i in range(epochs):\n",
    "        # step forward\n",
    "        Y_hat, cashe = full_forward_propagation(X, params_values, nn_architecture)\n",
    "\n",
    "        # calculating metrics and saving them in history\n",
    "        cost = get_cost_value(Y_hat, Y)\n",
    "        cost_history.append(cost)\n",
    "        accuracy = get_accuracy_value(Y_hat, Y)\n",
    "        accuracy_history.append(accuracy)\n",
    "\n",
    "        # step backward - calculating gradient\n",
    "        grads_values = full_backward_propagation(Y_hat, Y, cashe, params_values, nn_architecture)\n",
    "        # updating model state\n",
    "        params_values = update(params_values, grads_values, nn_architecture, learning_rate)\n",
    "\n",
    "        if(i % 1000 == 0):\n",
    "            if(verbose):\n",
    "                print(\"Iteration: {:05} - cost: {:.5f} - accuracy: {:.5f}\".format(i, cost, accuracy))\n",
    "            if(callback is not None):\n",
    "                callback(i, params_values)\n",
    "\n",
    "    return params_values"
   ],
   "id": "fce33f70bba3898",
   "outputs": [],
   "execution_count": 143
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.698142Z",
     "start_time": "2025-11-05T13:38:02.692258Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "\n",
    "import sklearn.datasets as ds\n",
    "from sklearn.datasets import make_moons\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "sns.set_style(\"whitegrid\")\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "# from keras.utils import np_utils\n",
    "from keras import regularizers\n",
    "\n",
    "from sklearn.metrics import accuracy_score"
   ],
   "id": "cccd73b5018799d4",
   "outputs": [],
   "execution_count": 134
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.713606Z",
     "start_time": "2025-11-05T13:38:02.710429Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# number of samples in the data set\n",
    "N_SAMPLES = 1000\n",
    "# ratio between training and test sets\n",
    "TEST_SIZE = 0.1"
   ],
   "id": "4f66ffa878f01c02",
   "outputs": [],
   "execution_count": 135
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:02.731937Z",
     "start_time": "2025-11-05T13:38:02.725571Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X, y = ds.make_circles(n_samples = N_SAMPLES, noise=0.2, random_state=100)\n",
    "# X, y = make_moons(n_samples = N_SAMPLES, noise=0.2, random_state=100)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=TEST_SIZE, random_state=42)"
   ],
   "id": "bebe0ed00a2d514",
   "outputs": [],
   "execution_count": 136
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:25.210667Z",
     "start_time": "2025-11-05T13:38:02.745281Z"
    }
   },
   "cell_type": "code",
   "source": [
    "params_values = train(np.transpose(X_train), np.transpose(y_train.reshape((y_train.shape[0], 1))), nn_architecture, 30000, 0.01, verbose=True)\n",
    "# params_values\n"
   ],
   "id": "ce04892d496c5147",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 00000 - cost: 0.69318 - accuracy: 0.50444\n",
      "Iteration: 01000 - cost: 0.69315 - accuracy: 0.50444\n",
      "Iteration: 02000 - cost: 0.69312 - accuracy: 0.50444\n",
      "Iteration: 03000 - cost: 0.69310 - accuracy: 0.50444\n",
      "Iteration: 04000 - cost: 0.69308 - accuracy: 0.50444\n",
      "Iteration: 05000 - cost: 0.69306 - accuracy: 0.50444\n",
      "Iteration: 06000 - cost: 0.69304 - accuracy: 0.50444\n",
      "Iteration: 07000 - cost: 0.69301 - accuracy: 0.50444\n",
      "Iteration: 08000 - cost: 0.69298 - accuracy: 0.50444\n",
      "Iteration: 09000 - cost: 0.69296 - accuracy: 0.50444\n",
      "Iteration: 10000 - cost: 0.69292 - accuracy: 0.50444\n",
      "Iteration: 11000 - cost: 0.69288 - accuracy: 0.50444\n",
      "Iteration: 12000 - cost: 0.69284 - accuracy: 0.50444\n",
      "Iteration: 13000 - cost: 0.69278 - accuracy: 0.50444\n",
      "Iteration: 14000 - cost: 0.69272 - accuracy: 0.50444\n",
      "Iteration: 15000 - cost: 0.69265 - accuracy: 0.50444\n",
      "Iteration: 16000 - cost: 0.69256 - accuracy: 0.50444\n",
      "Iteration: 17000 - cost: 0.69244 - accuracy: 0.50444\n",
      "Iteration: 18000 - cost: 0.69229 - accuracy: 0.50778\n",
      "Iteration: 19000 - cost: 0.69210 - accuracy: 0.52778\n",
      "Iteration: 20000 - cost: 0.69184 - accuracy: 0.54111\n",
      "Iteration: 21000 - cost: 0.69148 - accuracy: 0.55778\n",
      "Iteration: 22000 - cost: 0.69097 - accuracy: 0.58333\n",
      "Iteration: 23000 - cost: 0.69021 - accuracy: 0.60222\n",
      "Iteration: 24000 - cost: 0.68900 - accuracy: 0.62222\n",
      "Iteration: 25000 - cost: 0.68693 - accuracy: 0.65111\n",
      "Iteration: 26000 - cost: 0.68299 - accuracy: 0.66889\n",
      "Iteration: 27000 - cost: 0.67457 - accuracy: 0.68222\n",
      "Iteration: 28000 - cost: 0.65530 - accuracy: 0.67333\n",
      "Iteration: 29000 - cost: 0.61861 - accuracy: 0.67111\n"
     ]
    }
   ],
   "execution_count": 137
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:25.280709Z",
     "start_time": "2025-11-05T13:38:25.278689Z"
    }
   },
   "cell_type": "code",
   "source": [
    "Y_test_hat, _ = full_forward_propagation(np.transpose(X_test), params_values, nn_architecture)\n",
    "\n",
    "acc_test = get_accuracy_value(Y_test_hat, np.transpose(y_test.reshape((y_test.shape[0], 1))))\n",
    "print(\"Test set accuracy: {:.2f} - David\".format(acc_test))\n"
   ],
   "id": "26e7a2a8848714d9",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 0.69 - David\n"
     ]
    }
   ],
   "execution_count": 138
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:43:01.544185Z",
     "start_time": "2025-11-05T13:43:01.537741Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# boundary of the graph\n",
    "GRID_X_START = -1.5\n",
    "GRID_X_END = 1.5\n",
    "GRID_Y_START = -1.5\n",
    "GRID_Y_END = 1.5\n",
    "# output directory (the folder must be created on the drive)\n",
    "OUTPUT_DIR = \"./binary_classification_vizualizations/\"\n",
    "os.makedirs(OUTPUT_DIR, exist_ok=True)\n",
    "### Definition of grid boundaries\n",
    "grid = np.mgrid[GRID_X_START:GRID_X_END:100j, GRID_Y_START:GRID_Y_END:100j]\n",
    "grid_2d = grid.reshape(2, -1).T\n",
    "XX, YY = grid"
   ],
   "id": "b070f03d55981894",
   "outputs": [],
   "execution_count": 148
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:25.355642Z",
     "start_time": "2025-11-05T13:38:25.347259Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def make_plot(X, y, plot_name, file_name=None, XX=None, YY=None, preds=None, dark=False):\n",
    "    if (dark):\n",
    "        plt.style.use('dark_background')\n",
    "    else:\n",
    "        sns.set_style(\"whitegrid\")\n",
    "    plt.figure(figsize=(16,12))\n",
    "    axes = plt.gca()\n",
    "    axes.set(xlabel=\"$X_1$\", ylabel=\"$X_2$\")\n",
    "    plt.title(plot_name, fontsize=30)\n",
    "    plt.subplots_adjust(left=0.20)\n",
    "    plt.subplots_adjust(right=0.80)\n",
    "    if(XX is not None and YY is not None and preds is not None):\n",
    "        plt.contourf(XX, YY, preds.reshape(XX.shape), 25, alpha = 1, cmap=cm.Spectral)\n",
    "        plt.contour(XX, YY, preds.reshape(XX.shape), levels=[.5], cmap=\"Greys\", vmin=0, vmax=.6)\n",
    "    plt.scatter(X[:, 0], X[:, 1], c=y.ravel(), s=40, cmap=plt.cm.Spectral, edgecolors='black')\n",
    "    if(file_name):\n",
    "        plt.savefig(file_name)\n",
    "        plt.close()"
   ],
   "id": "553e08ddc23ab78c",
   "outputs": [],
   "execution_count": 140
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:47:50.552445Z",
     "start_time": "2025-11-05T13:47:50.545245Z"
    }
   },
   "cell_type": "code",
   "source": "X.T",
   "id": "b87560199ee27331",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.00304644,  0.27761395,  0.7121587 , ..., -0.13965464,\n",
       "        -0.68842612,  0.59102256],\n",
       "       [ 0.08532924, -1.33189304,  0.49761513, ..., -0.91986245,\n",
       "        -0.38146264,  0.40804492]], shape=(2, 1000))"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 158
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:46:41.287971Z",
     "start_time": "2025-11-05T13:46:41.280330Z"
    }
   },
   "cell_type": "code",
   "source": "y",
   "id": "e0c22b36e47fd9e4",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,\n",
       "       1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n",
       "       1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0,\n",
       "       0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1,\n",
       "       0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0,\n",
       "       1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0,\n",
       "       0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0,\n",
       "       0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n",
       "       0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n",
       "       0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0,\n",
       "       1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,\n",
       "       1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1,\n",
       "       0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1,\n",
       "       0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1,\n",
       "       1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0,\n",
       "       1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1,\n",
       "       1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1,\n",
       "       0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1,\n",
       "       0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0,\n",
       "       1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1,\n",
       "       1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1,\n",
       "       0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0,\n",
       "       1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1,\n",
       "       1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1,\n",
       "       1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1,\n",
       "       1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1,\n",
       "       1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1,\n",
       "       1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1,\n",
       "       0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0,\n",
       "       1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1,\n",
       "       0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0,\n",
       "       0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,\n",
       "       0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,\n",
       "       0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1,\n",
       "       0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0,\n",
       "       0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,\n",
       "       1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0,\n",
       "       1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0,\n",
       "       1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1,\n",
       "       1, 0, 1, 1, 0, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 157
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:38:25.471082Z",
     "start_time": "2025-11-05T13:38:25.370173Z"
    }
   },
   "cell_type": "code",
   "source": " make_plot(X, y, \"Dataset\")",
   "id": "36c83562b7404392",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1600x1200 with 1 Axes>"
      ],
      "image/png": "[encoded data, 413212 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 141
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:39:53.206765Z",
     "start_time": "2025-11-05T13:39:27.866713Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from time import sleep\n",
    "def callback_numpy_plot(index, params):\n",
    "    plot_title = \"NumPy Model - It: {:05}\".format(index)\n",
    "    file_name = \"numpy_model_{:05}.png\".format(index // 50)\n",
    "    file_path = os.path.join(OUTPUT_DIR, file_name)\n",
    "    prediction_probs, _ = full_forward_propagation(np.transpose(grid_2d), params, nn_architecture)\n",
    "    prediction_probs = prediction_probs.reshape(prediction_probs.shape[1], 1)\n",
    "    make_plot(X_test, y_test, plot_title, file_name=file_path, XX=XX, YY=YY, preds=prediction_probs, dark=True)\n",
    "\n",
    "# Training\n",
    "params_values = train(np.transpose(X_train), np.transpose(y_train.reshape((y_train.shape[0], 1))), nn_architecture,\n",
    "                      30000, 0.01, False, callback_numpy_plot)"
   ],
   "id": "b6a4d6a1a1fb289",
   "outputs": [],
   "execution_count": 144
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:43:06.906300Z",
     "start_time": "2025-11-05T13:43:06.780036Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "\n",
    "prediction_probs_numpy, _ = full_forward_propagation(np.transpose(grid_2d), params_values, nn_architecture)\n",
    "prediction_probs_numpy = prediction_probs_numpy.reshape(prediction_probs_numpy.shape[1], 1)\n",
    "\n",
    "make_plot(X_test, y_test, \"NumPy Model\", file_name=None, XX=XX, YY=YY, preds=prediction_probs_numpy)"
   ],
   "id": "6b36e606efa7f99a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1600x1200 with 1 Axes>"
      ],
      "image/png": "[encoded data, 149644 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 149
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "id": "dc0b9266ae37298a",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:45:25.105461Z",
     "start_time": "2025-11-05T13:45:17.001211Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(25, input_dim=2,activation='relu'))\n",
    "model.add(Dense(50, activation='relu'))\n",
    "model.add(Dense(50, activation='relu'))\n",
    "model.add(Dense(25, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"sgd\", metrics=['accuracy'])\n",
    "\n",
    "# Training\n",
    "history = model.fit(X_train, y_train, epochs=200, verbose=0)"
   ],
   "id": "f05ff40ed26e45c2",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/oskar/projects/nn-from-scratch/.venv/lib/python3.13/site-packages/keras/src/layers/core/dense.py:95: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    }
   ],
   "execution_count": 154
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:45:27.083553Z",
     "start_time": "2025-11-05T13:45:26.623760Z"
    }
   },
   "cell_type": "code",
   "source": [
    "Y_test_prob = model.predict(X_test)\n",
    "Y_test_hat = (Y_test_prob > 0.5).astype(int).ravel()\n",
    "acc_test = accuracy_score(y_test, Y_test_hat)\n",
    "print(\"Test set accuracy: {:.2f} - Goliath\".format(acc_test))"
   ],
   "id": "ef52bee9c93081d3",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[1m1/4\u001B[0m \u001B[32m━━━━━\u001B[0m\u001B[37m━━━━━━━━━━━━━━━\u001B[0m \u001B[1m0s\u001B[0m 202ms/stepWARNING:tensorflow:5 out of the last 62926 calls to <function TensorFlowTrainer.make_predict_function.<locals>.one_step_on_data_distributed at 0x7a1f345302c0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "\u001B[1m4/4\u001B[0m \u001B[32m━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[37m\u001B[0m \u001B[1m0s\u001B[0m 78ms/step\n",
      "Test set accuracy: 0.69 - Goliath\n"
     ]
    }
   ],
   "execution_count": 155
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:41:27.539268Z",
     "start_time": "2025-11-05T13:40:32.908771Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def callback_keras_plot(epoch, logs):\n",
    "    plot_title = \"Keras Model - It: {:05}\".format(epoch)\n",
    "    file_name = \"keras_model_{:05}.png\".format(epoch)\n",
    "    file_path = os.path.join(OUTPUT_DIR, file_name)\n",
    "    prediction_probs = model.predict(grid_2d, batch_size=32, verbose=0)\n",
    "    prediction_probs = prediction_probs.reshape(-1)\n",
    "    make_plot(X_test, y_test, plot_title, file_name=file_path, XX=XX, YY=YY, preds=prediction_probs)\n",
    "\n",
    "\n",
    "# Adding callback functions that they will run in every epoch\n",
    "testmodelcb = keras.callbacks.LambdaCallback(on_epoch_end=callback_keras_plot)\n",
    "\n",
    "# Building a model\n",
    "model = Sequential()\n",
    "model.add(Dense(25, input_dim=2, activation='relu'))\n",
    "model.add(Dense(50, activation='relu'))\n",
    "model.add(Dense(50, activation='relu'))\n",
    "model.add(Dense(25, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"sgd\", metrics=['accuracy'])\n",
    "\n",
    "# Training\n",
    "history = model.fit(X_train, y_train, epochs=200, verbose=0, callbacks=[testmodelcb])\n"
   ],
   "id": "6feab7da06e7a828",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/oskar/projects/nn-from-scratch/.venv/lib/python3.13/site-packages/keras/src/layers/core/dense.py:95: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
      "  super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"
     ]
    }
   ],
   "execution_count": 146
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-05T13:43:21.691060Z",
     "start_time": "2025-11-05T13:43:21.443563Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "prediction_probs = model.predict(grid_2d, batch_size=32, verbose=0)\n",
    "prediction_probs = prediction_probs.reshape(-1)\n",
    "make_plot(X_test, y_test, \"Keras Model\", file_name=None, XX=XX, YY=YY, preds=prediction_probs)\n"
   ],
   "id": "38dee4608746a358",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1600x1200 with 1 Axes>"
      ],
      "image/png": "[encoded data, 150240 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 150
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "!nvidia-smi\n",
   "id": "7369490697e1ed40",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import torch\n",
    "print(torch.cuda.is_available())"
   ],
   "id": "9958964dbe0d2732",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "print(torch.cuda.get_device_name(0))\n",
    "print(torch.cuda.current_device())"
   ],
   "id": "f88bc28cff654bde",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "print(tf.config.list_physical_devices('GPU'))",
   "id": "d60dd759b2bf2bf",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import torch, time\n",
    "\n",
    "x_cpu = torch.randn(10000, 10000)\n",
    "start = time.time()\n",
    "y_cpu = x_cpu @ x_cpu\n",
    "print(\"CPU:\", time.time() - start, \"s\")\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    x_gpu = x_cpu.cuda()\n",
    "    torch.cuda.synchronize()\n",
    "    start = time.time()\n",
    "    y_gpu = x_gpu @ x_gpu\n",
    "    torch.cuda.synchronize()\n",
    "    print(\"GPU:\", time.time() - start, \"s\")\n"
   ],
   "id": "b482f6b3594de45e",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
