models/failure_prediction/jnotebooks/LSTM_Attention.ipynb

   1 {
   2  "cells": [
   3   {
   4    "cell_type": "markdown",
   5    "metadata": {
   6     "id": "oQvr9zfcPRi-"
   7    },
   8    "source": [
   9     "Contributors: Rohit Singh Rathaur, Girish L.\n",
  10     "\n",
  11     "Copyright 2021 [Rohit Singh Rathaur, BIT Mesra and Girish L., CIT GUBBI, Karnataka]\n",
  12     "\n",
  13     "Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at\n",
  14     "\n",
  15     "http://www.apache.org/licenses/LICENSE-2.0\n",
  16     "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
  17    ]
  18   },
  19   {
  20    "cell_type": "code",
  21    "execution_count": null,
  22    "metadata": {
  23     "colab": {
  24      "base_uri": "https://localhost:8080/"
  25     },
  26     "id": "YQ6lT1e2hrx4",
  27     "outputId": "cb071d50-8fcb-426d-8fa9-6f7033f783d1"
  28    },
  29    "outputs": [],
  30    "source": [
  31     "from google.colab import drive\n",
  32     "drive.mount('/content/drive')"
  33    ]
  34   },
  35   {
  36    "cell_type": "code",
  37    "execution_count": null,
  38    "metadata": {
  39     "id": "tLhroy5BnMnC"
  40    },
  41    "outputs": [],
  42    "source": [
  43     "# Importing libraries\n",
  44     "import tensorflow as tf\n",
  45     "import matplotlib.pyplot as plt\n",
  46     "import matplotlib as mpl\n",
  47     "import pandas as pd\n",
  48     "import numpy as np\n",
  49     "import os"
  50    ]
  51   },
  52   {
  53    "cell_type": "code",
  54    "execution_count": null,
  55    "metadata": {
  56     "colab": {
  57      "base_uri": "https://localhost:8080/",
  58      "height": 521
  59     },
  60     "id": "2-UpMVsSnfCI",
  61     "outputId": "120b11e9-5c7d-4332-d133-9636a8d002aa"
  62    },
  63    "outputs": [],
  64    "source": [
  65     "df_Ellis  = pd.read_csv(\"/content/drive/MyDrive/LFN Anuket/Analysis/data/Final/Ellis_FinalTwoConditionwithOR.csv\")\n",
  66     "df_Ellis"
  67    ]
  68   },
  69   {
  70    "cell_type": "code",
  71    "execution_count": null,
  72    "metadata": {
  73     "colab": {
  74      "base_uri": "https://localhost:8080/",
  75      "height": 293
  76     },
  77     "id": "92xBt43BnjAo",
  78     "outputId": "2104e011-6066-4e86-e6f8-515dfb9ea715"
  79    },
  80    "outputs": [],
  81    "source": [
  82     "df_Ellis.plot()"
  83    ]
  84   },
  85   {
  86    "cell_type": "code",
  87    "execution_count": null,
  88    "metadata": {
  89     "colab": {
  90      "base_uri": "https://localhost:8080/",
  91      "height": 879
  92     },
  93     "id": "RSo-aa-SIoBR",
  94     "outputId": "ea97c36d-cb27-4fb3-d9bd-e8f1776b098d"
  95    },
  96    "outputs": [],
  97    "source": [
  98     "# we show here the hist\n",
  99     "df_Ellis.hist(bins=100,figsize=(20,15))\n",
 100     "#save_fig(\"attribute_histogram_plots\")\n",
 101     "plt.show()"
 102    ]
 103   },
 104   {
 105    "cell_type": "code",
 106    "execution_count": null,
 107    "metadata": {
 108     "colab": {
 109      "base_uri": "https://localhost:8080/",
 110      "height": 634
 111     },
 112     "id": "gggaMJ_2LtFs",
 113     "outputId": "9e21fcc0-a838-4449-ef31-8e7d1df4d060"
 114    },
 115    "outputs": [],
 116    "source": [
 117     "cpu_system_perc = df_Ellis[['ellis-cpu.system_perc']] \n",
 118     "cpu_system_perc.rolling(12).mean().plot(figsize=(20,10), linewidth=5, fontsize=20) \n",
 119     "plt.xlabel('Timestamp', fontsize=30);"
 120    ]
 121   },
 122   {
 123    "cell_type": "code",
 124    "execution_count": null,
 125    "metadata": {
 126     "colab": {
 127      "base_uri": "https://localhost:8080/",
 128      "height": 634
 129     },
 130     "id": "R_ctvXcQL1Xf",
 131     "outputId": "1a35f11e-eb94-4404-bd86-6f6fc2765ba2"
 132    },
 133    "outputs": [],
 134    "source": [
 135     "load_avg_1_min = df_Ellis[['ellis-load.avg_1_min']] \n",
 136     "load_avg_1_min.rolling(12).mean().plot(figsize=(20,10), linewidth=5, fontsize=20) \n",
 137     "plt.xlabel('Timestamp', fontsize=30);"
 138    ]
 139   },
 140   {
 141    "cell_type": "code",
 142    "execution_count": null,
 143    "metadata": {
 144     "colab": {
 145      "base_uri": "https://localhost:8080/",
 146      "height": 634
 147     },
 148     "id": "Gkd5ecCmL6Bw",
 149     "outputId": "2396e496-512e-45de-b804-acfb32e79b41"
 150    },
 151    "outputs": [],
 152    "source": [
 153     "cpu_wait_perc = df_Ellis[['ellis-cpu.wait_perc']] \n",
 154     "cpu_wait_perc.rolling(12).mean().plot(figsize=(20,10), linewidth=5, fontsize=20) \n",
 155     "plt.xlabel('Year', fontsize=30);"
 156    ]
 157   },
 158   {
 159    "cell_type": "code",
 160    "execution_count": null,
 161    "metadata": {
 162     "colab": {
 163      "base_uri": "https://localhost:8080/",
 164      "height": 624
 165     },
 166     "id": "EycZrQU0MBSX",
 167     "outputId": "dcf3d389-1ac0-4005-ae48-53e0f96548b1"
 168    },
 169    "outputs": [],
 170    "source": [
 171     "df_dg = pd.concat([cpu_system_perc.rolling(12).mean(), load_avg_1_min.rolling(12).mean(),cpu_wait_perc.rolling(12).mean()], axis=1) \n",
 172     "df_dg.plot(figsize=(20,10), linewidth=5, fontsize=20) \n",
 173     "plt.xlabel('Year', fontsize=20); "
 174    ]
 175   },
 176   {
 177    "cell_type": "code",
 178    "execution_count": null,
 179    "metadata": {
 180     "id": "YoQA_MIBMknS"
 181    },
 182    "outputs": [],
 183    "source": []
 184   },
 185   {
 186    "cell_type": "code",
 187    "execution_count": null,
 188    "metadata": {
 189     "colab": {
 190      "base_uri": "https://localhost:8080/",
 191      "height": 710
 192     },
 193     "id": "Pi8UMMitMa3Q",
 194     "outputId": "57b8d44c-ad3d-45e0-82da-1787d6d47d55"
 195    },
 196    "outputs": [],
 197    "source": [
 198     "# we establish the corrmartrice\n",
 199     "import seaborn as sns\n",
 200     "color = sns.color_palette()\n",
 201     "sns.set_style('darkgrid')\n",
 202     "\n",
 203     "correaltionMatrice = df_Ellis.corr()\n",
 204     "f, ax = plt.subplots(figsize=(20, 10))\n",
 205     "sns.heatmap(correaltionMatrice, cbar=True, vmin=0, vmax=1, square=True, annot=True);\n",
 206     "plt.show()"
 207    ]
 208   },
 209   {
 210    "cell_type": "code",
 211    "execution_count": null,
 212    "metadata": {
 213     "colab": {
 214      "base_uri": "https://localhost:8080/"
 215     },
 216     "id": "rkYwyKtXMvpy",
 217     "outputId": "f5386858-2a38-4b20-9be9-f841ee5af0d6"
 218    },
 219    "outputs": [],
 220    "source": [
 221     "df_Ellis.corrwith(df_Ellis['ellis-load.avg_1_min'])"
 222    ]
 223   },
 224   {
 225    "cell_type": "code",
 226    "execution_count": null,
 227    "metadata": {
 228     "colab": {
 229      "base_uri": "https://localhost:8080/",
 230      "height": 235
 231     },
 232     "id": "5oQK-ddinvCM",
 233     "outputId": "a611220f-df65-4d99-9c8f-dc0dd715f93f"
 234    },
 235    "outputs": [],
 236    "source": [
 237     "## ## using multivariate feature \n",
 238     "\n",
 239     "features_3 = ['ellis-cpu.wait_perc', 'ellis-load.avg_1_min', 'ellis-net.in_bytes_sec', 'Label']\n",
 240     "\n",
 241     "features = df_Ellis[features_3]\n",
 242     "features.index = df_Ellis['Timestamp']\n",
 243     "features.head()"
 244    ]
 245   },
 246   {
 247    "cell_type": "code",
 248    "execution_count": null,
 249    "metadata": {
 250     "colab": {
 251      "base_uri": "https://localhost:8080/",
 252      "height": 386
 253     },
 254     "id": "qbqn755fo81g",
 255     "outputId": "8750fe62-a1e7-40ca-a3c1-4285546f97ff"
 256    },
 257    "outputs": [],
 258    "source": [
 259     "features.plot(subplots=True)"
 260    ]
 261   },
 262   {
 263    "cell_type": "code",
 264    "execution_count": null,
 265    "metadata": {
 266     "id": "jJQD1x9psWCH"
 267    },
 268    "outputs": [],
 269    "source": [
 270     "features = features.values"
 271    ]
 272   },
 273   {
 274    "cell_type": "code",
 275    "execution_count": null,
 276    "metadata": {
 277     "colab": {
 278      "base_uri": "https://localhost:8080/"
 279     },
 280     "id": "xf8WCiykpUzN",
 281     "outputId": "b43752ab-2ff0-4e21-a4bc-633cd0aff7dd"
 282    },
 283    "outputs": [],
 284    "source": [
 285     "### standardize data\n",
 286     "train_split = 141600\n",
 287     "tf.random.set_seed(13)\n",
 288     "\n",
 289     "### standardize data\n",
 290     "features_mean = features[:train_split].mean()\n",
 291     "features_std = features[:train_split].std()\n",
 292     "features  = (features - features_mean)/ features_std\n",
 293     "\n",
 294     "print(type(features))\n",
 295     "print(features.shape)\n"
 296    ]
 297   },
 298   {
 299    "cell_type": "code",
 300    "execution_count": null,
 301    "metadata": {
 302     "id": "1a0hNDmppnLB"
 303    },
 304    "outputs": [],
 305    "source": [
 306     "### create mutlivariate data\n",
 307     "\n",
 308     "def mutlivariate_data(features , target , start_idx , end_idx , history_size , target_size,\n",
 309     "                      step ,  single_step = False):\n",
 310     "  data = []\n",
 311     "  labels = []\n",
 312     "  start_idx = start_idx + history_size\n",
 313     "  if end_idx is None:\n",
 314     "    end_idx = len(features)- target_size\n",
 315     "  for i in range(start_idx , end_idx ):\n",
 316     "    idxs = range(i-history_size, i, step) ### using step\n",
 317     "    data.append(features[idxs])\n",
 318     "    if single_step:\n",
 319     "      labels.append(target[i+target_size])\n",
 320     "    else:\n",
 321     "      labels.append(target[i:i+target_size])\n",
 322     "\n",
 323     "  return np.array(data) , np.array(labels)"
 324    ]
 325   },
 326   {
 327    "cell_type": "code",
 328    "execution_count": null,
 329    "metadata": {
 330     "colab": {
 331      "base_uri": "https://localhost:8080/"
 332     },
 333     "id": "Z0CivgkitfgE",
 334     "outputId": "614149cc-6746-4d02-a608-2957cef79338"
 335    },
 336    "outputs": [],
 337    "source": [
 338     "### generate multivariate data\n",
 339     "\n",
 340     "history = 720\n",
 341     "future_target = 72\n",
 342     "STEP = 6\n",
 343     "\n",
 344     "x_train_ss , y_train_ss = mutlivariate_data(features , features[:, 1], 0, train_split, history,\n",
 345     "                                            future_target, STEP , single_step = True)\n",
 346     "\n",
 347     "x_val_ss , y_val_ss = mutlivariate_data(features , features[:,1] , train_split , None , history ,\n",
 348     "                                        future_target, STEP, single_step = True)\n",
 349     "\n",
 350     "print(x_train_ss.shape , y_train_ss.shape)\n",
 351     "print(x_val_ss.shape , y_val_ss.shape)"
 352    ]
 353   },
 354   {
 355    "cell_type": "code",
 356    "execution_count": null,
 357    "metadata": {
 358     "colab": {
 359      "base_uri": "https://localhost:8080/"
 360     },
 361     "id": "VBdr2epGu3aq",
 362     "outputId": "f9dc4c4f-ccfa-4379-add5-cd732dc8a310"
 363    },
 364    "outputs": [],
 365    "source": [
 366     "## tensorflow dataset\n",
 367     "batch_size = 256\n",
 368     "buffer_size = 10000\n",
 369     "\n",
 370     "train_ss = tf.data.Dataset.from_tensor_slices((x_train_ss, y_train_ss))\n",
 371     "train_ss = train_ss.cache().shuffle(buffer_size).batch(batch_size).repeat()\n",
 372     "\n",
 373     "val_ss = tf.data.Dataset.from_tensor_slices((x_val_ss, y_val_ss))\n",
 374     "val_ss = val_ss.cache().shuffle(buffer_size).batch(batch_size).repeat()\n",
 375     "\n",
 376     "print(train_ss)\n",
 377     "print(val_ss)"
 378    ]
 379   },
 380   {
 381    "cell_type": "code",
 382    "execution_count": null,
 383    "metadata": {
 384     "id": "9eQpwUyGglu_"
 385    },
 386    "outputs": [],
 387    "source": [
 388     "def root_mean_squared_error(y_true, y_pred):\n",
 389     "        return K.sqrt(K.mean(K.square(y_pred - y_true))) "
 390    ]
 391   },
 392   {
 393    "cell_type": "code",
 394    "execution_count": null,
 395    "metadata": {
 396     "id": "81S0BbNa7Rce"
 397    },
 398    "outputs": [],
 399    "source": [
 400     "import tensorflow as tf\n",
 401     "from keras.layers import Activation, Dense, Dropout,Permute,Reshape,Lambda,dot,concatenate\n",
 402     "\n",
 403     "INPUT_DIM = 100\n",
 404     "TIME_STEPS = 20\n",
 405     "# if True, the attention vector is shared across the input_dimensions where the attention is applied.\n",
 406     "SINGLE_ATTENTION_VECTOR = True\n",
 407     "APPLY_ATTENTION_BEFORE_LSTM = False\n",
 408     "\n",
 409     "ATTENTION_SIZE = 128\n",
 410     "\n",
 411     "def attention_3d_block(hidden_states):\n",
 412     "    # hidden_states.shape = (batch_size, time_steps, hidden_size)\n",
 413     "    hidden_size = int(hidden_states.shape[2])\n",
 414     "    # _t stands for transpose\n",
 415     "    hidden_states_t =  Permute((2, 1), name='attention_input_t')(hidden_states)\n",
 416     "    # hidden_states_t.shape = (batch_size, hidden_size, time_steps)\n",
 417     "    # this line is not useful. It's just to know which dimension is what.\n",
 418     "    hidden_states_t = Reshape((hidden_size, TIME_STEPS), name='attention_input_reshape')(hidden_states_t)\n",
 419     "    # Inside dense layer\n",
 420     "    # a (batch_size, hidden_size, time_steps) dot W (time_steps, time_steps) => (batch_size, hidden_size, time_steps)\n",
 421     "    # W is the trainable weight matrix of attention\n",
 422     "    # Luong's multiplicative style score\n",
 423     "    score_first_part = Dense(TIME_STEPS, use_bias=False, name='attention_score_vec')(hidden_states_t)\n",
 424     "    score_first_part_t = Permute((2, 1), name='attention_score_vec_t')(score_first_part)\n",
 425     "    #            score_first_part_t         dot        last_hidden_state     => attention_weights\n",
 426     "    # (batch_size, time_steps, hidden_size) dot (batch_size, hidden_size, 1) => (batch_size, time_steps, 1)\n",
 427     "    h_t = Lambda(lambda x: x[:, :, -1], output_shape=(hidden_size, 1), name='last_hidden_state')(hidden_states_t)\n",
 428     "    score = dot([score_first_part_t, h_t], [2, 1], name='attention_score')\n",
 429     "    attention_weights = Activation('softmax', name='attention_weight')(score)\n",
 430     "    # if SINGLE_ATTENTION_VECTOR:\n",
 431     "    #     a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)\n",
 432     "    #     a = RepeatVector(hidden_size)(a)\n",
 433     "    # (batch_size, hidden_size, time_steps) dot (batch_size, time_steps, 1) => (batch_size, hidden_size, 1)\n",
 434     "    context_vector = dot([hidden_states_t, attention_weights], [2, 1], name='context_vector')\n",
 435     "    context_vector = Reshape((hidden_size,))(context_vector)\n",
 436     "    h_t = Reshape((hidden_size,))(h_t)\n",
 437     "    pre_activation = concatenate([context_vector, h_t], name='attention_output')\n",
 438     "    attention_vector = Dense(ATTENTION_SIZE, use_bias=False, activation='tanh', name='attention_vector')(pre_activation)\n",
 439     "    return attention_vector"
 440    ]
 441   },
 442   {
 443    "cell_type": "code",
 444    "execution_count": null,
 445    "metadata": {
 446     "colab": {
 447      "base_uri": "https://localhost:8080/"
 448     },
 449     "id": "1cKtTAzqyiyL",
 450     "outputId": "e1863cec-a19f-4419-fe8c-527e549bccdf"
 451    },
 452    "outputs": [],
 453    "source": [
 454     "from keras.layers import Activation, Dense, Dropout\n",
 455     "from keras.layers import Input, LSTM, merge ,Conv1D,Dropout,Bidirectional,Multiply,Flatten\n",
 456     "from keras.models import Model\n",
 457     "from keras.utils.vis_utils import plot_model\n",
 458     "### Modelling using LSTM\n",
 459     "steps = 50\n",
 460     "TIME_STEPS=120\n",
 461     "INPUT_DIMS=4\n",
 462     "\n",
 463     "EPOCHS =20\n",
 464     "\n",
 465     "single_step_model = tf.keras.models.Sequential()\n",
 466     "\n",
 467     "inputs = Input(shape=(120, 4))\n",
 468     "lstm_units = 32\n",
 469     "lstm_out = LSTM(lstm_units, return_sequences=True)(inputs)\n",
 470     "lstm_out=tf.keras.layers.Dropout(0.6)(lstm_out)\n",
 471     "attention_mul = attention_3d_block(lstm_out)\n",
 472     "#attention_mul = Flatten()(attention_mul)\n",
 473     "attention_mul=tf.keras.layers.Dropout(0.6)(attention_mul)\n",
 474     "output = Dense(1, activation='sigmoid')(attention_mul)\n",
 475     "\n",
 476     "single_step_model = Model(inputs, output)\n",
 477     "\n",
 478     "  \n",
 479     "single_step_model.summary()\n",
 480     "\n",
 481     "plot_model(single_step_model, to_file='/content/drive/MyDrive/LFN Anuket/Analysis/data/Final/LSTM-Attention-B.png', show_shapes=True, show_layer_names=True)\n",
 482     "\n",
 483     "single_step_model.compile(optimizer = tf.keras.optimizers.Adam(), loss = 'mae',metrics=[tf.keras.metrics.RootMeanSquaredError(name='rmse')])\n",
 484     "#single_step_model.compile(loss='mse', optimizer='rmsprop')\n",
 485     "\n",
 486     " "
 487    ]
 488   },
 489   {
 490    "cell_type": "code",
 491    "execution_count": null,
 492    "metadata": {
 493     "colab": {
 494      "base_uri": "https://localhost:8080/"
 495     },
 496     "id": "94000u5xIjWD",
 497     "outputId": "c3f7f857-fbb7-4fc5-b2bf-d860a828d0a9"
 498    },
 499    "outputs": [],
 500    "source": [
 501     "single_step_model_history = single_step_model.fit(train_ss, epochs = EPOCHS , \n",
 502     "                                                  steps_per_epoch =steps, validation_data = val_ss,\n",
 503     "                                                  validation_steps = 50)\n"
 504    ]
 505   },
 506   {
 507    "cell_type": "code",
 508    "execution_count": null,
 509    "metadata": {
 510     "colab": {
 511      "base_uri": "https://localhost:8080/",
 512      "height": 281
 513     },
 514     "id": "Pgev0dgzzBVx",
 515     "outputId": "a97e1688-da30-4042-c349-fa8971512907"
 516    },
 517    "outputs": [],
 518    "source": [
 519     "## plot train test loss \n",
 520     "\n",
 521     "def plot_loss(history , title):\n",
 522     "  loss = history.history['loss']\n",
 523     "  val_loss = history.history['val_loss']\n",
 524     "\n",
 525     "  epochs = range(len(loss))\n",
 526     "  plt.figure()\n",
 527     "  plt.plot(epochs, loss , 'b' , label = 'Train Loss')\n",
 528     "  plt.plot(epochs, val_loss , 'r' , label = 'Validation Loss')\n",
 529     "  plt.title(title)\n",
 530     "  plt.legend()\n",
 531     "  plt.grid()\n",
 532     "  plt.show()\n",
 533     "\n",
 534     "plot_loss(single_step_model_history , 'Single Step Training and validation loss')"
 535    ]
 536   },
 537   {
 538    "cell_type": "code",
 539    "execution_count": null,
 540    "metadata": {
 541     "colab": {
 542      "base_uri": "https://localhost:8080/",
 543      "height": 281
 544     },
 545     "id": "EnYf6j4okEoC",
 546     "outputId": "be978504-7501-42fe-c192-4c6b023e992a"
 547    },
 548    "outputs": [],
 549    "source": [
 550     "## plot train test loss \n",
 551     "\n",
 552     "def plot_loss(history , title):\n",
 553     "  loss = history.history['rmse']\n",
 554     "  val_loss = history.history['val_rmse']\n",
 555     "\n",
 556     "  epochs = range(len(loss))\n",
 557     "  plt.figure()\n",
 558     "  plt.plot(epochs, loss , 'b' , label = 'Train RMSE')\n",
 559     "  plt.plot(epochs, val_loss , 'r' , label = 'Validation RMSE')\n",
 560     "  plt.title(title)\n",
 561     "  plt.legend()\n",
 562     "  plt.grid()\n",
 563     "  plt.show()\n",
 564     "\n",
 565     "plot_loss(single_step_model_history , 'Single Step Training and validation loss')"
 566    ]
 567   },
 568   {
 569    "cell_type": "code",
 570    "execution_count": null,
 571    "metadata": {
 572     "id": "WMegV8mNAwe_"
 573    },
 574    "outputs": [],
 575    "source": [
 576     "### fucntion to create time steps\n",
 577     "def create_time_steps(length):\n",
 578     "  return list(range(-length,0))\n",
 579     "\n",
 580     "### function to plot time series data\n",
 581     "\n",
 582     "def plot_time_series(plot_data, delta , title):\n",
 583     "  labels = [\"History\" , 'True Future' , 'Model Predcited']\n",
 584     "  marker = ['.-' , 'rx' , 'go']\n",
 585     "  time_steps = create_time_steps(plot_data[0].shape[0])\n",
 586     "\n",
 587     "  if delta:\n",
 588     "    future = delta\n",
 589     "  else:\n",
 590     "    future = 0\n",
 591     "  plt.title(title)\n",
 592     "  for i , x in enumerate(plot_data):\n",
 593     "    if i :\n",
 594     "      plt.plot(future , plot_data[i] , marker[i], markersize = 10 , label = labels[i])\n",
 595     "    else:\n",
 596     "      plt.plot(time_steps, plot_data[i].flatten(), marker[i], label = labels[i])\n",
 597     "  plt.legend()\n",
 598     "  plt.xlim([time_steps[0], (future+5) *2])\n",
 599     "\n",
 600     "  plt.xlabel('Time_Step')\n",
 601     "  return plt"
 602    ]
 603   },
 604   {
 605    "cell_type": "code",
 606    "execution_count": null,
 607    "metadata": {
 608     "id": "q99i2c-9XKF3"
 609    },
 610    "outputs": [],
 611    "source": [
 612     "### Moving window average\n",
 613     "\n",
 614     "def MWA(history):\n",
 615     "  return np.mean(history)"
 616    ]
 617   },
 618   {
 619    "cell_type": "code",
 620    "execution_count": null,
 621    "metadata": {
 622     "colab": {
 623      "base_uri": "https://localhost:8080/",
 624      "height": 1000
 625     },
 626     "id": "xFJT1rZDAUVL",
 627     "outputId": "56ae677f-dadd-434f-f31b-d7cc9ee0eb89"
 628    },
 629    "outputs": [],
 630    "source": [
 631     "# plot time series and predicted values\n",
 632     "\n",
 633     "for x, y in val_ss.take(5):\n",
 634     "  plot = plot_time_series([x[0][:, 1].numpy(), y[0].numpy(),\n",
 635     "                    single_step_model.predict(x)[0]], 12,\n",
 636     "                   'Single Step Prediction')\n",
 637     "  plot.show()"
 638    ]
 639   },
 640   {
 641    "cell_type": "markdown",
 642    "metadata": {
 643     "id": "_KXWQVmyCSix"
 644    },
 645    "source": [
 646     "# **MultiStep Forcasting**"
 647    ]
 648   },
 649   {
 650    "cell_type": "code",
 651    "execution_count": null,
 652    "metadata": {
 653     "colab": {
 654      "base_uri": "https://localhost:8080/"
 655     },
 656     "id": "Lu7m2Rr4AbMK",
 657     "outputId": "9436a6eb-214e-47a3-d241-3a5bd056c31c"
 658    },
 659    "outputs": [],
 660    "source": [
 661     "future_target = 72 # 72 future values\n",
 662     "x_train_multi, y_train_multi = mutlivariate_data(features, features[:, 1], 0,\n",
 663     "                                                 train_split, history,\n",
 664     "                                                 future_target, STEP)\n",
 665     "x_val_multi, y_val_multi = mutlivariate_data(features, features[:, 1],\n",
 666     "                                             train_split, None, history,\n",
 667     "                                             future_target, STEP)\n",
 668     "\n",
 669     "print(x_train_multi.shape)\n",
 670     "print(y_train_multi.shape)"
 671    ]
 672   },
 673   {
 674    "cell_type": "code",
 675    "execution_count": null,
 676    "metadata": {
 677     "id": "GLRv5D16CrHj"
 678    },
 679    "outputs": [],
 680    "source": [
 681     "#  TF DATASET\n",
 682     "\n",
 683     "train_data_multi = tf.data.Dataset.from_tensor_slices((x_train_multi, y_train_multi))\n",
 684     "train_data_multi = train_data_multi.cache().shuffle(buffer_size).batch(batch_size).repeat()\n",
 685     "\n",
 686     "val_data_multi = tf.data.Dataset.from_tensor_slices((x_val_multi, y_val_multi))\n",
 687     "val_data_multi = val_data_multi.batch(batch_size).repeat()"
 688    ]
 689   },
 690   {
 691    "cell_type": "code",
 692    "execution_count": null,
 693    "metadata": {
 694     "colab": {
 695      "base_uri": "https://localhost:8080/"
 696     },
 697     "id": "fjXexah9C8yg",
 698     "outputId": "185bb543-b7ad-4f25-fd89-75be524af04e"
 699    },
 700    "outputs": [],
 701    "source": [
 702     "print(train_data_multi)\n",
 703     "print(val_data_multi)"
 704    ]
 705   },
 706   {
 707    "cell_type": "code",
 708    "execution_count": null,
 709    "metadata": {
 710     "colab": {
 711      "base_uri": "https://localhost:8080/",
 712      "height": 385
 713     },
 714     "id": "7mtLZ6S-DPU-",
 715     "outputId": "553066ce-28bf-4e95-95c1-f7f34f907647"
 716    },
 717    "outputs": [],
 718    "source": [
 719     "#plotting function\n",
 720     "def multi_step_plot(history, true_future, prediction):\n",
 721     "  plt.figure(figsize=(12, 6))\n",
 722     "  num_in = create_time_steps(len(history))\n",
 723     "  num_out = len(true_future)\n",
 724     "  plt.grid()\n",
 725     "  plt.plot(num_in, np.array(history[:, 1]), label='History')\n",
 726     "  plt.plot(np.arange(num_out)/STEP, np.array(true_future), 'bo',\n",
 727     "           label='True Future')\n",
 728     "  if prediction.any():\n",
 729     "    plt.plot(np.arange(num_out)/STEP, np.array(prediction), 'ro',\n",
 730     "             label='Predicted Future')\n",
 731     "  plt.legend(loc='upper left')\n",
 732     "  plt.show()\n",
 733     "  \n",
 734     "\n",
 735     "\n",
 736     "for x, y in train_data_multi.take(1):\n",
 737     "  multi_step_plot(x[0], y[0], np.array([0]))"
 738    ]
 739   },
 740   {
 741    "cell_type": "code",
 742    "execution_count": null,
 743    "metadata": {
 744     "colab": {
 745      "base_uri": "https://localhost:8080/"
 746     },
 747     "id": "snN_Flr5DWQN",
 748     "outputId": "8c9fb49d-7743-4966-845b-f56a06bbe9e6"
 749    },
 750    "outputs": [],
 751    "source": [
 752     "multi_step_model = tf.keras.models.Sequential()\n",
 753     "\n",
 754     "inputs = Input(shape=(120, 4))\n",
 755     "lstm_units = 32\n",
 756     "lstm_out = LSTM(lstm_units, return_sequences=True)(inputs)\n",
 757     "lstm_out=tf.keras.layers.Dropout(0.6)(lstm_out)\n",
 758     "attention_mul = attention_3d_block(lstm_out)\n",
 759     "#attention_mul = Flatten()(attention_mul)\n",
 760     "attention_mul=tf.keras.layers.Dropout(0.6)(attention_mul)\n",
 761     "output = Dense(1, activation='sigmoid')(attention_mul)\n",
 762     "\n",
 763     "multi_step_model = Model(inputs, output)\n",
 764     "multi_step_model.compile(optimizer=tf.keras.optimizers.RMSprop(clipvalue=1.0), loss='mae',metrics=[tf.keras.metrics.RootMeanSquaredError(name='rmse')])\n",
 765     "\n",
 766     "multi_step_history = multi_step_model.fit(train_data_multi, epochs=EPOCHS,\n",
 767     "                                          steps_per_epoch=steps,\n",
 768     "                                          validation_data=val_data_multi,\n",
 769     "                                          validation_steps=50)\n",
 770     "multi_step_model.summary()\n",
 771     "\n"
 772    ]
 773   },
 774   {
 775    "cell_type": "code",
 776    "execution_count": null,
 777    "metadata": {
 778     "colab": {
 779      "base_uri": "https://localhost:8080/"
 780     },
 781     "id": "Ay5m27doDsTt",
 782     "outputId": "e9e4b8ea-77d6-438f-9698-91b95180e6f2"
 783    },
 784    "outputs": [],
 785    "source": [
 786     "plot_loss(multi_step_history, 'Multi-Step Training and validation loss')"
 787    ]
 788   },
 789   {
 790    "cell_type": "code",
 791    "execution_count": null,
 792    "metadata": {
 793     "colab": {
 794      "base_uri": "https://localhost:8080/"
 795     },
 796     "id": "6ZFP49W4D2wp",
 797     "outputId": "41c0548f-3fd9-41eb-b2c8-045ec24f59c3"
 798    },
 799    "outputs": [],
 800    "source": [
 801     "for x, y in val_data_multi.take(5):\n",
 802     "  multi_step_plot(x[0], y[0], multi_step_model.predict(x)[0])"
 803    ]
 804   },
 805   {
 806    "cell_type": "code",
 807    "execution_count": null,
 808    "metadata": {
 809     "id": "DNKMjVoAVqZP"
 810    },
 811    "outputs": [],
 812    "source": [
 813     "scores = multi_step_model.evaluate(x_train_multi, y_train_multi, verbose=1, batch_size=200)\n",
 814     "print('MAE: {}'.format(scores[1]))"
 815    ]
 816   },
 817   {
 818    "cell_type": "code",
 819    "execution_count": null,
 820    "metadata": {
 821     "id": "YXcsNZ8yu9Ay"
 822    },
 823    "outputs": [],
 824    "source": [
 825     "scores_test = multi_step_model.evaluate(x_val_multi, y_val_multi, verbose=1, batch_size=200)\n",
 826     "print('MAE: {}'.format(scores[1]))\n"
 827    ]
 828   },
 829   {
 830    "cell_type": "code",
 831    "execution_count": null,
 832    "metadata": {
 833     "id": "uCFgbZEOvZ9A"
 834    },
 835    "outputs": [],
 836    "source": [
 837     "y_pred_test = multi_step_model.predict(x_val_multi, verbose=0)\n",
 838     "\n",
 839     "plt.figure(figsize=(10,5))\n",
 840     "plt.plot(y_pred_test)\n",
 841     "plt.plot(y_val_multi)\n",
 842     "plt.ylabel(\"RUL\")\n",
 843     "plt.xlabel(\"Unit Number\")\n",
 844     "plt.legend(loc='upper left')\n",
 845     "plt.show()"
 846    ]
 847   },
 848   {
 849    "cell_type": "code",
 850    "execution_count": null,
 851    "metadata": {
 852     "id": "1Pdu1pZyZExX"
 853    },
 854    "outputs": [],
 855    "source": []
 856   }
 857  ],
 858  "metadata": {
 859   "accelerator": "GPU",
 860   "colab": {
 861    "collapsed_sections": [],
 862    "name": "LSTM_Attention.ipynb",
 863    "provenance": []
 864   },
 865   "kernelspec": {
 866    "display_name": "Python 3 (ipykernel)",
 867    "language": "python",
 868    "name": "python3"
 869   },
 870   "language_info": {
 871    "codemirror_mode": {
 872     "name": "ipython",
 873     "version": 3
 874    },
 875    "file_extension": ".py",
 876    "mimetype": "text/x-python",
 877    "name": "python",
 878    "nbconvert_exporter": "python",
 879    "pygments_lexer": "ipython3",
 880    "version": "3.9.7"
 881   }
 882  },
 883  "nbformat": 4,
 884  "nbformat_minor": 1
 885 }