RNN Examples
Description for RNN Examples notebook.
Notebook Contents
This notebook covers:
- Topic 1
- Topic 2
- Topic 3
Use the buttons above to download the notebook or open it in your preferred environment.
📓 Notebook Preview
L3d: Recurrent Neural Networks¶
In [14]:
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from patsy import dmatrices
from collections import OrderedDict
import itertools
from sklearn.model_selection import train_test_split
from sklearn import svm, datasets
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.preprocessing import scale
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM, RNN, GRU, Attention
from tensorflow.keras.layers import Dropout
from tensorflow.keras import layers, Input
#from tensorflow.keras.layers.experimental import preprocessing
from tensorflow.keras import backend as K
from tensorflow.keras.callbacks import EarlyStopping
In [3]:
def reset_weights(model):
"""This function re-initializes model weights at each compile"""
for layer in model.layers:
if isinstance(layer, tf.keras.Model):
reset_weights(layer)
continue
for k, initializer in layer.__dict__.items():
if "initializer" not in k:
continue
# find the corresponding variable
var = getattr(layer, k.replace("_initializer", ""))
var.assign(initializer(var.shape, var.dtype))
In this example, we will use an LSTM to predict monthly average temperatures in Boston, using the 12-month lagged dataset.¶
In [4]:
#Read the data set into a pandas DataFrame
df = pd.read_csv('data/boston_monthly_avg_temps_1978_2019.csv', header=0, infer_datetime_format=True, parse_dates=[0], index_col=[0])
#resample at a month level
#df_resampled = df.resample('M').mean()
In [5]:
df.head()
Out[5]:
In [6]:
split_index = round(len(df)*0.8)
split_date = df.index[split_index]
df_train = df.loc[df.index <= split_date].copy()
df_test = df.loc[df.index > split_date].copy()
In [7]:
X_train = df_train.iloc[:,1:13]
y_train = df_train.iloc[:,0]
X_test = df_test.iloc[:,1:13]
y_test = df_test.iloc[:,0]
In [8]:
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
In [28]:
X_train.head()
Out[28]:
In [32]:
np.asarray(X_train)
Out[32]:
Now we reshape in the inputs and targets as [samples, timesteps, features]. Think of "timesteps" as the number of lag variables.
In [9]:
X_train = np.reshape(np.asarray(X_train), (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(np.asarray(X_test), (X_test.shape[0], X_test.shape[1], 1))
y_train = np.asarray(y_train)
y_test = np.asarray(y_test)
In [36]:
print(X_train.shape)
LSTM implementation¶
In [11]:
reset_weights(model)
In [12]:
K.clear_session()
model = Sequential()
model.add(LSTM(units = 8, input_shape = (X_train.shape[1], X_train.shape[2])))
# model.add(Attention()) # This applies self-attention
# model.add(Flatten()) # Flatten or use GlobalAveragePooling to reduce to 1D
model.add(Dense(1, activation = "linear")) # transform hidden unit activations to single output (monthly average temperature)
model.compile(loss = 'mse', optimizer = 'adam')
In [43]:
#model.fit(X_train, y_train, epochs=50, batch_size=16, verbose=0);
In [13]:
model.summary()
In [63]:
early_stopping = EarlyStopping(monitor='loss', patience=7, min_delta = 0.001, restore_best_weights=True,
mode = "min")
In [86]:
hist = model.fit(X_train, y_train, batch_size = 16,
validation_data = (X_test, y_test),
verbose = 0, epochs = 1000)
In [122]:
def plot_loss(history):
plt.plot(history.history['loss'], label='Train MSE')
plt.plot(history.history['val_loss'], label='Test MSE')
#plt.ylim([0, 40]) # adjust to zoom in/out accordingly
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.grid(True)
In [123]:
plot_loss(hist)
In [91]:
plot_loss(hist) # with ylimit at 40
We see that the optimism begins to increase significantly after 600 epochs (the early stopping monitor should automatically revert the weights of the network to the optimal point)
In [102]:
y_pred = model.predict(X_test)
In [114]:
y_test_pred = pd.DataFrame(df_test.iloc[:,0])
y_test_pred.columns = ['Observed']
y_test_pred['Predicted'] = y_pred
In [116]:
y_test_pred.plot(figsize=(15,5))
Out[116]:
In [120]:
error = y_pred.flatten() - y_test
plt.hist(error, bins=25)
plt.xlabel('Residuals')
plt.ylabel('Count')
Out[120]:
In [121]:
np.mean(error)
Out[121]: