Htydjtk

I have been trying to develop a neural network to measure the error in a linear series. What I would like the model to do is infer a linear regression line and then measure the mean absolute error around that line.

I have tried a number of neural network model configurations, including recurrent configurations, but the network learns a weak relationship and then overfits. I have also tried L1 and L2 regularization but neither work.

Any thoughts? Thanks!

Below is the code I am using to simulate the data and a fit sample model:

import numpy as np, matplotlib.pyplot as plt



from keras import layers

from keras.models import Sequential

from keras.optimizers import Adam

from keras.backend import clear_session



## Simulate the data:



np.random.seed(20190318)



X = np.array(()).reshape(0, 50)



Y = np.array(()).reshape(0, 1)



for _ in range(500):



    i = np.random.randint(100, 110) # Intercept.



    s = np.random.randint(1, 10) # Slope.



    e = np.random.normal(0, 25, 50) # Error.



    X_i = np.round(i + (s * np.arange(0, 50)) + e, 2).reshape(1, 50)



    Y_i = np.sum(np.abs(e)).reshape(1, 1)



    X = np.concatenate((X, X_i), axis = 0)



    Y = np.concatenate((Y, Y_i), axis = 0)



## Training and validation data:



split = 400



X_train = X[:split, :-1]

Y_train = Y[:split, -1:]



X_valid = X[split:, :-1]

Y_valid = Y[split:, -1:]



print(X_train.shape)

print(Y_train.shape)

print()

print(X_valid.shape)

print(Y_valid.shape)



## Graph of one of the series:



plt.plot(X_train[0])



## Sample model (takes about a minute to run):



clear_session()



model_fnn = Sequential()

model_fnn.add(layers.Dense(512, activation = 'relu', input_shape = (X_train.shape[1],)))

model_fnn.add(layers.Dense(512, activation = 'relu'))

model_fnn.add(layers.Dense(  1, activation = None))



# Compile model.



model_fnn.compile(optimizer = Adam(lr = 1e-4), loss = 'mse')



# Fit model.



history_fnn = model_fnn.fit(X_train, Y_train, batch_size = 32, epochs = 100, verbose = False,

    validation_data = (X_valid, Y_valid))



## Sample model learning curves:



loss_fnn = history_fnn.history['loss']

val_loss_fnn = history_fnn.history['val_loss']

epochs_fnn = range(1, len(loss_fnn) + 1)



plt.plot(epochs_fnn, loss_fnn, 'black', label = 'Training Loss')

plt.plot(epochs_fnn, val_loss_fnn, 'red', label = 'Validation Loss')

plt.title('FNN: Training and Validation Loss')

plt.legend()

plt.show()

UPDATE:

## Predict.



Y_train_fnn = model_fnn.predict(X_train)

Y_valid_fnn = model_fnn.predict(X_valid)



## Evaluate predictions with training data.



plt.scatter(Y_train, Y_train_fnn)

plt.xlabel("Actual")

plt.ylabel("Predicted")



## Evaluate predictions with training data.



plt.scatter(Y_valid, Y_valid_fnn)

plt.xlabel("Actual")

plt.ylabel("Predicted")

This problem is naturally hard. The underlying function that we try to learn is
$$mathbf{X}=i+s+mathbf{e} rightarrow Y=left | mathbf{X} - i - s right |_1 = left | mathbf{e} right |_1=sum_d|e_d|$$

where $i$ and $s$ are unknown random variables. For large $i$ and $s$, $left | mathbf{e} right |_1$ is naturally hard to recover from $mathbf{X}$. I found a working example (training error almost zero) by setting the intercept $i$ and slope $s$ to zero!, drastically shrinking the network size to work better with a small sample size (800), and increased the number of epochs to 800, which was crucial. Also, (true value, error) is plotted at the end for training data.

You can work up from this point to see the effect of increasing $i$ and $s$ on performance.

import numpy as np, matplotlib.pyplot as plt



from keras import layers

from keras.models import Sequential

from keras.optimizers import Adam

from keras.backend import clear_session



## Simulate the data:



np.random.seed(20190318)



dimension = 50



X = np.array(()).reshape(0, dimension)



Y = np.array(()).reshape(0, 1)



for _ in range(1000):



    i = 0   # np.random.randint(100, 110) # Intercept.



    s = 0   # np.random.randint(1, 10) # Slope.



    e = np.random.normal(0, 25, dimension) # Error.



    X_i = np.round(i + (s * np.arange(0, dimension)) + e, 2).reshape(1, dimension)



    Y_i = np.sum(np.abs(e)).reshape(1, 1)



    X = np.concatenate((X, X_i), axis = 0)



    Y = np.concatenate((Y, Y_i), axis = 0)



## Training and validation data:



split = 800



X_train = X[:split, :-1]

Y_train = Y[:split, -1:]



X_valid = X[split:, :-1]

Y_valid = Y[split:, -1:]



print(X_train.shape)

print(Y_train.shape)

print()

print(X_valid.shape)

print(Y_valid.shape)



## Graph of one of the series:



plt.plot(X_train[0])



## Sample model (takes about a minute to run):



clear_session()



model_fnn = Sequential()

model_fnn.add(layers.Dense(dimension, activation = 'relu', input_shape = (X_train.shape[1],)))

model_fnn.add(layers.Dense(dimension, activation = 'relu'))

model_fnn.add(layers.Dense(  1, activation = 'linear'))



# Compile model.



model_fnn.compile(optimizer = Adam(lr = 1e-4), loss = 'mse')



# Fit model.



history_fnn = model_fnn.fit(X_train, Y_train, batch_size = 32, epochs = 800, verbose = True,

    validation_data = (X_valid, Y_valid))



# Sample model learning curves:



loss_fnn = history_fnn.history['loss']

val_loss_fnn = history_fnn.history['val_loss']

epochs_fnn = range(1, len(loss_fnn) + 1)



plt.figure(1)

offset = 5

plt.plot(epochs_fnn[offset:], loss_fnn[offset:], 'black', label = 'Training Loss')

plt.plot(epochs_fnn[offset:], val_loss_fnn[offset:], 'red', label = 'Validation Loss')

plt.title('FNN: Training and Validation Loss')

plt.legend()



## Predict.



plt.figure(2)



Y_train_fnn = model_fnn.predict(X_train)



## Evaluate predictions with training data.

sorted_index = Y_train.argsort(axis=0)

Y_train_sorted = np.reshape(Y_train[sorted_index], (-1, 1))

Y_train_fnn_sorted = np.reshape(Y_train_fnn[sorted_index], (-1, 1))

plt.plot(Y_train_sorted, Y_train_sorted - Y_train_fnn_sorted)

plt.xlabel("Y(true) train")

plt.ylabel("Y(true) - Y(predicted) train")

plt.show()