通过AutoEncoder实现时序数据异常检测¶

# 导入 paddle
import paddle
import paddle.nn.functional as F
print(paddle.__version__)

2.2.2

# 导入其他模块
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

import warnings
warnings.filterwarnings("ignore")

class AutoEncoder(paddle.nn.Layer):
    def __init__(self):
        super(AutoEncoder, self).__init__()
        self.conv0 = paddle.nn.Conv1D(in_channels=1,out_channels=32,kernel_size=7,stride=2)
        self.conv1 = paddle.nn.Conv1D(in_channels=32,out_channels=16,kernel_size=7,stride=2)
        self.convT0 = paddle.nn.Conv1DTranspose(in_channels=16,out_channels=32,kernel_size=7,stride=2)
        self.convT1 = paddle.nn.Conv1DTranspose(in_channels=32,out_channels=1,kernel_size=7,stride=2)

    def forward(self, x):
        x = self.conv0(x)
        x = F.relu(x)
        x = F.dropout(x,0.2)
        x = self.conv1(x)
        x = F.relu(x)
        x = self.convT0(x)
        x = F.relu(x)
        x = F.dropout(x,0.2)
        x = self.convT1(x)
        return x

import tqdm
# 参数设置
epoch_num = 200
batch_size = 128
learning_rate = 0.001

def train():
    print('训练开始')
    # 实例化模型
    model = AutoEncoder()
    # 将模型转换为训练模式
    model.train()
    # 设置优化器，学习率，并且把模型参数给优化器
    opt = paddle.optimizer.Adam(learning_rate=learning_rate,parameters=model.parameters())
    # 设置损失函数
    mse_loss = paddle.nn.MSELoss()
    # 设置数据读取器
    data_reader = paddle.io.DataLoader(train_dataset,
                        batch_size=batch_size,
                        shuffle=True,
                        drop_last=True)
    history_loss = []
    iter_epoch = []
    for epoch in tqdm.tqdm(range(epoch_num)):
        for batch_id, data in enumerate(data_reader()):             
            x = data[0]
            y = data[1]
            out = model(x)
            avg_loss = mse_loss(out,(y[:,:,:-1]))   # 输入的数据经过卷积会丢掉最后一个数据
            avg_loss.backward()
            opt.step()
            opt.clear_grad()
        iter_epoch.append(epoch)
        history_loss.append(avg_loss.numpy()[0])
    # 绘制loss
    plt.plot(iter_epoch,history_loss, label = 'loss')
    plt.legend()
    plt.xlabel('iters')
    plt.ylabel('Loss')
    plt.show()
    # 保存模型参数
    paddle.save(model.state_dict(),'model')

train()

100%|██████████| 200/200 [00:53<00:00,  3.77it/s]

# 计算阀值

param_dict = paddle.load('model')   # 读取保存的参数
model = AutoEncoder()    
model.load_dict(param_dict)    # 加载参数
model.eval()   # 预测
total_loss = []
datas = []
# 预测所有正常时序
mse_loss = paddle.nn.loss.MSELoss()
# 这里设置batch_size为1，单独求得每个数据的loss
data_reader = paddle.io.DataLoader(train_dataset,
                        places=[paddle.CPUPlace()],
                        batch_size=1,
                        shuffle=False,
                        drop_last=False,
                        num_workers=0)
for batch_id, data in enumerate(data_reader()):
    x = data[0]
    y = data[1]
    out = model(x)
    avg_loss = mse_loss(out,(y[:,:,:-1]))
    total_loss.append(avg_loss.numpy()[0])
    datas.append(batch_id)

plt.bar(datas, total_loss)
plt.ylabel("reconstruction loss")
plt.xlabel("data samples")
plt.show()

# 获取重建loss的阀值
threshold = np.max(total_loss)
print("阀值:", threshold)

阀值: 0.029842736

import sys
param_dict = paddle.load('model')   # 读取保存的参数
model = AutoEncoder()    
model.load_dict(param_dict)    # 加载参数
model.eval()   # 预测
data_reader = paddle.io.DataLoader(train_dataset,
                        places=[paddle.CPUPlace()],
                        batch_size=128,
                        shuffle=False,
                        drop_last=False,
                        num_workers=0)
for batch_id, data in enumerate(data_reader()):
    x = data[0]
    out = model(x)
    step = np.arange(287)
    plt.plot(step,x[0,0,:-1].numpy())
    plt.plot(step,out[0,0].numpy())
    plt.show()
    sys.exit

df_test_value = (df_daily_jumpsup - training_mean) / training_std
fig, ax = plt.subplots()
df_test_value.plot(legend=False, ax=ax)
plt.show()
# 这是测试集里面的异常数据，可以看到第11~~12天发生了异常

# 探测异常数据
threshold = 0.033    # 阀值设定，即刚才求得的值
param_dict = paddle.load('model')   # 读取保存的参数
model = AutoEncoder()    
model.load_dict(param_dict)    # 加载参数
model.eval()   # 预测
mse_loss = paddle.nn.loss.MSELoss()

def create_sequences(values, time_steps=288):
    '''
    探测数据预处理
    '''
    output = []
    for i in range(len(values) - time_steps):
        output.append(values[i : (i + time_steps)])
    return np.stack(output)


x_test = create_sequences(df_test_value.values)
x = paddle.to_tensor(x_test).astype('float32')

abnormal_index = [] # 记录检测到异常时数据的索引

for i in range(len(x_test)):
    input_x = paddle.reshape(x[i],(1,1,288))
    out = model(input_x)
    loss = mse_loss(input_x[:,:,:-1],out)
    if loss.numpy()[0]>threshold:
        # 开始检测到异常时序列末端靠近异常点，所以要加上序列长度，得到真实索引位置
        abnormal_index.append(i+288)

# 不再检测异常时序列的前端靠近异常点，所以要减去索引长度得到异常点真实索引，为了结果明显，给异常位置加宽40单位
abnormal_index = abnormal_index[:(-288+40)]
print(len(abnormal_index))
print(abnormal_index)

# 异常检测结果可视化
df_subset = df_daily_jumpsup.iloc[abnormal_index]
fig, ax = plt.subplots()
df_daily_jumpsup.plot(legend=False, ax=ax)
df_subset.plot(legend=False, ax=ax, color="r")
plt.show()