異常數(shù)據(jù)剔除方法有哪些？

outlier detection異常點識別方法1. isolation forest 孤立森林1.1 測試樣本示例

文件 test.pkl

1.2 孤立森林 demo

孤立森林原理

通過對特征進(jìn)行隨機(jī)劃分，建立隨機(jī)森林，將經(jīng)過較少次數(shù)進(jìn)行劃分就可以劃分出來的點認(rèn)為時異常點。

# 參考https://blog.csdn.net/ye1215172385/article/details/79762317 # 官方例子https://scikit-learn.org/stable/auto_examples/ensemble/plot_isolation_forest.html#sphx-glr-auto-examples-ensemble-plot-isolation-forest-pyimport numpy as npimport matplotlib.pyplot as pltfrom sklearn.ensemble import IsolationForest rng = np.random.RandomState(42) # 構(gòu)造訓(xùn)練樣本n_samples = 200 #樣本總數(shù)outliers_fraction = 0.25 #異常樣本比例n_inliers = int((1. - outliers_fraction) * n_samples)n_outliers = int(outliers_fraction * n_samples) X = 0.3 * rng.randn(n_inliers // 2, 2)X_train = np.r_[X + 2, X - 2] #正常樣本X_train = np.r_[X_train, np.random.uniform(low=-6, high=6, size=(n_outliers, 2))] #正常樣本加上異常樣本 # 構(gòu)造模型并擬合clf = IsolationForest(max_samples=n_samples, random_state=rng, contamination=outliers_fraction)clf.fit(X_train)# 計算得分并設(shè)置閾值scores_pred = clf.decision_function(X_train)threshold = np.percentile(scores_pred, 100 * outliers_fraction) #根據(jù)訓(xùn)練樣本中異常樣本比例，得到閾值，用于繪圖 # plot the line, the samples, and the nearest vectors to the planexx, yy = np.meshgrid(np.linspace(-7, 7, 50), np.linspace(-7, 7, 50))Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])Z = Z.reshape(xx.shape) plt.title("IsolationForest")# plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r)plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), threshold, 7), cmap=plt.cm.Blues_r) #繪制異常點區(qū)域，值從最小的到閾值的那部分a = plt.contour(xx, yy, Z, levels=[threshold], linewidths=2, colors='red') #繪制異常點區(qū)域和正常點區(qū)域的邊界plt.contourf(xx, yy, Z, levels=[threshold, Z.max()], colors='palevioletred') #繪制正常點區(qū)域，值從閾值到最大的那部分 b = plt.scatter(X_train[:-n_outliers, 0], X_train[:-n_outliers, 1], c='white', s=20, edgecolor='k')c = plt.scatter(X_train[-n_outliers:, 0], X_train[-n_outliers:, 1], c='black', s=20, edgecolor='k')plt.axis('tight')plt.xlim((-7, 7))plt.ylim((-7, 7))plt.legend([a.collections[0], b, c], ['learned decision function', 'true inliers', 'true outliers'], loc="upper left")plt.show()1.3 自己修改的，X_train能夠改成自己需要的數(shù)據(jù)

此處沒有進(jìn)行標(biāo)準(zhǔn)化，可以先進(jìn)行標(biāo)準(zhǔn)化再在標(biāo)準(zhǔn)化的基礎(chǔ)上去除異常點， from sklearn.preprocessing import StandardScaler

import numpy as npimport matplotlib.pyplot as pltfrom sklearn.ensemble import IsolationForestfrom scipy import stats rng = np.random.RandomState(42) X_train = X_train_demo.valuesoutliers_fraction = 0.1n_samples = 500# 構(gòu)造模型并擬合clf = IsolationForest(max_samples=n_samples, random_state=rng, contamination=outliers_fraction)clf.fit(X_train)# 計算得分并設(shè)置閾值scores_pred = clf.decision_function(X_train)threshold = stats.scoreatpercentile(scores_pred, 100 * outliers_fraction) #根據(jù)訓(xùn)練樣本中異常樣本比例，得到閾值，用于繪圖 # plot the line, the samples, and the nearest vectors to the planerange_max_min0 = (X_train[:,0].max()-X_train[:,0].min())*0.2range_max_min1 = (X_train[:,1].max()-X_train[:,1].min())*0.2xx, yy = np.meshgrid(np.linspace(X_train[:,0].min()-range_max_min0, X_train[:,0].max()+range_max_min0, 500), np.linspace(X_train[:,1].min()-range_max_min1, X_train[:,1].max()+range_max_min1, 500))Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])Z = Z.reshape(xx.shape) plt.title("IsolationForest")# plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r)plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), threshold, 7), cmap=plt.cm.Blues_r) #繪制異常點區(qū)域，值從最小的到閾值的那部分a = plt.contour(xx, yy, Z, levels=[threshold], linewidths=2, colors='red') #繪制異常點區(qū)域和正常點區(qū)域的邊界plt.contourf(xx, yy, Z, levels=[threshold, Z.max()], colors='palevioletred') #繪制正常點區(qū)域，值從閾值到最大的那部分 is_in = clf.predict(X_train)>0b = plt.scatter(X_train[is_in, 0], X_train[is_in, 1], c='white', s=20, edgecolor='k')c = plt.scatter(X_train[~is_in, 0], X_train[~is_in, 1], c='black', s=20, edgecolor='k')plt.axis('tight')plt.xlim((X_train[:,0].min()-range_max_min0, X_train[:,0].max()+range_max_min0,))plt.ylim((X_train[:,1].min()-range_max_min1, X_train[:,1].max()+range_max_min1,))plt.legend([a.collections[0], b, c], ['learned decision function', 'inliers', 'outliers'], loc="upper left")plt.show()1.4 核心代碼1.4.1 示例樣本import numpy as np# 構(gòu)造訓(xùn)練樣本n_samples = 200 #樣本總數(shù)outliers_fraction = 0.25 #異常樣本比例n_inliers = int((1. - outliers_fraction) * n_samples)n_outliers = int(outliers_fraction * n_samples) X = 0.3 * rng.randn(n_inliers // 2, 2)X_train = np.r_[X + 2, X - 2] #正常樣本X_train = np.r_[X_train, np.random.uniform(low=-6, high=6, size=(n_outliers, 2))] #正常樣本加上異常樣本1.4.2 核心代碼實現(xiàn)

clf = IsolationForest(max_samples=0.8, contamination=0.25)

from sklearn.ensemble import IsolationForest# fit the model# max_samples 構(gòu)造一棵樹使用的樣本數(shù)，輸入大于1的整數(shù)則使用該數(shù)字作為構(gòu)造的最大樣本數(shù)目，# 如果數(shù)字屬于(0,1]則使用該比例的數(shù)字作為構(gòu)造iforest# outliers_fraction 多少比例的樣本可以作為異常值clf = IsolationForest(max_samples=0.8, contamination=0.25)clf.fit(X_train)# y_pred_train = clf.predict(X_train)scores_pred = clf.decision_function(X_train)threshold = np.percentile(scores_pred, 100 * outliers_fraction) #根據(jù)訓(xùn)練樣本中異常樣本比例，得到閾值，用于繪圖## 以下兩種方法的篩選結(jié)果，完全相同X_train_predict1 = X_train[clf.predict(X_train)==1]X_train_predict2 = X_train[scores_pred>=threshold,:]# 其中，1的表示非異常點，-1的表示為異常點clf.predict(X_train)array([ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1])2. DBSCAN

DBSCAN(Density-Based Spatial Clustering of Applications with Noise) 原理

以每個點為中心，設(shè)定鄰域及鄰域內(nèi)需要有多少個點，如果樣本點大于指定要求，則認(rèn)為該點與鄰域內(nèi)的點屬于同一類，如果小于指定值，若該點位于其它點的鄰域內(nèi)，則屬于邊界點。

2.1 DBSCAN demo# 參考https://blog.csdn.net/hb707934728/article/details/71515160## 官方示例 https://scikit-learn.org/stable/auto_examples/cluster/plot_dbscan.html#sphx-glr-auto-examples-cluster-plot-dbscan-pyimport numpy as npimport matplotlib.pyplot as pltimport matplotlib.colorsimport sklearn.datasets as dsfrom sklearn.cluster import DBSCANfrom sklearn.preprocessing import StandardScalerdef expand(a, b): d = (b - a) * 0.1 return a-d, b+dif __name__ == "__main__": N = 1000 centers = [[1, 2], [-1, -1], [1, -1], [-1, 1]] #scikit中的make_blobs方法常被用來生成聚類算法的測試數(shù)據(jù)，直觀地說，make_blobs會根據(jù)用戶指定的特征數(shù)量、 # 中心點數(shù)量、范圍等來生成幾類數(shù)據(jù)，這些數(shù)據(jù)可用于測試聚類算法的效果。 #函數(shù)原型：sklearn.datasets.make_blobs(n_samples=100, n_features=2, # centers=3, cluster_std=1.0, center_box=(-10.0, 10.0), shuffle=True, random_state=None)[source] #參數(shù)解析： # n_samples是待生成的樣本的總數(shù)。 # # n_features是每個樣本的特征數(shù)。 # # centers表示類別數(shù)。 # # cluster_std表示每個類別的方差，例如我們希望生成2類數(shù)據(jù)，其中一類比另一類具有更大的方差，可以將cluster_std設(shè)置為[1.0, 3.0]。 data, y = ds.make_blobs(N, n_features=2, centers=centers, cluster_std=[0.5, 0.25, 0.7, 0.5], random_state=0) data = StandardScaler().fit_transform(data) # 數(shù)據(jù)1的參數(shù)：(epsilon, min_sample) params = ((0.2, 5), (0.2, 10), (0.2, 15), (0.3, 5), (0.3, 10), (0.3, 15)) plt.figure(figsize=(12, 8), facecolor='w') plt.suptitle(u'DBSCAN clustering', fontsize=20) for i in range(6): eps, min_samples = params[i] #參數(shù)含義： #eps:半徑，表示以給定點P為中心的圓形鄰域的范圍 #min_samples:以點P為中心的鄰域內(nèi)最少點的數(shù)量 #如果滿足,以點P為中心,半徑為EPS的鄰域內(nèi)點的個數(shù)不少于MinPts,則稱點P為核心點 model = DBSCAN(eps=eps, min_samples=min_samples) model.fit(data) y_hat = model.labels_ core_indices = np.zeros_like(y_hat, dtype=bool) # 生成數(shù)據(jù)類型和數(shù)據(jù)shape和指定array一致的變量 core_indices[model.core_sample_indices_] = True # model.core_sample_indices_ border point位于y_hat中的下標(biāo) # 統(tǒng)計總共有積累，其中為-1的為未分類樣本 y_unique = np.unique(y_hat) n_clusters = y_unique.size - (1 if -1 in y_hat else 0) print (y_unique, '聚類簇的個數(shù)為：', n_clusters) plt.subplot(2, 3, i+1) # 對第幾個圖繪制，2行3列，繪制第i+1個圖 # plt.cm.spectral https://blog.csdn.net/robin_Xu_shuai/article/details/79178857 clrs = plt.cm.Spectral(np.linspace(0, 0.8, y_unique.size)) #用于給畫圖灰色 for k, clr in zip(y_unique, clrs): cur = (y_hat == k) if k == -1: # 用于繪制未分類樣本 plt.scatter(data[cur, 0], data[cur, 1], s=20, c='k') continue # 繪制正常節(jié)點 plt.scatter(data[cur, 0], data[cur, 1], s=30, c=clr, edgecolors='k') # 繪制邊緣點 plt.scatter(data[cur & core_indices][:, 0], data[cur & core_indices][:, 1], s=60, c=clr, marker='o', edgecolors='k') x1_min, x2_min = np.min(data, axis=0) x1_max, x2_max = np.max(data, axis=0) x1_min, x1_max = expand(x1_min, x1_max) x2_min, x2_max = expand(x2_min, x2_max) plt.xlim((x1_min, x1_max)) plt.ylim((x2_min, x2_max)) plt.grid(True) plt.title(u'$epsilon$ = %.1f m = %d clustering num %d'%(eps, min_samples, n_clusters), fontsize=16) plt.tight_layout() plt.subplots_adjust(top=0.9) plt.show()[-1 0 1 2 3] 聚類簇的個數(shù)為： 4[-1 0 1 2 3] 聚類簇的個數(shù)為： 4[-1 0 1 2 3 4] 聚類簇的個數(shù)為： 5[-1 0] 聚類簇的個數(shù)為： 1[-1 0 1] 聚類簇的個數(shù)為： 2[-1 0 1 2 3] 聚類簇的個數(shù)為： 4

2.2 使用自定義測試樣例## 參考https://blog.csdn.net/hb707934728/article/details/71515160## 官方示例 https://scikit-learn.org/stable/auto_examples/cluster/plot_dbscan.html#sphx-glr-auto-examples-cluster-plot-dbscan-pyimport numpy as npimport matplotlib.pyplot as pltimport matplotlib.colorsimport sklearn.datasets as dsfrom sklearn.cluster import DBSCANfrom sklearn.preprocessing import StandardScalerdef expand(a, b): d = (b - a) * 0.1 return a-d, b+dif __name__ == "__main__": N = 1000 data = X_train_demo.values # 數(shù)據(jù)1的參數(shù)：(epsilon, min_sample) params = ((0.2, 5), (0.2, 10), (0.2, 15), (0.2, 20), (0.2, 25), (0.2, 30)) plt.figure(figsize=(12, 8), facecolor='w') plt.suptitle(u'DBSCAN clustering', fontsize=20) for i in range(6): eps, min_samples = params[i] #參數(shù)含義： #eps:半徑，表示以給定點P為中心的圓形鄰域的范圍 #min_samples:以點P為中心的鄰域內(nèi)最少點的數(shù)量 #如果滿足,以點P為中心,半徑為EPS的鄰域內(nèi)點的個數(shù)不少于MinPts,則稱點P為核心點 model = DBSCAN(eps=eps, min_samples=min_samples) model.fit(data) y_hat = model.labels_ core_indices = np.zeros_like(y_hat, dtype=bool) # 生成數(shù)據(jù)類型和數(shù)據(jù)shape和指定array一致的變量 core_indices[model.core_sample_indices_] = True # model.core_sample_indices_ border point位于y_hat中的下標(biāo) # 統(tǒng)計總共有積累，其中為-1的為未分類樣本 y_unique = np.unique(y_hat) n_clusters = y_unique.size - (1 if -1 in y_hat else 0) print (y_unique, '聚類簇的個數(shù)為：', n_clusters) plt.subplot(2, 3, i+1) # 對第幾個圖繪制，2行3列，繪制第i+1個圖 # plt.cm.spectral https://blog.csdn.net/robin_Xu_shuai/article/details/79178857 clrs = plt.cm.Spectral(np.linspace(0, 0.8, y_unique.size)) #用于給畫圖灰色 for k, clr in zip(y_unique, clrs): cur = (y_hat == k) if k == -1: # 用于繪制未分類樣本 plt.scatter(data[cur, 0], data[cur, 1], s=20, c='k') continue # 繪制正常節(jié)點 plt.scatter(data[cur, 0], data[cur, 1], s=30, c=clr, edgecolors='k') # 繪制邊緣點 plt.scatter(data[cur & core_indices][:, 0], data[cur & core_indices][:, 1], s=60, c=clr, marker='o', edgecolors='k') x1_min, x2_min = np.min(data, axis=0) x1_max, x2_max = np.max(data, axis=0) x1_min, x1_max = expand(x1_min, x1_max) x2_min, x2_max = expand(x2_min, x2_max) plt.xlim((x1_min, x1_max)) plt.ylim((x2_min, x2_max)) plt.grid(True) plt.title(u'$epsilon$ = %.1f m = %d clustering num %d'%(eps, min_samples, n_clusters), fontsize=14) plt.tight_layout() plt.subplots_adjust(top=0.9) plt.show()

注意：可以看到在測試樣例的兩端，相比與孤立森林，DBSCAN能夠很好地對“尖端”處的樣本的分類。

2.3 核心代碼

model = DBSCAN(eps=eps, min_samples=min_samples) # 構(gòu)造分類器

from sklearn.cluster import DBSCANfrom sklearn import metricsdata = X_train_demo.valueseps, min_samples = 0.2, 10# eps為領(lǐng)域的大小，min_samples為領(lǐng)域內(nèi)最小點的個數(shù)model = DBSCAN(eps=eps, min_samples=min_samples) # 構(gòu)造分類器model.fit(data) # 擬合labels = model.labels_ # 獲取類別標(biāo)簽，-1表示未分類# 獲取其中的core pointscore_indices = np.zeros_like(labels, dtype=bool) # 生成數(shù)據(jù)類型和數(shù)據(jù)shape和指定array一致的變量core_indices[model.core_sample_indices_] = True # model.core_sample_indices_ border point位于labels中的下標(biāo)core_point = data[core_indices]# 獲取非異常點normal_point = data[labels>=0]# 繪制剔除了異常值后的圖plt.scatter(normal_point[:,0],normal_point[:,1],edgecolors='k')plt.show()

2.4 構(gòu)造過濾函數(shù)

該函數(shù)先進(jìn)行了標(biāo)準(zhǔn)化，方便使用固定的參數(shù)進(jìn)行分析

2.4.1 過濾函數(shù)def filter_data(data0, params): from sklearn.cluster import DBSCAN from sklearn import metrics scaler = StandardScaler() scaler.fit(data0) data = scaler.transform(data0) eps, min_samples = params # eps為領(lǐng)域的大小，min_samples為領(lǐng)域內(nèi)最小點的個數(shù) model = DBSCAN(eps=eps, min_samples=min_samples) # 構(gòu)造分類器 model.fit(data) # 擬合 labels = model.labels_ # 獲取類別標(biāo)簽，-1表示未分類 # 獲取其中的core points core_indices = np.zeros_like(labels, dtype=bool) # 生成數(shù)據(jù)類型和數(shù)據(jù)shape和指定array一致的變量 core_indices[model.core_sample_indices_] = True # model.core_sample_indices_ border point位于labels中的下標(biāo) core_point = data[core_indices] # 獲取非異常點 normal_point = data0[labels>=0] return normal_point2.4.2 衡量分類結(jié)果

（markdown格式懶得轉(zhuǎn)，直接截圖了::>_0]進(jìn)行剔除異常點之前

剔除異常點之后plt.scatter(X_train_normal[:,0],X_train_normal[:,1])plt.show()

4. Local Outlier Factor（LOF）

LOF通過計算一個數(shù)值score來反映一個樣本的異常程度。 這個數(shù)值的大致意思是：

一個樣本點周圍的樣本點所處位置的平均密度比上該樣本點所在位置的密度。比值越大于1，則該點所在位置的密度越小于其周圍樣本所在位置的密度。

## 參考https://blog.csdn.net/hb707934728/article/details/71515160## 官方示例 https://scikit-learn.org/stable/auto_examples/cluster/plot_dbscan.html#sphx-glr-auto-examples-cluster-plot-dbscan-pyimport numpy as npimport matplotlib.pyplot as pltimport matplotlib.colorsfrom sklearn.neighbors import LocalOutlierFactordef expand(a, b): d = (b - a) * 0.1 return a-d, b+dif __name__ == "__main__": N = 1000 data = X_train_demo.values # 數(shù)據(jù)1的參數(shù)：(epsilon, min_sample) params = ((0.01, 5), (0.05, 10), (0.1, 15), (0.15, 20), (0.2, 25), (0.25, 30)) plt.figure(figsize=(12, 8), facecolor='w') plt.suptitle(u'DBSCAN clustering', fontsize=20) for i in range(6): outliers_fraction, min_samples = params[i] #參數(shù)含義： #eps:半徑，表示以給定點P為中心的圓形鄰域的范圍 #min_samples:以點P為中心的鄰域內(nèi)最少點的數(shù)量 #如果滿足,以點P為中心,半徑為EPS的鄰域內(nèi)點的個數(shù)不少于MinPts,則稱點P為核心點 model = LocalOutlierFactor(n_neighbors=min_samples, contamination=outliers_fraction) y_hat = model.fit_predict(X_train) # 統(tǒng)計總共有積累，其中為-1的為未分類樣本 y_unique = np.unique(y_hat) # clrs = [] # for c in np.linspace(16711680, 255, y_unique.size): # clrs.append('#%06x' % c) plt.subplot(2, 3, i+1) # 對第幾個圖繪制，2行3列，繪制第i+1個圖 # plt.cm.spectral https://blog.csdn.net/robin_Xu_shuai/article/details/79178857 clrs = plt.cm.Spectral(np.linspace(0, 0.8, y_unique.size)) #用于給畫圖灰色 for k, clr in zip(y_unique, clrs): cur = (y_hat == k) if k == -1: # 用于繪制未分類樣本 plt.scatter(data[cur, 0], data[cur, 1], s=20, c='k') continue # 繪制正常節(jié)點 plt.scatter(data[cur, 0], data[cur, 1], s=30, c=clr, edgecolors='k') x1_max, x2_max = np.max(data, axis=0) x1_min, x2_min = np.min(data, axis=0) x1_min, x1_max = expand(x1_min, x1_max) x2_min, x2_max = expand(x2_min, x2_max) plt.xlim((x1_min, x1_max)) plt.ylim((x2_min, x2_max)) plt.grid(True) plt.title(u'outliers_fraction = %.1f min_samples = %d'%(outliers_fraction, min_samples), fontsize=12) plt.tight_layout() plt.subplots_adjust(top=0.9) plt.show()

4.1 核心代碼from sklearn.neighbors import LocalOutlierFactorX_train = X_train_demo.values# 構(gòu)造分類器## 25個樣本點為一組，異常值點比例為0.2clf = LocalOutlierFactor(n_neighbors=25, contamination=0.2)# 預(yù)測，結(jié)果為-1或者1labels = clf.fit_predict(X_train)# 獲取正常點X_train_normal = X_train[labels>0]進(jìn)行剔除異常點之前plt.scatter(X_train[:,0],X_train[:,1])plt.show()

剔除異常點之后plt.scatter(X_train_normal[:,0],X_train_normal[:,1])plt.show()