第六章部分:
#代码6-1 描述性统计分析 import numpy as np import pandas as pd inputfile = "data.csv" data = pd.read_csv(inputfile) #依次计算最小值,最大值,均值,标准差 description = [data.min(), data.max(), data.mean(), data.std()] description = pd.DataFrame(description, index = ['Min','Max','Mean','Std']).T print('描述性统计结果:\n',np.round(description,2)) #保留两位小数
#代码6-2 corr = data.corr(method='pearson') print('相关系数矩阵为:\n',np.round(corr,2)) #保留两位小数
#代码6-3 import matplotlib.pyplot as plt import seaborn as sns plt.subplots(figsize=(10,10)) #设置画面大小 sns.heatmap(corr,annot=True,vmax=1,square=True,cmap="Blues") plt.title('相关性热力图') plt.show() plt.close
#代码6—4 import numpy as np import pandas as pd from sklearn.linear_model import Lasso inputflie='data.csv' data = pd.read_csv(inputfile) lasso = Lasso(1000) #调用Lasso函数,设置λ的值为1000 lasso.fit(data.iloc[:,0:13],data['y']) print('相关系数为:',np.round(lasso.coef_,5)) #输出结果,保留五位小数 print('相关系数非零个数为:',np.sum(lasso.coef_ != 0)) #计算非零个数 mask = lasso.coef_ != 0 print('相关系数是否为零:',mask) outputfile='data2/new_reg_data.csv' #输出的数据文件 mask = np.append(mask,True) print(mask) new_reg_data = data.iloc[:,mask] #返回相关系数非零的数据 new_reg_data.to_csv(outputfile) #存储数据 print('输出数据的维度为:',new_reg_data.shape) #查看输出数据的维度
#代码6—5 import sys sys.path.append('data2/code') #设置路径 import numpy as np import pandas as pd from GM11 import GM11 inputfile1 = 'data2/new_reg_data.csv' inputfile2 = 'data.csv' new_reg_data = pd.read_csv(inputfile1) data = pd.read_csv(inputfile2) new_reg_data.index = range(1994,2014) new_reg_data.loc[2014]=None new_reg_data.loc[2015]=None cols = ['x1','x3','x4','x5','x6','x7','x8','x13'] for i in cols: f = GM11(new_reg_data.loc[range(1994,2014),i].to_numpy())[0] new_reg_data.loc[2014,i] = f(len(new_reg_data)-1) new_reg_data.loc[2015,i] = f(len(new_reg_data)) new_reg_data[i] = new_reg_data[i].round(2) outputfile = 'data2/new_reg_data_GM11.xls' y = list(data['y'].values) y.extend([np.nan,np.nan]) new_reg_data['y'] = y new_reg_data.to_excel(outputfile) print('预测结果为:\n',new_reg_data.loc[2014:2015,:])
#代码6-6构建支持向量回归预测模型 import matplotlib.pyplot as plt from sklearn.svm import LinearSVR inputfile = 'data2/new_reg_data_GM11.xls' data = pd.read_excel(inputfile) feature = ['x1', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x13'] #属性所在列 data.index = range(1994,2016) data_train = data.loc[range(1994, 2014)].copy() #灰色预测后的数据,取2014年前的数据建模 data_mean = data_train.mean() data_std = data_train.std() data_train = (data_train - data_mean)/data_std #数据标准化 x_train = data_train[feature].to_numpy() y_train = data_train['y'].to_numpy() #标签数据 linearsvr = LinearSVR() #调用LinearSVR()函数 linearsvr.fit(x_train, y_train) x = ((data[feature] - data_mean[feature])/data_std[feature]).to_numpy() #预测,并还原结果 data[u'y_pred'] = linearsvr.predict(x) * data_std['y'] + data_mean['y'] outputfile = 'data2/new_reg_data_GM11_revenue.xls' data.to_excel(outputfile) print('真实值与预测值分别为:\n', data[['y', 'y_pred']]) fig = data[['y', 'y_pred']].plot(subplots = True, style=['b-o', 'r-*']) plt.show()
#2016 import sys sys.path.append('data2/code') import numpy as np import pandas as pd from GM11 import GM11 inputfile1='data2/new_reg_data_GM11.xls' inputfile2='data.csv' new_reg_data=pd.read_excel(inputfile1) data=pd.read_csv(inputfile2) new_reg_data.index=range(1994,2016) #new_reg_data.loc[2014]=None new_reg_data.loc[2016]=None cols=['x1','x3','x4','x5','x6','x7','x8','x13'] for i in cols: f=GM11(new_reg_data.loc[range(1994,2015),i].values)[0] new_reg_data.loc[2016,i]=f(len(new_reg_data)-1) # new_reg_data.loc[2015,i]=f(len(new_reg_data)) new_reg_data[i]=new_reg_data[i].round(2) outputfile='data2/new_reg_data_GM11_2.xls' y=list(data['y'].values) y.extend([np.nan,np.nan]) #new_reg_data['y']=y new_reg_data.to_excel(outputfile) print('预测结果为:\n',new_reg_data.loc[2016,:])
第四章部分
#代码4-9 对一个10*4维的随机矩阵进行主成分分析 import numpy as np from sklearn.decomposition import PCA D = np.random.rand(10,4) pca = PCA() pca.fit(D) pca.components_ pca.explained_variance_ratio_
import random import pandas as pd import numpy as np import matplotlib.pyplot as plt # 计算欧拉距离 def calcDis(dataSet, centroids, k): clalist=[] for data in dataSet: diff = np.tile(data, (k, 1)) - centroids #相减 (np.tile(a,(2,1))就是把a先沿x轴复制1倍,即没有复制,仍然是 [0,1,2]。 再把结果沿y方向复制2倍得到array([[0,1,2],[0,1,2]])) squaredDiff = diff ** 2 #平方 squaredDist = np.sum(squaredDiff, axis=1) #和 (axis=1表示行) distance = squaredDist ** 0.5 #开根号 clalist.append(distance) clalist = np.array(clalist) #返回一个每个点到质点的距离len(dateSet)*k的数组 return clalist # 计算质心 def classify(dataSet, centroids, k): # 计算样本到质心的距离 clalist = calcDis(dataSet, centroids, k) # 分组并计算新的质心 minDistIndices = np.argmin(clalist, axis=1) #axis=1 表示求出每行的最小值的下标 newCentroids = pd.DataFrame(dataSet).groupby(minDistIndices).mean() #DataFramte(dataSet)对DataSet分组,groupby(min)按照min进行统计分类,mean()对分类结果求均值 newCentroids = newCentroids.values # 计算变化量 changed = newCentroids - centroids return changed, newCentroids # 使用k-means分类 def kmeans(dataSet, k): # 随机取质心 centroids = random.sample(dataSet, k) # 更新质心 直到变化量全为0 changed, newCentroids = classify(dataSet, centroids, k) while np.any(changed != 0): changed, newCentroids = classify(dataSet, newCentroids, k) centroids = sorted(newCentroids.tolist()) #tolist()将矩阵转换成列表 sorted()排序 # 根据质心计算每个集群 cluster = [] clalist = calcDis(dataSet, centroids, k) #调用欧拉距离 minDistIndices = np.argmin(clalist, axis=1) for i in range(k): cluster.append([]) for i, j in enumerate(minDistIndices): #enymerate()可同时遍历索引和遍历元素 cluster[j].append(dataSet[i]) return centroids, cluster # 创建数据集 def createDataSet(): return [[1, 1], [1, 2], [2, 1], [6, 4], [6, 3], [5, 4]] if __name__=='__main__': dataset = createDataSet() centroids, cluster = kmeans(dataset, 2) print('质心为:%s' % centroids) print('集群为:%s' % cluster) for i in range(len(dataset)): plt.scatter(dataset[i][0],dataset[i][1], marker = 'o',color = 'green', s = 40 ,label = '原始点') # 记号形状 颜色 点的大小 设置标签 for j in range(len(centroids)): plt.scatter(centroids[j][0],centroids[j][1],marker='x',color='red',s=50,label='质心') plt.show()
import numpy as np import random import matplotlib.pyplot as plt def distance(point1, point2): # 计算距离(欧几里得距离) return np.sqrt(np.sum((point1 - point2) ** 2)) def k_means(data, k, max_iter=10000): centers = {} # 初始聚类中心 # 初始化,随机选k个样本作为初始聚类中心。 random.sample(): 随机不重复抽取k个值 n_data = data.shape[0] # 样本个数 for idx, i in enumerate(random.sample(range(n_data), k)): # idx取值范围[0, k-1],代表第几个聚类中心; data[i]为随机选取的样本作为聚类中心 centers[idx] = data[i] # 开始迭代 for i in range(max_iter): # 迭代次数 print("开始第{}次迭代".format(i+1)) clusters = {} # 聚类结果,聚类中心的索引idx -> [样本集合] for j in range(k): # 初始化为空列表 clusters[j] = [] for sample in data: # 遍历每个样本 distances = [] # 计算该样本到每个聚类中心的距离 (只会有k个元素) for c in centers: # 遍历每个聚类中心 # 添加该样本点到聚类中心的距离 distances.append(distance(sample, centers[c])) idx = np.argmin(distances) # 最小距离的索引 clusters[idx].append(sample) # 将该样本添加到第idx个聚类中心 pre_centers = centers.copy() # 记录之前的聚类中心点 for c in clusters.keys(): # 重新计算中心点(计算该聚类中心的所有样本的均值) centers[c] = np.mean(clusters[c], axis=0) is_convergent = True for c in centers: if distance(pre_centers[c], centers[c]) > 1e-8: # 中心点是否变化 is_convergent = False break if is_convergent == True: # 如果新旧聚类中心不变,则迭代停止 break return centers, clusters def predict(p_data, centers): # 预测新样本点所在的类 # 计算p_data 到每个聚类中心的距离,然后返回距离最小所在的聚类。 distances = [distance(p_data, centers[c]) for c in centers] return np.argmin(distances)
x = np.random.randint(0,high=10,size=(200,2)) centers,clusters = k_means(x,3)
print(centers) clusters
for center in centers: plt.scatter(centers[center][0],centers[center][1],marker='*',s=150) colors = ['r','b','y','m','c','g'] for c in clusters: for point in clusters[c]: plt.scatter(point[0],point[1],c = colors[c])
import numpy as np from sklearn.datasets import make_blobs n_samples = 1500 random_state = 170 transformation = [[0.60834549, -0.63667341], [-0.40887718, 0.85253229]] X, y = make_blobs(n_samples=n_samples, random_state=random_state) X_aniso = np.dot(X, transformation) # Anisotropic blobs X_varied, y_varied = make_blobs( n_samples=n_samples, cluster_std=[1.0, 2.5, 0.5], random_state=random_state ) # Unequal variance X_filtered = np.vstack( (X[y == 0][:500], X[y == 1][:100], X[y == 2][:10]) ) # Unevenly sized blobs y_filtered = [0] * 500 + [1] * 100 + [2] * 10
import matplotlib.pyplot as plt fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(12, 12)) axs[0, 0].scatter(X[:, 0], X[:, 1], c=y) axs[0, 0].set_title("Mixture of Gaussian Blobs") axs[0, 1].scatter(X_aniso[:, 0], X_aniso[:, 1], c=y) axs[0, 1].set_title("Anisotropically Distributed Blobs") axs[1, 0].scatter(X_varied[:, 0], X_varied[:, 1], c=y_varied) axs[1, 0].set_title("Unequal Variance") axs[1, 1].scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_filtered) axs[1, 1].set_title("Unevenly Sized Blobs") plt.suptitle("Ground truth clusters").set_y(0.95) plt.show()
from sklearn.cluster import KMeans common_params = { "n_init": "auto", "random_state": random_state, } fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(12, 12)) y_pred = KMeans(n_clusters=2, **common_params).fit_predict(X) axs[0, 0].scatter(X[:, 0], X[:, 1], c=y_pred) axs[0, 0].set_title("Non-optimal Number of Clusters") y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X_aniso) axs[0, 1].scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred) axs[0, 1].set_title("Anisotropically Distributed Blobs") y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X_varied) axs[1, 0].scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred) axs[1, 0].set_title("Unequal Variance") y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X_filtered) axs[1, 1].scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred) axs[1, 1].set_title("Unevenly Sized Blobs") plt.suptitle("Unexpected KMeans clusters").set_y(0.95) plt.show()
y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X) plt.scatter(X[:, 0], X[:, 1], c=y_pred) plt.title("Optimal Number of Clusters") plt.show()
y_pred = KMeans(n_clusters=3, n_init=10, random_state=random_state).fit_predict( X_filtered ) plt.scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred) plt.title("Unevenly Sized Blobs \nwith several initializations") plt.show()
from sklearn.mixture import GaussianMixture fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(12, 6)) y_pred = GaussianMixture(n_components=3).fit_predict(X_aniso) ax1.scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred) ax1.set_title("Anisotropically Distributed Blobs") y_pred = GaussianMixture(n_components=3).fit_predict(X_varied) ax2.scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred) ax2.set_title("Unequal Variance") plt.suptitle("Gaussian mixture clusters").set_y(0.95) plt.show()