1、决策树算法是一种非参数的决策算法，它根据数据的不同特征进行多层次的分类和判断，最终决策出所需要预测的结果。它既可以解决分类算法，也可以解决回归问题，具有很好的解释能力。另外，对于决策树的构建方法具有多种出发点，它具有多种构建方式，如何构建决策树的出发点主要在于决策树每一个决策点上需要在哪些维度上进行划分以及在这些维度的哪些阈值节点做划分等细节问题。
具体在sklearn中调用决策树算法解决分类问题和回归问题的程序代码如下所示：
#1-1导入基础训练数据集import numpy as npimport matplotlib.pyplot as pltfrom sklearn import datasetsd=datasets.load_iris()x=d.data[:,2:]y=d.targetplt.figure()plt.scatter(x[y==0,0],x[y==0,1],color="r")plt.scatter(x[y==1,0],x[y==1,1],color="g")plt.scatter(x[y==2,0],x[y==2,1],color="b")plt.show()#1-2导入sklearn中的决策树算法进行数据的分类问题实现训练预测from sklearn.tree import DecisionTreeClassifierdt1=DecisionTreeClassifier(max_depth=2,criterion="entropy") #定义决策树的分类器相关决策超参数dt1.fit(x,y)def plot_decision_boundary(model,axis): #决策边界输出函数（二维数据点） x0,x1=np.meshgrid( np.linspace(axis[0],axis[1],int((axis[1]-axis[0])*100)).reshape(-1,1), np.linspace(axis[2],axis[3], int((axis[3] - axis[2]) * 100)).reshape(-1,1) ) x_new=np.c_[x0.ravel(),x1.ravel()] y_pre=model.predict(x_new) zz=y_pre.reshape(x0.shape) from matplotlib.colors import ListedColormap cus=ListedColormap(["#EF9A9A","#FFF59D","#90CAF9"]) plt.contourf(x0,x1,zz,cmap=cus)plot_decision_boundary(dt1,axis=[0.5,8,0,3])plt.scatter(x[y==0,0],x[y==0,1],color="r")plt.scatter(x[y==1,0],x[y==1,1],color="g")plt.scatter(x[y==2,0],x[y==2,1],color="b")plt.show()#定义二分类问题的信息熵计算函数np.sum(-p*np.log(p))def entropy(p): return -p*np.log(p)-(1-p)*np.log(1-p)x1=np.linspace(0.01,0.99,100)y1=entropy(x1)plt.plot(x1,y1,"r")plt.show()#利用信息熵的原理对数据进行实现划分，决策树信息熵构建方式的原理实现代码def split(x,y,d,value): index_a=(x[:,d]<=value) index_b=(x[:,d]>value) return x[index_a],x[index_b],y[index_a],y[index_b]from collections import Counterdef entropy(y): Counter1=Counter(y) res=0.0 for num in Counter1.values(): p=num/len(y) res+=-p*np.log(p) return resdef try_spit(x,y): best_entropy=float("inf") best_d,best_v=-1,-1 for d in range(x.shape[1]): sorted_index=np.argsort(x[:,d]) for i in range(1,len(x)): if x[sorted_index[i-1],d] != x[sorted_index[i],d]: v=(x[sorted_index[i-1],d]+x[sorted_index[i],d])/2 x_l,x_r,y_l,y_r=split(x,y,d,v) e=entropy(y_l)+entropy(y_r) if e<best_entropy: best_entropy,best_d,best_v=e,d,v return best_entropy,best_d,best_vprint(try_spit(x,y))best_entropy=try_spit(x,y)[0]best_d=try_spit(x,y)[1]best_v=try_spit(x,y)[2]x_l,x_r,y_l,y_r=split(x,y,best_d,best_v)print(entropy(y_l))print(entropy(y_r))#基尼系数方式构建决策树的代码实现from sklearn.tree import DecisionTreeClassifierdt2=DecisionTreeClassifier(max_depth=2,criterion="gini") #定义决策树的分类器相关决策超参数dt2.fit(x,y)plot_decision_boundary(dt2,axis=[0.5,8,0,3])plt.scatter(x[y==0,0],x[y==0,1],color="r")plt.scatter(x[y==1,0],x[y==1,1],color="g")plt.scatter(x[y==2,0],x[y==2,1],color="b")plt.show()def split(x,y,d,value): index_a=(x[:,d]<=value) index_b=(x[:,d]>value) return x[index_a],x[index_b],y[index_a],y[index_b]from collections import Counterdef gini(y): Counter1 = Counter(y) res = 1.0 for num in Counter1.values(): p = num / len(y) res -= p**2 return resdef try_spit1(x,y): best_gini=float("inf") best_d,best_v=-1,-1 for d in range(x.shape[1]): sorted_index=np.argsort(x[:,d]) for i in range(1,len(x)): if x[sorted_index[i-1],d] != x[sorted_index[i],d]: v=(x[sorted_index[i-1],d]+x[sorted_index[i],d])/2 x_l,x_r,y_l,y_r=split(x,y,d,v) g=gini(y_l)+gini(y_r) if g<best_gini: best_gini,best_d,best_v=g,d,v return [best_gini,best_d,best_v]best_gini,best_d,best_v=try_spit1(x,y)print(best_gini,best_d,best_v)#对于决策数进行相应的剪枝，尽可能降低过拟合的情况import numpy as npimport matplotlib.pyplot as pltfrom sklearn import datasetsx,y=datasets.make_moons(noise=0.25,random_state=666) #生成数据默认为100个数据样本print(x.shape)print(y.shape)plt.figure()plt.scatter(x[y==0,0],x[y==0,1],color="r")plt.scatter(x[y==1,0],x[y==1,1],color="g")plt.show()from sklearn.tree import DecisionTreeClassifierdt2=DecisionTreeClassifier(max_depth=2,min_samples_split=10,min_samples_leaf=6,max_leaf_nodes=4) #默认情况下则为基尼系数，对于深度会一直划分下去使得基尼系数为0为止#决策树的主要超参数dt2.fit(x,y)plot_decision_boundary(dt2,axis=[-2,3,-1,1.5])plt.scatter(x[y==0,0],x[y==0,1],color="r")plt.scatter(x[y==1,0],x[y==1,1],color="g")plt.show()#使用决策树解决回归问题import numpy as npimport matplotlib.pyplot as pltfrom sklearn import datasetsd=datasets.load_boston()x=d.datay=d.targetprint(x.shape)print(y.shape)from sklearn.model_selection import train_test_splitx_train,x_test,y_train,y_test=train_test_split(x,y,random_state=666)from sklearn.tree import DecisionTreeRegressordr=DecisionTreeRegressor()dr.fit(x_train,y_train)print(dr.score(x_test,y_test))print(dr.score(x_train,y_train)) #在训练数据集的R2=1，而在测试集上比较小，因此已经产生了过拟合，学习曲线可以比较好的反映过拟合情况#绘制不同参数组合情况下的学习曲线from sklearn.metrics import mean_squared_errordef plot_learning_curve(algo,x_train,x_test,y_train,y_test): train_score = [] test_score = [] for i in range(1, len(x_train)): algo.fit(x_train[:i], y_train[:i]) y_train_pre = algo.predict(x_train[:i]) y_test_pre =algo.predict(x_test) train_score.append(mean_squared_error(y_train[:i], y_train_pre)) test_score.append(mean_squared_error(y_test, y_test_pre)) plt.figure() plt.plot([i for i in range(1, len(x_train))], np.sqrt(train_score), "g", label="train_error") plt.plot([i for i in range(1, len(x_train))], np.sqrt(test_score), "r", label="test_error") plt.legend() #plt.axis([0,len(x_train)+1,0,5]) plt.show()plot_learning_curve(DecisionTreeRegressor(max_depth=1),x_train,x_test,y_train,y_test) #欠拟合的情况plot_learning_curve(DecisionTreeRegressor(max_depth=5),x_train,x_test,y_train,y_test) #较好拟合的情况plot_learning_curve(DecisionTreeRegressor(max_depth=15),x_train,x_test,y_train,y_test) #过拟合的情况
