數(shù)學推導+純Python實現(xiàn)機器學習算法15：GBDT
點擊上方“小白學視覺”，選擇加"星標"或“置頂”
重磅干貨，第一時間送達
時隔大半年，機器學習算法推導系列終于有時間繼續(xù)更新了。在之前的14講中，筆者將監(jiān)督模型中主要的單模型算法基本都過了一遍。預計在接下來的10講中，筆者將努力更新完以GBDT代表的集成學習模型，以EM算法、CRF和隱馬為代表的概率圖模型以及以聚類降維為代表的無監(jiān)督學習算法。
(2) 對
有：
對每個樣本
，計算負梯度，即殘差
將上步得到的殘差作為樣本新的真實值，并將數(shù)據(jù)
作為下棵樹的訓練數(shù)據(jù)，得到一顆新的回歸樹
其對應的葉子節(jié)點區(qū)域為
。其中
為回歸樹t的葉子節(jié)點的個數(shù)。對葉子區(qū)域
計算最佳擬合值
更新強學習器
class TreeNode():def __init__(self, feature_i=None, threshold=None,value=None, true_branch=None, false_branch=None):pass
class Tree(object):def __init__(self, min_samples_split=2, min_impurity=1e-7,max_depth=float("inf"), loss=None):self.root = None # Root node in dec. tree# Minimum n of samples to justify splitself.min_samples_split = min_samples_split# The minimum impurity to justify splitself.min_impurity = min_impurity# The maximum depth to grow the tree toself.max_depth = max_depth# Function to calculate impurity (classif.=>info gain, regr=>variance reduct.)# 切割樹的方法，gini，方差等self._impurity_calculation = None# Function to determine prediction of y at leaf# 樹節(jié)點取值的方法，分類樹：選取出現(xiàn)最多次數(shù)的值，回歸樹：取所有值的平均值self._leaf_value_calculation = None# If y is one-hot encoded (multi-dim) or not (one-dim)self.one_dim = None# If Gradient Boostself.loss = lossdef fit(self, X, y, loss=None):""" Build decision tree """passdef _build_tree(self, X, y, current_depth=0):""" Recursive method which builds out the decision tree and splits X and respective ypassdef predict_value(self, x, tree=None):""" Do a recursive search down the tree and make a prediction of the data sample by thevalue of the leaf that we end up at """passdef predict(self, X):""" Classify samples one by one and return the set of labels """passdef print_tree(self, tree=None, indent=" "):pass
class RegressionTree(Tree):# 使用方差法進行樹分割def _calculate_variance_reduction(self, y, y1, y2):var_tot = calculate_variance(y)var_1 = calculate_variance(y1)var_2 = calculate_variance(y2)frac_1 = len(y1) / len(y)frac_2 = len(y2) / len(y)# Calculate the variance reductionvariance_reduction = var_tot - (frac_1 * var_1 + frac_2 * var_2)return sum(variance_reduction)# 使用均值法取葉子結點值def _mean_of_y(self, y):value = np.mean(y, axis=0)return value if len(value) > 1 else value[0]# 回歸樹擬合def fit(self, X, y):self._impurity_calculation = self._calculate_variance_reductionself._leaf_value_calculation = self._mean_of_ysuper(RegressionTree, self).fit(X, y)
class Loss(object):def loss(self, y_true, y_pred):return NotImplementedError()def gradient(self, y, y_pred):raise NotImplementedError()def acc(self, y, y_pred):return 0class SquareLoss(Loss):def __init__(self): passdef loss(self, y, y_pred):return 0.5 * np.power((y - y_pred), 2)def gradient(self, y, y_pred):return -(y - y_pred)
class GBDT(object):def __init__(self, n_estimators, learning_rate, min_samples_split,min_impurity, max_depth, regression):# 基本參數(shù)self.n_estimators = n_estimatorsself.learning_rate = learning_rateself.min_samples_split = min_samples_splitself.min_impurity = min_impurityself.max_depth = max_depthself.regression = regressionself.loss = SquareLoss()if not self.regression:self.loss = SotfMaxLoss()# 分類問題也可以使用回歸樹，利用殘差去學習概率self.estimators = []for i in range(self.n_estimators):self.estimators.append(RegressionTree(min_samples_split=self.min_samples_split,min_impurity=self.min_impurity,max_depth=self.max_depth))# 擬合方法def fit(self, X, y):# 讓第一棵樹去擬合模型self.estimators[0].fit(X, y)y_pred = self.estimators[0].predict(X)for i in range(1, self.n_estimators):gradient = self.loss.gradient(y, y_pred)self.estimators[i].fit(X, gradient)y_pred -= np.multiply(self.learning_rate, self.estimators[i].predict(X))# 預測方法def predict(self, X):y_pred = self.estimators[0].predict(X)for i in range(1, self.n_estimators):y_pred -= np.multiply(self.learning_rate, self.estimators[i].predict(X))if not self.regression:# Turn into probability distributiony_pred = np.exp(y_pred) / np.expand_dims(np.sum(np.exp(y_pred), axis=1), axis=1)# Set label to the value that maximizes probabilityy_pred = np.argmax(y_pred, axis=1)return y_pred
# regression treeclass GBDTRegressor(GBDT):def __init__(self, n_estimators=200, learning_rate=0.5, min_samples_split=2,min_var_red=1e-7, max_depth=4, debug=False):super(GBDTRegressor, self).__init__(n_estimators=n_estimators,learning_rate=learning_rate,min_samples_split=min_samples_split,min_impurity=min_var_red,max_depth=max_depth,regression=True)# classification treeclass GBDTClassifier(GBDT):def __init__(self, n_estimators=200, learning_rate=.5, min_samples_split=2,min_info_gain=1e-7, max_depth=2, debug=False):super(GBDTClassifier, self).__init__(n_estimators=n_estimators,learning_rate=learning_rate,min_samples_split=min_samples_split,min_impurity=min_info_gain,max_depth=max_depth,regression=False)def fit(self, X, y):y = to_categorical(y)super(GBDTClassifier, self).fit(X, y)
from sklearn import datasetsboston = datasets.load_boston()X, y = shuffle_data(boston.data, boston.target, seed=13)X = X.astype(np.float32)offset = int(X.shape[0] * 0.9)X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)model = GBDTRegressor()model.fit(X_train, y_train)y_pred = model.predict(X_test)# Color mapcmap = plt.get_cmap('viridis')mse = mean_squared_error(y_test, y_pred)print ("Mean Squared Error:", mse)# Plot the resultsm1 = plt.scatter(range(X_test.shape[0]), y_test, color=cmap(0.5), s=10)m2 = plt.scatter(range(X_test.shape[0]), y_pred, color='black', s=10)plt.suptitle("Regression Tree")plt.title("MSE: %.2f" % mse, fontsize=10)plt.xlabel('sample')plt.ylabel('house price')plt.legend((m1, m2), ("Test data", "Prediction"), loc='lower right')plt.show();
好消息！
小白學視覺知識星球
開始面向外開放啦??????
下載1：OpenCV-Contrib擴展模塊中文版教程 在「小白學視覺」公眾號后臺回復：擴展模塊中文教程，即可下載全網(wǎng)第一份OpenCV擴展模塊教程中文版，涵蓋擴展模塊安裝、SFM算法、立體視覺、目標跟蹤、生物視覺、超分辨率處理等二十多章內容。 下載2：Python視覺實戰(zhàn)項目52講 在「小白學視覺」公眾號后臺回復：Python視覺實戰(zhàn)項目，即可下載包括圖像分割、口罩檢測、車道線檢測、車輛計數(shù)、添加眼線、車牌識別、字符識別、情緒檢測、文本內容提取、面部識別等31個視覺實戰(zhàn)項目，助力快速學校計算機視覺。 下載3：OpenCV實戰(zhàn)項目20講 在「小白學視覺」公眾號后臺回復：OpenCV實戰(zhàn)項目20講，即可下載含有20個基于OpenCV實現(xiàn)20個實戰(zhàn)項目，實現(xiàn)OpenCV學習進階。 交流群
歡迎加入公眾號讀者群一起和同行交流，目前有SLAM、三維視覺、傳感器、自動駕駛、計算攝影、檢測、分割、識別、醫(yī)學影像、GAN、算法競賽等微信群（以后會逐漸細分），請掃描下面微信號加群，備注：”昵稱+學校/公司+研究方向“，例如：”張三 + 上海交大 + 視覺SLAM“。請按照格式備注，否則不予通過。添加成功后會根據(jù)研究方向邀請進入相關微信群。請勿在群內發(fā)送廣告，否則會請出群，謝謝理解~
評論
圖片
表情