快照集成（Snapshot Ensemble）

快照集成

Snapshot Ensemble
发表于 ICLR 2017 论文地址及代码见下
Paper： https://arxiv.org/pdf/1704.00109.pdf
找论文搭配 Sci-Hub 食用更佳
Sci-Hub 实时更新 : https://tool.yovisun.com/scihub/
公益科研通文献求助：https://www.ablesci.com/

代码在最下面，也可以访问Github，及其他集成学习代码也在

只需要训练一次就可以得到收敛于多个不同最小值处的模型，从而进行模型集成。

快拿去测试你的数据吧！

from sklearn.datasets import make_blobs
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import Callback
from keras.optimizers import SGD
from keras import backend
from math import pi, cos, floor
from sklearn.datasets import make_blobs
from sklearn.metrics import accuracy_score
from keras.models import load_model, Sequential
from keras.layers import Dense
from matplotlib import pyplot
from numpy import std, mean, array, argmax
import numpy as np
 
# snapshot ensemble with custom learning rate schedule
class SnapshotEnsemble(Callback):
	# constructor
	def __init__(self, n_epochs, n_cycles, lrate_max, verbose=0):
		self.epochs = n_epochs
		self.cycles = n_cycles
		self.lr_max = lrate_max
		self.lrates = list()
 
	# calculate learning rate for epoch
	def cosine_annealing(self, epoch, n_epochs, n_cycles, lrate_max):
		epochs_per_cycle = floor(n_epochs/n_cycles)
		cos_inner = (pi * (epoch % epochs_per_cycle)) / (epochs_per_cycle)
		return lrate_max/2 * (cos(cos_inner) + 1)
 
	# calculate and set learning rate at the start of the epoch
	def on_epoch_begin(self, epoch, logs={
    }):
		# calculate learning rate
		lr = self.cosine_annealing(epoch, self.epochs, self.cycles, self.lr_max)
		# set learning rate
		backend.set_value(self.model.optimizer.lr, lr)
		# log value
		self.lrates.append(lr)
 
	# save models at the end of each cycle
	def on_epoch_end(self, epoch, logs={
    }):
		# check if we can save model
		epochs_per_cycle = floor(self.epochs / self.cycles)
		if epoch != 0 and (epoch + 1) % epochs_per_cycle == 0:
			# save model to file
			filename = "snapshot_model_energy_%d.h5" % int((epoch + 1) / epochs_per_cycle)
			self.model.save(filename)
			print('>saved snapshot %s, epoch %d' % (filename, epoch))
			
# load models from file
def load_all_models(n_models):
	all_models = list()
	for i in range(n_models):
		# define filename for this ensemble
		filename = 'snapshot_model_energy_' + str(i + 1) + '.h5'
		# load model from file
		model = load_model(filename)
		# add to list of members
		all_models.append(model)
		print('>loaded %s' % filename)
	return all_models
 
# make an ensemble prediction for multi-class classification
def ensemble_predictions(members, testX):
	# make predictions
	yhats = [model.predict(testX) for model in members]
	yhats = array(yhats)
	# sum across ensemble members
	summed = np.sum(yhats, axis=0)  # soft voting
	# argmax across classes
	result = argmax(summed, axis=1)
	return result
 
# evaluate a specific number of members in an ensemble
def evaluate_n_members(members, n_members, testX, testy):
	# select a subset of members
	subset = members[:n_members]
	# make prediction
	yhat = ensemble_predictions(subset, testX)
	# calculate accuracy
	return accuracy_score(testy, yhat) # precision f1 recall
	
from sklearn.model_selection import train_test_split
# load data train_test_split
data_train = np.loadtxt(open("D:/what.csv", encoding='gb18030', errors="ignore"), delimiter=",", skiprows=0)
X_train, y_train = data_train[:, :-1], data_train[:, -1]
trainX, testX, trainy, testy = train_test_split(X_train, y_train, test_size = .3, random_state = 3)
# print(trainX.shape, testX.shape, trainy.shape)
trainy_enc = to_categorical(trainy)
testy_enc = to_categorical(testy)
#print(trainX.shape, testX.shape, trainy.shape)
# define model
model = Sequential()
model.add(Dense(96, input_dim=28*28, activation='relu'))
model.add(Dense(10, activation='softmax'))
opt = SGD(momentum=0.99)  
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
# create snapshot ensemble callback
n_epochs = 400
n_cycles = n_epochs / 50
batch_size = 128
ca = SnapshotEnsemble(n_epochs, n_cycles, 0.001)
# fit model
model.fit(trainX, trainy_enc, batch_size=batch_size, validation_data=(testX, testy_enc), epochs=n_epochs, verbose=0, callbacks=[ca])
# load models in order
members = load_all_models(10)
print('Loaded %d models' % len(members))
# reverse loaded models so we build the ensemble with the last models first
members = list(reversed(members))
# evaluate different numbers of ensembles on hold out set
single_scores, ensemble_scores = list(), list()
for i in range(1, len(members)+1):
	# evaluate model with i members
	ensemble_score = evaluate_n_members(members, i, testX, testy)
	# evaluate the i'th model standalone
	_, single_score = members[i-1].evaluate(testX, testy_enc, verbose=0)
	# summarize this step
	print('> %d: single=%.5f, ensemble=%.5f' % (i, single_score, ensemble_score))
	ensemble_scores.append(ensemble_score)
	single_scores.append(single_score)
# summarize average accuracy of a single final model
print('Accuracy %.5f (%.5f)' % (mean(single_scores), std(single_scores)))
# plot score vs number of ensemble members
x_axis = [i for i in range(1, len(members)+1)]
pyplot.plot(x_axis, single_scores, marker='o', linestyle='None')
pyplot.plot(x_axis, ensemble_scores, marker='x')
pyplot.show()