Add UnsuperLearnFindBestWeight to find best from 4000

This commit is contained in:
colin 2019-09-13 21:03:38 +08:00
parent 6201163e14
commit f858ae1821
22 changed files with 199 additions and 199 deletions

View File

@ -0,0 +1,113 @@
from __future__ import print_function
import os
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
import torchvision.models as models
import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import random
import cv2
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
print("Current Path :" + CurrentPath)
sys.path.append(CurrentPath+'../tools')
sys.path.append(CurrentPath+'../')
import Model as Model
from tools import utils, Train, Loader
import EvaluatorUnsuper
batchsize = 128
# model = utils.SetDevice(Model.Net5Grad35())
# model = utils.SetDevice(Model.Net31535())
model = utils.SetDevice(Model.Net3Grad335())
# model = utils.SetDevice(Model.Net3())
layers = model.PrintLayer()
layer = 0
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
weight = np.load(CurrentPath+"WeightSearch.npy")
# traindata, testdata = Loader.MNIST(batchsize)
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
# weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
# np.save("WeightSearch.npy", weight)
# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata)
# np.save("WeightTrain.npy", weight)
weight = np.load(CurrentPath+"WeightSearch.npy")
def DistanceOfKernel(k1,k2):
k1 = k1.reshape((1,-1))
k1r = 1.0/k1.reshape((-1,1))
k1dot = np.dot(k1r,k1)
k2 = k2.reshape((1,-1))
k2r = 1.0/k2.reshape((-1,1))
k2dot = np.dot(k2r,k2)
diff = np.abs(np.mean(k1dot - k2dot))
return diff
indexs = np.random.randint(0,weight.shape[0],(10000,2))
mindis = 0
manindex = []
for i in indexs:
if i[0] == i[1]:
continue
dis = DistanceOfKernel(weight[i[0]], weight[i[1]])
if dis > mindis:
manindex = i
mindis = dis
a = weight[manindex[0]]
b = weight[manindex[1]]
utils.NumpyToImage(a, CurrentPath+"image","a")
utils.NumpyToImage(b, CurrentPath+"image","b")
a = 0
# utils.NumpyToImage(weight, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")

View File

@ -1,10 +1,6 @@
from __future__ import print_function from __future__ import print_function
import os import os
import sys import sys
# import multiprocessing
# multiprocessing.set_start_method('spawn', True)
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -20,6 +16,7 @@ import random
import cv2 import cv2
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
print("Current Path :" + CurrentPath) print("Current Path :" + CurrentPath)
@ -28,6 +25,9 @@ sys.path.append(CurrentPath+'../')
import Model as Model import Model as Model
from tools import utils, Train, Loader from tools import utils, Train, Loader
import EvaluatorUnsuper
batchsize = 128 batchsize = 128
@ -40,8 +40,7 @@ model = utils.SetDevice(Model.Net3Grad335())
layers = model.PrintLayer() layers = model.PrintLayer()
layer = 0 layer = 0
# model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
@ -53,132 +52,43 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine") # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra") # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize) # traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True) traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
# layerout = []
# layerint = []
# def getnet(self, input, output):
# layerout.append(output)
# layerint.append(input)
# handle = netmodel.features[layer].register_forward_hook(getnet)
# netmodel.ForwardLayer(data,layer)
# output = layerout[0][:,:,:,:]
# handle.remove()
for batch_idx, (data, target) in enumerate(DataSet):
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
data.detach()
target.detach()
output = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample = torch.cat((sample,output),1)
if Interation > 0 and batch_idx >= (Interation-1):
break
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
return dat2.cpu().detach().numpy()
def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newchannels = tl.out_channels * SaveChannelRatio
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumSearch):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
score = GetScore(model, layer, newlayer, dataloader, Interation)
minactive = np.append(minactive, score)
minweight = np.concatenate((minweight, newweight))
index = minactive.argsort()
minactive = minactive[index[0:newchannels]]
minweight = minweight[index[0:newchannels]]
print("search random :" + str(i))
if i % (NumSearch/10) == 0:
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
return minweight
def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
SearchLayer.weight.data.requires_grad=True
optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
for e in range(Epoch):
for batch_idx, (data, target) in enumerate(DataSet):
optimizer.zero_grad()
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
label = dat2*0.5
loss = F.l1_loss(dat2, label)
loss.backward()
optimizer.step()
print(" epoch :" + str(e))
return SearchLayer.weight.data.cpu().detach().numpy()
def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=20):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumTrain):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
newweight = TrainLayer(model, layer, newlayer, dataloader, Epoch)
minweight = np.concatenate((minweight, newweight))
print("search :" + str(i))
return minweight
# weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
# weight = UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
# np.save("WeightSearch.npy", weight) # np.save("WeightSearch.npy", weight)
weight = UnsuperLearnTrainWeight(model, layer, traindata) # weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata)
np.save("WeightTrain.npy", weight) # np.save("WeightTrain.npy", weight)
utils.NumpyToImage(weight, CurrentPath+"image")
# weight = np.load(CurrentPath+"WeightSearch.npy")
# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
# np.save(CurrentPath+"bestweightSearch.npy", bestweight)
# utils.NumpyToImage(bestweight, CurrentPath+"image")
# weight = np.load(CurrentPath+"WeightTrain.npy")
# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
# np.save(CurrentPath+"bestweightTrain.npy", bestweight)
# utils.NumpyToImage(bestweight, CurrentPath+"image")
weight = np.load(CurrentPath+"bestweightSearch.npy")
EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
weight = np.load(CurrentPath+"bestweightTrain.npy")
EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
# utils.NumpyToImage(weight, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl") # utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess") print("save model sucess")

View File

@ -1,10 +1,6 @@
from __future__ import print_function from __future__ import print_function
import os import os
import sys import sys
# import multiprocessing
# multiprocessing.set_start_method('spawn', True)
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -19,50 +15,13 @@ from PIL import Image
import random import random
import cv2 import cv2
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
print("Current Path :" + CurrentPath)
sys.path.append(CurrentPath+'../tools') sys.path.append(CurrentPath+'../tools')
sys.path.append(CurrentPath+'../') sys.path.append(CurrentPath+'../')
import Model as Model import Model as Model
from tools import utils, Train, Loader from tools import utils, Train, Loader
batchsize = 128
# model = utils.SetDevice(Model.Net5Grad35())
# model = utils.SetDevice(Model.Net31535())
model = utils.SetDevice(Model.Net3Grad335())
# model = utils.SetDevice(Model.Net3())
layers = model.PrintLayer()
layer = 0
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.MNIST(batchsize)
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True)
def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1): def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
netmodel.eval() netmodel.eval()
@ -175,35 +134,49 @@ def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannel
return minweight return minweight
def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=128,interation=10000):
weight = weight.astype("float32")
netmodel.eval()
tl = netmodel.features[layer]
outchannels = tl.out_channels
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
datas = []
for batch_idx, (data, target) in enumerate(dataloader):
datas.append(utils.SetDevice(data))
if batch_idx >= databatchs-1:
break
indexs = np.random.randint(0,weight.shape[0],interation*outchannels).reshape(interation,-1)
entropys = []
forwardlayer = layer -1
shift = []
for i in range(outchannels):
shift.append(1<<i)
shift = utils.SetDevice(torch.from_numpy(np.array(shift).astype("uint8")))
for i in range(len(indexs)):
newlayer.weight.data=utils.SetDevice(torch.from_numpy(weight[indexs[i]]))
outputs = []
for d in datas:
outputs.append(newlayer(netmodel.ForwardLayer(d,forwardlayer)))
outputs = torch.cat(outputs).transpose(0,1)
outputs = outputs.reshape(outputs.shape[0],-1)
mean = outputs.mean(1)
outputs = outputs.transpose(0,1)
posi = outputs > mean
posi = posi*shift
sums = posi.sum(1).type(torch.float32)
histc = sums.histc(256,0,255).type(torch.float32)
histc = histc[histc>0]
histc = histc/histc.sum()
entropys.append((histc.log2()*histc).sum())
print("search index : " + str(i))
argmin = np.argmin(entropys)
bestweight = weight[indexs[argmin]]
return bestweight
def SetModelConvWeight(model, layer, weight):
w = utils.SetDevice(torch.from_numpy(weight))
model.features[layer].weight.data = w
weight = np.load(CurrentPath+"WeightSearch.npy")
# weight = np.zeros((990,3,3));
# weight[:,1]=100
# weight[:,:,1]=100
utils.NumpyToImage(weight,CurrentPath+"image")
a=0
# weight = UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
# np.save("WeightSearch.npy", weight)
weight = UnsuperLearnTrainWeight(model, layer, traindata)
np.save("WeightTrain.npy", weight)
ConvKernelToImage(weight, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")

View File

@ -66,6 +66,8 @@ for i in range(1000):
window.AppendData(linePretrain,Train.test(model,testdata)) window.AppendData(linePretrain,Train.test(model,testdata))
model = utils.SetDevice(Model.Net3335()) model = utils.SetDevice(Model.Net3335())
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 273 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 273 B

View File

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

View File

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

View File

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

View File

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

View File

@ -9,6 +9,7 @@ from easydict import EasyDict as edict
import yaml import yaml
import numpy as np import numpy as np
import argparse import argparse
import cv2
class AverageMeter(object): class AverageMeter(object):
""" Computes ans stores the average and current value""" """ Computes ans stores the average and current value"""
@ -103,7 +104,7 @@ def SaveModel(model , filename='checkpoint_'+str(time.time()) + '.pkl'):
# 'loss': loss, # 'loss': loss,
}, filename) }, filename)
def LoadModel(model, filename): def LoadModel(model, filename):
checkpoint = torch.load(filename) checkpoint = torch.load(filename, map_location="cpu")
model.load_state_dict(checkpoint['model_state_dict']) model.load_state_dict(checkpoint['model_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
# epoch = checkpoint['epoch'] # epoch = checkpoint['epoch']
@ -164,7 +165,8 @@ def ConvKernelToImage(model, layer, foldname):
d = d.astype(int) d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d) cv2.imwrite(foldname+"/"+str(i)+".png", d)
def NumpyToImage(numpydate, foldname ,maxImageWidth = 128,maxImageHeight = 128):
def NumpyToImage(numpydate, foldname, title="", maxImageWidth=128, maxImageHeight=128):
if not os.path.exists(foldname): if not os.path.exists(foldname):
os.mkdir(foldname) os.mkdir(foldname)
numpydatemin = np.min(numpydate) numpydatemin = np.min(numpydate)
@ -187,5 +189,5 @@ def NumpyToImage(numpydate, foldname ,maxImageWidth = 128,maxImageHeight = 128):
d=np.reshape(d, (imagerows, imagecols, datashape[1], datashape[2])) d=np.reshape(d, (imagerows, imagecols, datashape[1], datashape[2]))
d=np.swapaxes(d, 1, 2) d=np.swapaxes(d, 1, 2)
d=np.reshape(d, (imagerows*datashape[1],imagecols*datashape[2])) d=np.reshape(d, (imagerows*datashape[1],imagecols*datashape[2]))
d = d.astype(int) d = d.astype("uint8")
cv2.imwrite(foldname+"/"+str(i)+"-"+str(i+stepimages)+".png", d) cv2.imwrite(foldname+"/"+title+str(i)+"-"+str(i+stepimages)+".png", d)