Add two method of real wit kernel genarator.

1 get 8 from 4000 with entropy calculation.
2 Auto Grad to regrethion kernel.
This commit is contained in:
colin 2019-09-01 12:21:22 +08:00
parent 5220bf6885
commit d4a22775d4
21 changed files with 393 additions and 30 deletions

View File

@ -55,8 +55,33 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.Cifar10Mono(batchsize) # traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
def ConvKernelToImage(model, layer, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = model.features[layer].weight.data
a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1): def ConvKernelToImage(numpydate, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
netmodel.eval() netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0))) sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
@ -92,35 +117,13 @@ def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
dat2 = torch.mean(dat2 * dat2,dim=1) dat2 = torch.mean(dat2 * dat2,dim=1)
return dat2.cpu().detach().numpy() return dat2.cpu().detach().numpy()
def ConvKernelToImage(model, layer, foldname): def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = model.features[layer].weight.data
a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def GetRandomSocre(netmodel,layer,dataloader,iteration=-1):
weightshape = netmodel.features[layer].weight.data.shape
newweight = np.random.uniform(-1.0,1.0,weightshape).astype("float32")
netmodel.features[layer].weight.data=utils.SetDevice(torch.from_numpy(newweight))
score = GetSample(netmodel,0,dataloader,iteration)
return np.array(score), newweight
def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelRatio=1,Interation=10):
tl = model.features[layer] tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * ChannelRatio, tl.kernel_size, newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode) tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer) newlayer = utils.SetDevice(newlayer)
newchannels = tl.out_channels * ChannelRatio newchannels = tl.out_channels * SaveChannelRatio
newweightshape = list(newlayer.weight.data.shape) newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0)) minactive = np.empty((0))
@ -130,7 +133,7 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelR
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32") newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight)) newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
score = GetSample(model, layer, newlayer, dataloader, Interation) score = GetScore(model, layer, newlayer, dataloader, Interation)
minactive = np.append(minactive, score) minactive = np.append(minactive, score)
minweight = np.concatenate((minweight, newweight)) minweight = np.concatenate((minweight, newweight))
@ -142,11 +145,64 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelR
if i % (NumSearch/10) == 0: if i % (NumSearch/10) == 0:
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels])) tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
utils.SaveModel(model, CurrentPath+"/checkpoint.pkl") utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels])) tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
return minweight
UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, ChannelRatio=8, Interation=10) def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
SearchLayer.weight.data.requires_grad=True
optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
for e in range(Epoch):
for batch_idx, (data, target) in enumerate(DataSet):
optimizer.zero_grad()
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
label = dat2*0.5
loss = F.l1_loss(dat2, label)
loss.backward()
optimizer.step()
print(" epoch :" + str(e))
return SearchLayer.weight.data.cpu().detach().numpy()
ConvKernelToImage(model, layer, CurrentPath+"image") def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
utils.SaveModel(model,CurrentPath+"/checkpoint.pkl") tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumTrain):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
newweight = TrainLayer(model, layer, newlayer, dataloader, Epoch)
minweight = np.concatenate((minweight, newweight))
print("search :" + str(i))
return minweight
# weight = UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
# np.save("WeightSearch.npy", weight)
weight = UnsuperLearnTrainWeight(model, layer, traindata)
np.save("WeightTrain.npy", weight)
ConvKernelToImage(weight, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess") print("save model sucess")

View File

@ -0,0 +1,154 @@
from __future__ import print_function
import os
import sys
# import multiprocessing
# multiprocessing.set_start_method('spawn', True)
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
import torchvision.models as models
import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import random
import cv2
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
print("Current Path :" + CurrentPath)
sys.path.append(CurrentPath+'../tools')
sys.path.append(CurrentPath+'../')
import Model as Model
from tools import utils, Train, Loader
batchsize = 128
# model = utils.SetDevice(Model.Net5Grad35())
# model = utils.SetDevice(Model.Net31535())
model = utils.SetDevice(Model.Net3Grad335())
# model = utils.SetDevice(Model.Net3())
layers = model.PrintLayer()
layer = 0
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.MNIST(batchsize)
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
# layerout = []
# layerint = []
# def getnet(self, input, output):
# layerout.append(output)
# layerint.append(input)
# handle = netmodel.features[layer].register_forward_hook(getnet)
# netmodel.ForwardLayer(data,layer)
# output = layerout[0][:,:,:,:]
# handle.remove()
for batch_idx, (data, target) in enumerate(DataSet):
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
data.detach()
target.detach()
output = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample = torch.cat((sample,output),1)
if Interation > 0 and batch_idx >= (Interation-1):
break
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
return dat2.cpu().detach().numpy()
def ConvKernelToImage(model, layer, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = model.features[layer].weight.data
a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newchannels = tl.out_channels * SaveChannelRatio
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumSearch):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
score = GetSample(model, layer, newlayer, dataloader, Interation)
minactive = np.append(minactive, score)
minweight = np.concatenate((minweight, newweight))
index = minactive.argsort()
minactive = minactive[index[0:newchannels]]
minweight = minweight[index[0:newchannels]]
print("search random :" + str(i))
if i % (NumSearch/10) == 0:
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
return minweight
weight = UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
np.save("weight.npy", weight)
ConvKernelToImage(model, layer, CurrentPath+"image")
utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")
weight = np.load("weight.npy")

View File

@ -0,0 +1,153 @@
from __future__ import print_function
import os
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
import torchvision.models as models
import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import random
import cv2
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
print("Current Path :" + CurrentPath)
sys.path.append(CurrentPath+'../tools')
sys.path.append(CurrentPath+'../')
import Model as Model
from tools import utils, Train, Loader
batchsize = 128
# model = utils.SetDevice(Model.Net5Grad35())
# model = utils.SetDevice(Model.Net31535())
model = utils.SetDevice(Model.Net3Grad335())
# model = utils.SetDevice(Model.Net3())
layers = model.PrintLayer()
layer = 0
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.MNIST(batchsize)
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
# k = np.ones((3,3)).astype("float32")*1.0
# d = np.ones((3,3)).astype("float32")*3.0
# l = np.ones((3,3)).astype("float32")*3.5
# k = torch.from_numpy(k)
# d = torch.from_numpy(d)
# l = torch.from_numpy(l)
# k.requires_grad=True
# optimizer = optim.SGD([k], lr=0.1)
# optimizer.zero_grad()
# output = k + d
# loss = F.l1_loss(output, l)
# loss.backward()
# optimizer.step()
def ConvKernelToImage(numpydate, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
SearchLayer.weight.data.requires_grad=True
optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
for e in range(Epoch):
for batch_idx, (data, target) in enumerate(DataSet):
optimizer.zero_grad()
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
label = dat2*0.5
loss = F.l1_loss(dat2, label)
loss.backward()
optimizer.step()
print(" epoch :" + str(e))
return SearchLayer.weight.data.cpu().detach().numpy()
def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumTrain):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
newweight = TrainLayer(model, layer, newlayer, dataloader, 5)
minweight = np.concatenate((minweight, newweight))
ConvKernelToImage(minweight, CurrentPath+"image")
print("search :" + str(i))
return minweight
weight = UnsuperLearnTrainWeight(model, layer, traindata)
np.save("WeightTrain.npy", weight)
# ConvKernelToImage(model, layer, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")

Binary file not shown.

Binary file not shown.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 B