colin
/
witnn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add UnsuperLearnTrainWeight to train new weight kernel

This commit is contained in:
colin 2019-09-02 19:28:04 +08:00
parent d4a22775d4
commit 6201163e14
10 changed files with 127 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
FilterEvaluator/Evaluator.py
View File

@ -53,32 +53,8 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True)
def ConvKernelToImage(model, layer, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = model.features[layer].weight.data
a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def ConvKernelToImage(numpydate, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
@ -175,7 +151,7 @@ def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
print(" epoch :" + str(e))
return SearchLayer.weight.data.cpu().detach().numpy()
def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=20):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
@ -203,6 +179,6 @@ np.save("WeightTrain.npy", weight)
ConvKernelToImage(weight, CurrentPath+"image")
utils.NumpyToImage(weight, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")

113
FilterEvaluator/EvaluatorEntropy.py
View File

@ -47,16 +47,24 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.MNIST(batchsize)
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True)
def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
@ -92,21 +100,7 @@ def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
dat2 = torch.mean(dat2 * dat2,dim=1)
return dat2.cpu().detach().numpy()
def ConvKernelToImage(model, layer, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = model.features[layer].weight.data
a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
@ -122,7 +116,7 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChanne
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
score = GetSample(model, layer, newlayer, dataloader, Interation)
score = GetScore(model, layer, newlayer, dataloader, Interation)
minactive = np.append(minactive, score)
minweight = np.concatenate((minweight, newweight))
@ -139,16 +133,77 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChanne
return minweight
weight = UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
np.save("weight.npy", weight)
def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
SearchLayer.weight.data.requires_grad=True
optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
for e in range(Epoch):
for batch_idx, (data, target) in enumerate(DataSet):
optimizer.zero_grad()
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
label = dat2*0.5
loss = F.l1_loss(dat2, label)
loss.backward()
optimizer.step()
print(" epoch :" + str(e))
return SearchLayer.weight.data.cpu().detach().numpy()
ConvKernelToImage(model, layer, CurrentPath+"image")
utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=20):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumTrain):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
newweight = TrainLayer(model, layer, newlayer, dataloader, Epoch)
minweight = np.concatenate((minweight, newweight))
print("search :" + str(i))
return minweight
weight = np.load(CurrentPath+"WeightSearch.npy")
# weight = np.zeros((990,3,3));
# weight[:,1]=100
# weight[:,:,1]=100
utils.NumpyToImage(weight,CurrentPath+"image")
a=0
# weight = UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
# np.save("WeightSearch.npy", weight)
weight = UnsuperLearnTrainWeight(model, layer, traindata)
np.save("WeightTrain.npy", weight)
ConvKernelToImage(weight, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")
weight = np.load("weight.npy")

153
FilterEvaluator/EvaluatorGrad.py
View File

@ -1,153 +0,0 @@
from __future__ import print_function
import os
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
import torchvision.models as models
import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import random
import cv2
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
print("Current Path :" + CurrentPath)
sys.path.append(CurrentPath+'../tools')
sys.path.append(CurrentPath+'../')
import Model as Model
from tools import utils, Train, Loader
batchsize = 128
# model = utils.SetDevice(Model.Net5Grad35())
# model = utils.SetDevice(Model.Net31535())
model = utils.SetDevice(Model.Net3Grad335())
# model = utils.SetDevice(Model.Net3())
layers = model.PrintLayer()
layer = 0
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
# traindata, testdata = Loader.MNIST(batchsize)
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
# traindata, testdata = Loader.Cifar10Mono(batchsize)
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
# k = np.ones((3,3)).astype("float32")*1.0
# d = np.ones((3,3)).astype("float32")*3.0
# l = np.ones((3,3)).astype("float32")*3.5
# k = torch.from_numpy(k)
# d = torch.from_numpy(d)
# l = torch.from_numpy(l)
# k.requires_grad=True
# optimizer = optim.SGD([k], lr=0.1)
# optimizer.zero_grad()
# output = k + d
# loss = F.l1_loss(output, l)
# loss.backward()
# optimizer.step()
def ConvKernelToImage(numpydate, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
netmodel.eval()
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
layer = layer-1
SearchLayer.weight.data.requires_grad=True
optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
for e in range(Epoch):
for batch_idx, (data, target) in enumerate(DataSet):
optimizer.zero_grad()
data = utils.SetDevice(data)
target = utils.SetDevice(target)
output = netmodel.ForwardLayer(data,layer)
output = SearchLayer(output)
sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
sample_mean=torch.mean(sample,dim=1,keepdim=True)
dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
dat2 = (sample - sample_mean)/dat1
dat2 = torch.mean(dat2 * dat2,dim=1)
label = dat2*0.5
loss = F.l1_loss(dat2, label)
loss.backward()
optimizer.step()
print(" epoch :" + str(e))
return SearchLayer.weight.data.cpu().detach().numpy()
def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
tl = model.features[layer]
newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
newlayer = utils.SetDevice(newlayer)
newweightshape = list(newlayer.weight.data.shape)
minactive = np.empty((0))
minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
for i in range(NumTrain):
newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
newweight = TrainLayer(model, layer, newlayer, dataloader, 5)
minweight = np.concatenate((minweight, newweight))
ConvKernelToImage(minweight, CurrentPath+"image")
print("search :" + str(i))
return minweight
weight = UnsuperLearnTrainWeight(model, layer, traindata)
np.save("WeightTrain.npy", weight)
# ConvKernelToImage(model, layer, CurrentPath+"image")
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
print("save model sucess")

BIN
FilterEvaluator/WeightTrain.npy Normal file
View File

Binary file not shown.

BIN
FilterEvaluator/image/0-1024.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
FilterEvaluator/image/1024-2048.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
FilterEvaluator/image/2048-3072.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
FilterEvaluator/image/3072-4096.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 11 KiB

40
tools/utils.py
View File

@ -149,3 +149,43 @@ def SetCUDAVISIBLEDEVICES(deviceid):
print("Use GPU IDs :" + str(tuple(deviceid))[1:-1])
def ConvKernelToImage(model, layer, foldname):
if not os.path.exists(foldname):
os.mkdir(foldname)
a2 = model.features[layer].weight.data
a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
for i in range(a2.shape[0]):
d = a2[i]
dmin = np.min(d)
dmax = np.max(d)
d = (d - dmin)*255.0/(dmax-dmin)
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+".png", d)
def NumpyToImage(numpydate, foldname ,maxImageWidth = 128,maxImageHeight = 128):
if not os.path.exists(foldname):
os.mkdir(foldname)
numpydatemin = np.min(numpydate)
numpydatemax = np.max(numpydate)
numpydate = (numpydate - numpydatemin)*255.0/(numpydatemax-numpydatemin)
data = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
datashape = data.shape
newdata = np.ones((datashape[0],datashape[1]+1,datashape[2]+1))*numpydatemin
newdata[:, 0:datashape[1], 0:datashape[2]]=data
datashape = newdata.shape
imagecols = int(maxImageWidth/datashape[2])
imagerows = int(maxImageHeight/datashape[1])
stepimages = imagecols*imagerows
for i in range(0,datashape[0],stepimages):
d = np.zeros((stepimages,datashape[1],datashape[2]))
left = newdata[i:min(i+stepimages,datashape[0])]
d[0:left.shape[0]]=left
d=np.reshape(d, (imagerows, imagecols, datashape[1], datashape[2]))
d=np.swapaxes(d, 1, 2)
d=np.reshape(d, (imagerows*datashape[1],imagecols*datashape[2]))
d = d.astype(int)
cv2.imwrite(foldname+"/"+str(i)+"-"+str(i+stepimages)+".png", d)

0
train Normal file
View File