Refine Distance of kernels
This commit is contained in:
colin
parent
b0246d2a95
commit
7f39f884fc
|
|
@ -1,113 +0,0 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
import torchvision
|
||||
from torchvision import datasets, transforms
|
||||
import torchvision.models as models
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
from PIL import Image
|
||||
import random
|
||||
import cv2
|
||||
|
||||
|
||||
|
||||
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
|
||||
print("Current Path :" + CurrentPath)
|
||||
|
||||
sys.path.append(CurrentPath+'../tools')
|
||||
sys.path.append(CurrentPath+'../')
|
||||
|
||||
import Model as Model
|
||||
from tools import utils, Train, Loader
|
||||
import EvaluatorUnsuper
|
||||
|
||||
|
||||
|
||||
batchsize = 128
|
||||
|
||||
|
||||
# model = utils.SetDevice(Model.Net5Grad35())
|
||||
# model = utils.SetDevice(Model.Net31535())
|
||||
model = utils.SetDevice(Model.Net3Grad335())
|
||||
# model = utils.SetDevice(Model.Net3())
|
||||
|
||||
|
||||
layers = model.PrintLayer()
|
||||
layer = 0
|
||||
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
|
||||
|
||||
weight = np.load(CurrentPath+"WeightSearch.npy")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# traindata, testdata = Loader.MNIST(batchsize)
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
|
||||
# traindata, testdata = Loader.Cifar10Mono(batchsize)
|
||||
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
|
||||
|
||||
|
||||
|
||||
|
||||
# weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
|
||||
# np.save("WeightSearch.npy", weight)
|
||||
|
||||
# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata)
|
||||
# np.save("WeightTrain.npy", weight)
|
||||
|
||||
|
||||
|
||||
|
||||
weight = np.load(CurrentPath+"WeightSearch.npy")
|
||||
|
||||
|
||||
def DistanceOfKernel(k1,k2):
|
||||
k1 = k1.reshape((1,-1))
|
||||
k1r = 1.0/k1.reshape((-1,1))
|
||||
k1dot = np.dot(k1r,k1)
|
||||
k2 = k2.reshape((1,-1))
|
||||
k2r = 1.0/k2.reshape((-1,1))
|
||||
k2dot = np.dot(k2r,k2)
|
||||
diff = np.abs(np.mean(k1dot - k2dot))
|
||||
return diff
|
||||
|
||||
|
||||
indexs = np.random.randint(0,weight.shape[0],(10000,2))
|
||||
|
||||
mindis = 0
|
||||
manindex = []
|
||||
for i in indexs:
|
||||
if i[0] == i[1]:
|
||||
continue
|
||||
dis = DistanceOfKernel(weight[i[0]], weight[i[1]])
|
||||
if dis > mindis:
|
||||
manindex = i
|
||||
mindis = dis
|
||||
|
||||
a = weight[manindex[0]]
|
||||
b = weight[manindex[1]]
|
||||
|
||||
utils.NumpyToImage(a, CurrentPath+"image","a")
|
||||
utils.NumpyToImage(b, CurrentPath+"image","b")
|
||||
|
||||
a = 0
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# utils.NumpyToImage(weight, CurrentPath+"image")
|
||||
# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
|
||||
print("save model sucess")
|
||||
|
|
@ -24,7 +24,7 @@ sys.path.append(CurrentPath+'../tools')
|
|||
sys.path.append(CurrentPath+'../')
|
||||
|
||||
import Model as Model
|
||||
from tools import utils, Train, Loader
|
||||
from tools import utils, Train, Loader, WebVisual
|
||||
import EvaluatorUnsuper
|
||||
|
||||
|
||||
|
|
@ -67,16 +67,32 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
|
|||
|
||||
|
||||
# weight = np.load(CurrentPath+"WeightSearch.npy")
|
||||
# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
|
||||
# bestweight,index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
|
||||
# np.save(CurrentPath+"bestweightSearch.npy", bestweight)
|
||||
# utils.NumpyToImage(bestweight, CurrentPath+"image")
|
||||
|
||||
|
||||
weight = np.load(CurrentPath+"WeightTrain.npy")
|
||||
bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(
|
||||
model, layer, weight, traindata, databatchs=128, interation=100000)
|
||||
np.save(CurrentPath+"bestweightTrain.npy", bestweight)
|
||||
utils.NumpyToImage(bestweight, CurrentPath+"image")
|
||||
# weight = np.load(CurrentPath+"WeightTrain.npy")
|
||||
# bestweight, index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(
|
||||
# model, layer, weight, traindata, databatchs=16, interation=1000000)
|
||||
# np.save(CurrentPath+"bestweightTrain.npy", bestweight)
|
||||
# utils.NumpyToImage(bestweight, CurrentPath+"image")
|
||||
|
||||
|
||||
|
||||
|
||||
# weight = np.load(CurrentPath+"WeightTrain.npy")
|
||||
# WebVisual.InitVisdom()
|
||||
# window = WebVisual.LineWin()
|
||||
# HisLine = WebVisual.Line(window, "HisLine")
|
||||
# dis = EvaluatorUnsuper.CrossDistanceOfKernel(weight,weight,weight.shape[0],weight.shape[0])
|
||||
# his = np.histogram(dis,512)
|
||||
# HisLine.AppendData(his[1],his[0])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -84,9 +100,14 @@ utils.NumpyToImage(bestweight, CurrentPath+"image")
|
|||
# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
# utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
|
||||
|
||||
# weight = np.load(CurrentPath+"bestweightTrain.npy")
|
||||
# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
|
||||
|
||||
weight = np.load(CurrentPath+"bestweightTrain.npy")
|
||||
EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -171,11 +171,60 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1
|
|||
entropy = (histc.log2()*histc).sum()
|
||||
entropys.append(entropy.detach().cpu().numpy())
|
||||
interationbar.update(1)
|
||||
sortindex = np.argsort(entropys)
|
||||
argmin = np.argmin(entropys)
|
||||
bestweight = weight[indexs[argmin]]
|
||||
interationbar.close()
|
||||
return bestweight
|
||||
return bestweight,indexs[sortindex]
|
||||
|
||||
def SetModelConvWeight(model, layer, weight):
|
||||
w = utils.SetDevice(torch.from_numpy(weight))
|
||||
model.features[layer].weight.data = w
|
||||
|
||||
|
||||
def DistanceOfKernel(k1, k2, BatchSize=1):
|
||||
"""
|
||||
Distance of Kernel with N scale value
|
||||
BatchSize : Kernel number in k1 and k2
|
||||
|
||||
scalek1 = (each value of K1 / each value of K1)
|
||||
scalek2 = (each value of K2 / each value of K2)
|
||||
so if N = 9 , size(scaleK1) = 81
|
||||
return diff = abs(mean(scalek1 - scalek2))
|
||||
"""
|
||||
k1revert = k1.reshape((BatchSize, -1, 1))
|
||||
k1 = 1.0 / k1.reshape((BatchSize, 1, -1))
|
||||
k2revert = k2.reshape((BatchSize, -1, 1))
|
||||
k2 = 1.0 / k2.reshape((BatchSize, 1, -1))
|
||||
diff = np.abs(np.mean((np.matmul(k1revert, k1)- np.matmul(k2revert, k2)).reshape(BatchSize, -1), 1))
|
||||
return diff
|
||||
|
||||
|
||||
def CrossDistanceOfKernel(k1, k2, BatchSize1=1, BatchSize2=1):
|
||||
"""
|
||||
Distance of Kernel with N scale value
|
||||
BatchSize : Kernel number in k1 and k2
|
||||
K1 k2 dims must >= 2
|
||||
|
||||
scalek1 = (each value of K1 / each value of K1)
|
||||
scalek2 = (each value of K2 / each value of K2)
|
||||
so if N = 9 , size(scaleK1) = 81
|
||||
return diff = abs(mean(scalek1 - scalek2))
|
||||
"""
|
||||
|
||||
tileshape = np.array(k2.shape)
|
||||
tileshape[1:] = 1
|
||||
tileshape[0] = k1.shape[0]
|
||||
right = np.tile(k2, tileshape)
|
||||
|
||||
tileshape = np.array(k1.shape)
|
||||
tileshape[1] = k2.shape[0]
|
||||
tileshape[0] = 1
|
||||
tileshape[2:] = 1
|
||||
left = np.tile(k1, tileshape)
|
||||
|
||||
leftshape = np.array(k1.shape)
|
||||
leftshape[0] = -1
|
||||
left = left.reshape(leftshape)
|
||||
|
||||
return DistanceOfKernel(right, left, left.shape[0])
|
||||
|
|
@ -14,7 +14,6 @@ from torch.utils.data import Dataset, DataLoader
|
|||
from PIL import Image
|
||||
import random
|
||||
import cv2
|
||||
from visdom import Visdom
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
|
|
@ -45,7 +44,7 @@ batchsize = 128
|
|||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra")
|
||||
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140)
|
||||
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500)
|
||||
|
||||
|
||||
WebVisual.InitVisdom()
|
||||
|
|
@ -65,15 +64,15 @@ Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain)
|
|||
|
||||
|
||||
model = utils.SetDevice(Model.Net3335())
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
|
||||
# model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
|
||||
optimizer = optim.SGD(model.parameters(), lr=0.1)
|
||||
Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre)
|
||||
|
||||
|
||||
model = utils.SetDevice(Model.Net3Grad335())
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
|
||||
optimizer = optim.SGD(model.parameters(), lr=0.1)
|
||||
Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
|
||||
# model = utils.SetDevice(Model.Net3Grad335())
|
||||
# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
|
||||
# optimizer = optim.SGD(model.parameters(), lr=0.1)
|
||||
# Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 413 B |
Binary file not shown.
|
|
@ -39,6 +39,10 @@ class Line():
|
|||
def AppendData(self, y):
|
||||
self.win.AppendData(self, y)
|
||||
|
||||
def AppendData(self, x, y):
|
||||
self.win.AppendData(self, x, y)
|
||||
|
||||
|
||||
class LineWin():
|
||||
def __init__(self):
|
||||
super(LineWin, self).__init__()
|
||||
|
|
@ -77,3 +81,20 @@ class LineWin():
|
|||
name=line.name
|
||||
)
|
||||
line.size = line.size + len(y)
|
||||
|
||||
def AppendData(self, line, x, y):
|
||||
if not isinstance(y, list) and not isinstance(y, np.ndarray):
|
||||
y = [y]
|
||||
if not isinstance(x, list) and not isinstance(x, np.ndarray):
|
||||
x = [x]
|
||||
X = np.array(x)
|
||||
Y = np.array(y)
|
||||
X = X[0:len(Y)]
|
||||
self.viz.line(
|
||||
Y=Y,
|
||||
X=X,
|
||||
win=self.win,
|
||||
update="append",
|
||||
name=line.name
|
||||
)
|
||||
line.size = line.size + len(y)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
nohup: ignoring input
|
||||
/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops.
|
||||
ioloop.install() # Needs to happen before any tornado imports!
|
||||
Checking for scripts.
|
||||
It's Alive!
|
||||
Traceback (most recent call last):
|
||||
File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main
|
||||
"__main__", mod_spec)
|
||||
|
|
@ -21,5 +22,3 @@ Traceback (most recent call last):
|
|||
File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets
|
||||
sock.bind(sockaddr)
|
||||
OSError: [Errno 98] Address already in use
|
||||
Checking for scripts.
|
||||
It's Alive!
|
||||
|
|
|
|||
Loading…
Reference in New Issue