diff --git a/FilterEvaluator/Evaluator copy.py b/FilterEvaluator/Evaluator copy.py deleted file mode 100644 index a25d007..0000000 --- a/FilterEvaluator/Evaluator copy.py +++ /dev/null @@ -1,113 +0,0 @@ -from __future__ import print_function -import os -import sys -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -import torchvision -from torchvision import datasets, transforms -import torchvision.models as models -import matplotlib.pyplot as plt -import numpy as np -from torch.utils.data import Dataset, DataLoader -from PIL import Image -import random -import cv2 - - - -CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" -print("Current Path :" + CurrentPath) - -sys.path.append(CurrentPath+'../tools') -sys.path.append(CurrentPath+'../') - -import Model as Model -from tools import utils, Train, Loader -import EvaluatorUnsuper - - - -batchsize = 128 - - -# model = utils.SetDevice(Model.Net5Grad35()) -# model = utils.SetDevice(Model.Net31535()) -model = utils.SetDevice(Model.Net3Grad335()) -# model = utils.SetDevice(Model.Net3()) - - -layers = model.PrintLayer() -layer = 0 - -model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") - -weight = np.load(CurrentPath+"WeightSearch.npy") - - - - - -# traindata, testdata = Loader.MNIST(batchsize) -# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical") -# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal") -# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine") -# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra") -# traindata, testdata = Loader.Cifar10Mono(batchsize) -traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) - - - - -# weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10) -# np.save("WeightSearch.npy", weight) - -# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata) -# np.save("WeightTrain.npy", weight) - - - - -weight = np.load(CurrentPath+"WeightSearch.npy") - - -def DistanceOfKernel(k1,k2): - k1 = k1.reshape((1,-1)) - k1r = 1.0/k1.reshape((-1,1)) - k1dot = np.dot(k1r,k1) - k2 = k2.reshape((1,-1)) - k2r = 1.0/k2.reshape((-1,1)) - k2dot = np.dot(k2r,k2) - diff = np.abs(np.mean(k1dot - k2dot)) - return diff - - -indexs = np.random.randint(0,weight.shape[0],(10000,2)) - -mindis = 0 -manindex = [] -for i in indexs: - if i[0] == i[1]: - continue - dis = DistanceOfKernel(weight[i[0]], weight[i[1]]) - if dis > mindis: - manindex = i - mindis = dis - -a = weight[manindex[0]] -b = weight[manindex[1]] - -utils.NumpyToImage(a, CurrentPath+"image","a") -utils.NumpyToImage(b, CurrentPath+"image","b") - -a = 0 - - - - - - -# utils.NumpyToImage(weight, CurrentPath+"image") -# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl") -print("save model sucess") diff --git a/FilterEvaluator/Evaluator.py b/FilterEvaluator/Evaluator.py index b4e5ee6..a93c1c8 100644 --- a/FilterEvaluator/Evaluator.py +++ b/FilterEvaluator/Evaluator.py @@ -24,7 +24,7 @@ sys.path.append(CurrentPath+'../tools') sys.path.append(CurrentPath+'../') import Model as Model -from tools import utils, Train, Loader +from tools import utils, Train, Loader, WebVisual import EvaluatorUnsuper @@ -67,16 +67,32 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) # weight = np.load(CurrentPath+"WeightSearch.npy") -# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000) +# bestweight,index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000) # np.save(CurrentPath+"bestweightSearch.npy", bestweight) # utils.NumpyToImage(bestweight, CurrentPath+"image") -weight = np.load(CurrentPath+"WeightTrain.npy") -bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight( - model, layer, weight, traindata, databatchs=128, interation=100000) -np.save(CurrentPath+"bestweightTrain.npy", bestweight) -utils.NumpyToImage(bestweight, CurrentPath+"image") +# weight = np.load(CurrentPath+"WeightTrain.npy") +# bestweight, index = EvaluatorUnsuper.UnsuperLearnFindBestWeight( +# model, layer, weight, traindata, databatchs=16, interation=1000000) +# np.save(CurrentPath+"bestweightTrain.npy", bestweight) +# utils.NumpyToImage(bestweight, CurrentPath+"image") + + + + +# weight = np.load(CurrentPath+"WeightTrain.npy") +# WebVisual.InitVisdom() +# window = WebVisual.LineWin() +# HisLine = WebVisual.Line(window, "HisLine") +# dis = EvaluatorUnsuper.CrossDistanceOfKernel(weight,weight,weight.shape[0],weight.shape[0]) +# his = np.histogram(dis,512) +# HisLine.AppendData(his[1],his[0]) + + + + + @@ -84,9 +100,14 @@ utils.NumpyToImage(bestweight, CurrentPath+"image") # EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) # utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl") -# weight = np.load(CurrentPath+"bestweightTrain.npy") -# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) -# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl") + +weight = np.load(CurrentPath+"bestweightTrain.npy") +EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) +utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl") + + + + diff --git a/FilterEvaluator/EvaluatorUnsuper.py b/FilterEvaluator/EvaluatorUnsuper.py index a25bcfa..83a4f1a 100644 --- a/FilterEvaluator/EvaluatorUnsuper.py +++ b/FilterEvaluator/EvaluatorUnsuper.py @@ -171,11 +171,60 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1 entropy = (histc.log2()*histc).sum() entropys.append(entropy.detach().cpu().numpy()) interationbar.update(1) + sortindex = np.argsort(entropys) argmin = np.argmin(entropys) bestweight = weight[indexs[argmin]] interationbar.close() - return bestweight + return bestweight,indexs[sortindex] def SetModelConvWeight(model, layer, weight): w = utils.SetDevice(torch.from_numpy(weight)) model.features[layer].weight.data = w + + +def DistanceOfKernel(k1, k2, BatchSize=1): + """ + Distance of Kernel with N scale value + BatchSize : Kernel number in k1 and k2 + + scalek1 = (each value of K1 / each value of K1) + scalek2 = (each value of K2 / each value of K2) + so if N = 9 , size(scaleK1) = 81 + return diff = abs(mean(scalek1 - scalek2)) + """ + k1revert = k1.reshape((BatchSize, -1, 1)) + k1 = 1.0 / k1.reshape((BatchSize, 1, -1)) + k2revert = k2.reshape((BatchSize, -1, 1)) + k2 = 1.0 / k2.reshape((BatchSize, 1, -1)) + diff = np.abs(np.mean((np.matmul(k1revert, k1)- np.matmul(k2revert, k2)).reshape(BatchSize, -1), 1)) + return diff + + +def CrossDistanceOfKernel(k1, k2, BatchSize1=1, BatchSize2=1): + """ + Distance of Kernel with N scale value + BatchSize : Kernel number in k1 and k2 + K1 k2 dims must >= 2 + + scalek1 = (each value of K1 / each value of K1) + scalek2 = (each value of K2 / each value of K2) + so if N = 9 , size(scaleK1) = 81 + return diff = abs(mean(scalek1 - scalek2)) + """ + + tileshape = np.array(k2.shape) + tileshape[1:] = 1 + tileshape[0] = k1.shape[0] + right = np.tile(k2, tileshape) + + tileshape = np.array(k1.shape) + tileshape[1] = k2.shape[0] + tileshape[0] = 1 + tileshape[2:] = 1 + left = np.tile(k1, tileshape) + + leftshape = np.array(k1.shape) + leftshape[0] = -1 + left = left.reshape(leftshape) + + return DistanceOfKernel(right, left, left.shape[0]) \ No newline at end of file diff --git a/FilterEvaluator/TrainNetwork.py b/FilterEvaluator/TrainNetwork.py index d50903a..884589b 100644 --- a/FilterEvaluator/TrainNetwork.py +++ b/FilterEvaluator/TrainNetwork.py @@ -14,7 +14,6 @@ from torch.utils.data import Dataset, DataLoader from PIL import Image import random import cv2 -from visdom import Visdom from tqdm import tqdm @@ -45,7 +44,7 @@ batchsize = 128 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra") -traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140) +traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500) WebVisual.InitVisdom() @@ -65,15 +64,15 @@ Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain) model = utils.SetDevice(Model.Net3335()) -model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl") +# model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl") optimizer = optim.SGD(model.parameters(), lr=0.1) Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre) -model = utils.SetDevice(Model.Net3Grad335()) -model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl") -optimizer = optim.SGD(model.parameters(), lr=0.1) -Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch) +# model = utils.SetDevice(Model.Net3Grad335()) +# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl") +# optimizer = optim.SGD(model.parameters(), lr=0.1) +# Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch) diff --git a/FilterEvaluator/bestweightTrain.npy b/FilterEvaluator/bestweightTrain.npy index c1aa5ac..d842fac 100644 Binary files a/FilterEvaluator/bestweightTrain.npy and b/FilterEvaluator/bestweightTrain.npy differ diff --git a/FilterEvaluator/checkpointTrain.pkl b/FilterEvaluator/checkpointTrain.pkl index 7bdf15b..cadd6c3 100644 Binary files a/FilterEvaluator/checkpointTrain.pkl and b/FilterEvaluator/checkpointTrain.pkl differ diff --git a/FilterEvaluator/image/0-1024.png b/FilterEvaluator/image/0-1024.png new file mode 100644 index 0000000..73aede3 Binary files /dev/null and b/FilterEvaluator/image/0-1024.png differ diff --git a/FilterEvaluator/visdom.server.log b/FilterEvaluator/visdom.server.log index 35d93f2..8457fc6 100644 Binary files a/FilterEvaluator/visdom.server.log and b/FilterEvaluator/visdom.server.log differ diff --git a/tools/WebVisual.py b/tools/WebVisual.py index 7ca9870..1e00b4b 100644 --- a/tools/WebVisual.py +++ b/tools/WebVisual.py @@ -39,6 +39,10 @@ class Line(): def AppendData(self, y): self.win.AppendData(self, y) + def AppendData(self, x, y): + self.win.AppendData(self, x, y) + + class LineWin(): def __init__(self): super(LineWin, self).__init__() @@ -77,3 +81,20 @@ class LineWin(): name=line.name ) line.size = line.size + len(y) + + def AppendData(self, line, x, y): + if not isinstance(y, list) and not isinstance(y, np.ndarray): + y = [y] + if not isinstance(x, list) and not isinstance(x, np.ndarray): + x = [x] + X = np.array(x) + Y = np.array(y) + X = X[0:len(Y)] + self.viz.line( + Y=Y, + X=X, + win=self.win, + update="append", + name=line.name + ) + line.size = line.size + len(y) diff --git a/visdom.server.log b/visdom.server.log index 9bc7cb7..6d80f5d 100644 --- a/visdom.server.log +++ b/visdom.server.log @@ -1,6 +1,7 @@ -nohup: ignoring input /home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops. ioloop.install() # Needs to happen before any tornado imports! +Checking for scripts. +It's Alive! Traceback (most recent call last): File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) @@ -21,5 +22,3 @@ Traceback (most recent call last): File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets sock.bind(sockaddr) OSError: [Errno 98] Address already in use -Checking for scripts. -It's Alive!