Refine Distance of kernels

FilterEvaluator/Evaluator copy.py

@@ -1,113 +0,0 @@
-from __future__ import print_function
-import os
-import sys
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-import torch.optim as optim
-import torchvision
-from torchvision import datasets, transforms
-import torchvision.models as models
-import matplotlib.pyplot as plt
-import numpy as np
-from torch.utils.data import Dataset, DataLoader
-from PIL import Image
-import random
-import cv2
-
-
-
-CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
-print("Current Path :" + CurrentPath)
-
-sys.path.append(CurrentPath+'../tools')
-sys.path.append(CurrentPath+'../')
-
-import Model as Model
-from tools import utils, Train, Loader
-import EvaluatorUnsuper
-
-
-
-batchsize = 128
-
-
-# model = utils.SetDevice(Model.Net5Grad35())
-# model = utils.SetDevice(Model.Net31535())
-model = utils.SetDevice(Model.Net3Grad335())
-# model = utils.SetDevice(Model.Net3())
-
-
-layers = model.PrintLayer()
-layer = 0
-
-model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
-
-weight = np.load(CurrentPath+"WeightSearch.npy")
-
-
-
-
-
-# traindata, testdata = Loader.MNIST(batchsize)
-# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
-# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
-# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
-# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
-# traindata, testdata = Loader.Cifar10Mono(batchsize)
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
-
-
-
-
-# weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
-# np.save("WeightSearch.npy", weight)
-
-# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata)
-# np.save("WeightTrain.npy", weight)
-
-
-
-
-weight = np.load(CurrentPath+"WeightSearch.npy")
-
-
-def DistanceOfKernel(k1,k2):
-    k1 = k1.reshape((1,-1))
-    k1r = 1.0/k1.reshape((-1,1))
-    k1dot = np.dot(k1r,k1)
-    k2 = k2.reshape((1,-1))
-    k2r = 1.0/k2.reshape((-1,1))
-    k2dot = np.dot(k2r,k2)
-    diff = np.abs(np.mean(k1dot - k2dot))
-    return diff
-
-
-indexs = np.random.randint(0,weight.shape[0],(10000,2))
-
-mindis = 0
-manindex = []
-for i in indexs:
-    if i[0] == i[1]:
-      continue
-    dis = DistanceOfKernel(weight[i[0]], weight[i[1]])
-    if dis > mindis: 
-      manindex = i
-      mindis = dis
-
-a = weight[manindex[0]]
-b = weight[manindex[1]]
-
-utils.NumpyToImage(a, CurrentPath+"image","a")
-utils.NumpyToImage(b, CurrentPath+"image","b")
-
-a = 0
-
-
-
-
-
-
-# utils.NumpyToImage(weight, CurrentPath+"image")
-# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
-print("save model sucess")

FilterEvaluator/Evaluator.py

@@ -24,7 +24,7 @@ sys.path.append(CurrentPath+'../tools')
 sys.path.append(CurrentPath+'../')
 
 import Model as Model
-from tools import utils, Train, Loader
+from tools import utils, Train, Loader, WebVisual
 import EvaluatorUnsuper
 
 
@@ -67,16 +67,32 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
 
 
 # weight = np.load(CurrentPath+"WeightSearch.npy")
-# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
+# bestweight,index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
 # np.save(CurrentPath+"bestweightSearch.npy", bestweight)
 # utils.NumpyToImage(bestweight, CurrentPath+"image")
 
 
-weight = np.load(CurrentPath+"WeightTrain.npy")
+# weight = np.load(CurrentPath+"WeightTrain.npy")
-bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(
+# bestweight, index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(
-    model, layer, weight, traindata, databatchs=128, interation=100000)
+#     model, layer, weight, traindata, databatchs=16, interation=1000000)
-np.save(CurrentPath+"bestweightTrain.npy", bestweight)
+# np.save(CurrentPath+"bestweightTrain.npy", bestweight)
-utils.NumpyToImage(bestweight, CurrentPath+"image")
+# utils.NumpyToImage(bestweight, CurrentPath+"image")
+
+
+
+
+# weight = np.load(CurrentPath+"WeightTrain.npy")
+# WebVisual.InitVisdom()
+# window = WebVisual.LineWin()
+# HisLine = WebVisual.Line(window, "HisLine")
+# dis = EvaluatorUnsuper.CrossDistanceOfKernel(weight,weight,weight.shape[0],weight.shape[0])
+# his = np.histogram(dis,512)
+# HisLine.AppendData(his[1],his[0])
+
+
+
+
+
 
 
 
@@ -84,9 +100,14 @@ utils.NumpyToImage(bestweight, CurrentPath+"image")
 # EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
 # utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
 
-# weight = np.load(CurrentPath+"bestweightTrain.npy")
+
-# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
+weight = np.load(CurrentPath+"bestweightTrain.npy")
-# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
+EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
+utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
+
+
+
+
 
 
 

FilterEvaluator/EvaluatorUnsuper.py

@@ -171,11 +171,60 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1
         entropy = (histc.log2()*histc).sum()
         entropys.append(entropy.detach().cpu().numpy())
         interationbar.update(1)
+    sortindex = np.argsort(entropys)
     argmin = np.argmin(entropys)
     bestweight = weight[indexs[argmin]]
     interationbar.close()
-    return bestweight
+    return bestweight,indexs[sortindex]
 
 def SetModelConvWeight(model, layer, weight):
     w = utils.SetDevice(torch.from_numpy(weight))
     model.features[layer].weight.data = w
+
+
+def DistanceOfKernel(k1, k2, BatchSize=1):
+    """
+    Distance of Kernel with N scale value
+    BatchSize : Kernel number in k1 and k2
+
+    scalek1 = (each value of K1 / each value of K1)
+    scalek2 = (each value of K2 / each value of K2)
+    so if N = 9 , size(scaleK1) = 81
+    return diff = abs(mean(scalek1 - scalek2))
+    """
+    k1revert = k1.reshape((BatchSize, -1, 1))
+    k1 = 1.0 / k1.reshape((BatchSize, 1, -1))
+    k2revert = k2.reshape((BatchSize, -1, 1))
+    k2 = 1.0 / k2.reshape((BatchSize, 1, -1))
+    diff = np.abs(np.mean((np.matmul(k1revert, k1)- np.matmul(k2revert, k2)).reshape(BatchSize, -1), 1))
+    return diff
+
+
+def CrossDistanceOfKernel(k1, k2, BatchSize1=1, BatchSize2=1):
+    """
+    Distance of Kernel with N scale value
+    BatchSize : Kernel number in k1 and k2
+    K1 k2 dims must >= 2
+
+    scalek1 = (each value of K1 / each value of K1)
+    scalek2 = (each value of K2 / each value of K2)
+    so if N = 9 , size(scaleK1) = 81
+    return diff = abs(mean(scalek1 - scalek2))
+    """
+
+    tileshape = np.array(k2.shape)
+    tileshape[1:] = 1
+    tileshape[0] = k1.shape[0]
+    right = np.tile(k2, tileshape)
+    
+    tileshape = np.array(k1.shape)
+    tileshape[1] = k2.shape[0]
+    tileshape[0] = 1
+    tileshape[2:] = 1
+    left = np.tile(k1, tileshape)
+
+    leftshape = np.array(k1.shape)
+    leftshape[0] = -1
+    left = left.reshape(leftshape)
+
+    return DistanceOfKernel(right, left, left.shape[0])

FilterEvaluator/TrainNetwork.py

@@ -14,7 +14,6 @@ from torch.utils.data import Dataset, DataLoader
 from PIL import Image
 import random
 import cv2
-from visdom import Visdom
 from tqdm import tqdm
 
 
@@ -45,7 +44,7 @@ batchsize = 128
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra")
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500)
 
 
 WebVisual.InitVisdom()
@@ -65,15 +64,15 @@ Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain)
 
 
 model = utils.SetDevice(Model.Net3335())
-model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
+# model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
 optimizer = optim.SGD(model.parameters(), lr=0.1)
 Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre)
 
 
-model = utils.SetDevice(Model.Net3Grad335())
+# model = utils.SetDevice(Model.Net3Grad335())
-model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
+# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
-optimizer = optim.SGD(model.parameters(), lr=0.1)
+# optimizer = optim.SGD(model.parameters(), lr=0.1)
-Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
+# Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
 
 
 

tools/WebVisual.py

@@ -39,6 +39,10 @@ class Line():
     def AppendData(self, y):
         self.win.AppendData(self, y)
 
+    def AppendData(self, x, y):
+        self.win.AppendData(self, x, y)
+
+
 class LineWin():
     def __init__(self):
         super(LineWin, self).__init__()
@@ -77,3 +81,20 @@ class LineWin():
             name=line.name
         )
         line.size = line.size + len(y)
+
+    def AppendData(self, line, x, y):
+        if not isinstance(y, list) and not isinstance(y, np.ndarray):
+            y = [y]
+        if not isinstance(x, list) and not isinstance(x, np.ndarray):
+            x = [x]
+        X = np.array(x)
+        Y = np.array(y)
+        X = X[0:len(Y)]
+        self.viz.line(
+            Y=Y,
+            X=X,
+            win=self.win,
+            update="append",
+            name=line.name
+        )
+        line.size = line.size + len(y)

visdom.server.log

@@ -1,6 +1,7 @@
-nohup: ignoring input
 /home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops.
   ioloop.install()  # Needs to happen before any tornado imports!
+Checking for scripts.
+It's Alive!
 Traceback (most recent call last):
   File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main
     "__main__", mod_spec)
@@ -21,5 +22,3 @@ Traceback (most recent call last):
   File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets
     sock.bind(sockaddr)
 OSError: [Errno 98] Address already in use
-Checking for scripts.
-It's Alive!