Add two method of real wit kernel genarator.

1 get 8 from 4000 with entropy calculation.
2 Auto Grad to regrethion kernel.

FilterEvaluator/Evaluator.py

@@ -55,8 +55,33 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
 # traindata, testdata = Loader.Cifar10Mono(batchsize)
 traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
 
+def ConvKernelToImage(model, layer, foldname):
+    if not os.path.exists(foldname):
+        os.mkdir(foldname)
+    a2 = model.features[layer].weight.data
+    a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
+    for i in range(a2.shape[0]):
+        d = a2[i]
+        dmin = np.min(d)
+        dmax = np.max(d)
+        d = (d - dmin)*255.0/(dmax-dmin)
+        d = d.astype(int)
+        cv2.imwrite(foldname+"/"+str(i)+".png", d)
 
-def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
+def ConvKernelToImage(numpydate, foldname):
+    if not os.path.exists(foldname):
+        os.mkdir(foldname)
+    a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
+    for i in range(a2.shape[0]):
+        d = a2[i]
+        dmin = np.min(d)
+        dmax = np.max(d)
+        d = (d - dmin)*255.0/(dmax-dmin)
+        d = d.astype(int)
+        cv2.imwrite(foldname+"/"+str(i)+".png", d)
+
+
+def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
     netmodel.eval()
     sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
 
@@ -92,35 +117,13 @@ def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
     dat2 = torch.mean(dat2 * dat2,dim=1)
     return dat2.cpu().detach().numpy()
 
-def ConvKernelToImage(model, layer, foldname):
+def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
-    if not os.path.exists(foldname):
-        os.mkdir(foldname)
-    a2 = model.features[layer].weight.data
-    a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
-    for i in range(a2.shape[0]):
-        d = a2[i]
-        dmin = np.min(d)
-        dmax = np.max(d)
-        d = (d - dmin)*255.0/(dmax-dmin)
-        d = d.astype(int)
-        cv2.imwrite(foldname+"/"+str(i)+".png", d)
-
-def GetRandomSocre(netmodel,layer,dataloader,iteration=-1):
-    weightshape = netmodel.features[layer].weight.data.shape
-    newweight = np.random.uniform(-1.0,1.0,weightshape).astype("float32")
-    netmodel.features[layer].weight.data=utils.SetDevice(torch.from_numpy(newweight))
-
-    score = GetSample(netmodel,0,dataloader,iteration)
-    return np.array(score), newweight
-
-
-def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelRatio=1,Interation=10):
     tl = model.features[layer]
-    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * ChannelRatio, tl.kernel_size,
+    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
                          tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
     newlayer = utils.SetDevice(newlayer)
 
-    newchannels = tl.out_channels * ChannelRatio
+    newchannels = tl.out_channels * SaveChannelRatio
     newweightshape = list(newlayer.weight.data.shape)
     
     minactive = np.empty((0))
@@ -130,7 +133,7 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelR
         newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
         newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
 
-        score = GetSample(model, layer, newlayer, dataloader, Interation)
+        score = GetScore(model, layer, newlayer, dataloader, Interation)
 
         minactive = np.append(minactive, score)
         minweight = np.concatenate((minweight, newweight))
@@ -142,11 +145,64 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelR
         if i % (NumSearch/10) == 0:
             tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
             utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
+    
     tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
+    return minweight
 
 
-UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, ChannelRatio=8, Interation=10)
+def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
+    netmodel.eval()
+    sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
+    layer = layer-1
+    SearchLayer.weight.data.requires_grad=True
+    optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
+    for e in range(Epoch):
+        for batch_idx, (data, target) in enumerate(DataSet):
+            optimizer.zero_grad()
+            data = utils.SetDevice(data)
+            target = utils.SetDevice(target)
+            output = netmodel.ForwardLayer(data,layer)
+            output = SearchLayer(output)
+            sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
+            sample_mean=torch.mean(sample,dim=1,keepdim=True)
+            dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
+            dat2 = (sample - sample_mean)/dat1
+            dat2 = torch.mean(dat2 * dat2,dim=1)
+            label = dat2*0.5
+            loss = F.l1_loss(dat2, label)
+            loss.backward()
+            optimizer.step()
+        print("  epoch :" + str(e))
+    return SearchLayer.weight.data.cpu().detach().numpy()
 
-ConvKernelToImage(model, layer, CurrentPath+"image")
+def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
-utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
+    tl = model.features[layer]
+    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
+                         tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
+    newlayer = utils.SetDevice(newlayer)
+    newweightshape = list(newlayer.weight.data.shape)
+    minactive = np.empty((0))
+    minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
+    for i in range(NumTrain):
+        newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
+        newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
+        newweight = TrainLayer(model, layer, newlayer, dataloader, Epoch)
+        minweight = np.concatenate((minweight, newweight))
+        print("search :" + str(i))
+    return minweight
+
+
+
+
+
+# weight = UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
+# np.save("WeightSearch.npy", weight)
+
+weight = UnsuperLearnTrainWeight(model, layer, traindata)
+np.save("WeightTrain.npy", weight)
+
+
+
+ConvKernelToImage(weight, CurrentPath+"image")
+# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
 print("save model sucess")

FilterEvaluator/EvaluatorEntropy.py

@@ -0,0 +1,154 @@
+from __future__ import print_function
+import os
+import sys
+
+# import multiprocessing
+# multiprocessing.set_start_method('spawn', True)
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+import random
+import cv2
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+sys.path.append(CurrentPath+'../tools')
+sys.path.append(CurrentPath+'../')
+
+import Model as Model
+from tools import utils, Train, Loader
+
+batchsize = 128
+
+
+# model = utils.SetDevice(Model.Net5Grad35())
+# model = utils.SetDevice(Model.Net31535())
+model = utils.SetDevice(Model.Net3Grad335())
+# model = utils.SetDevice(Model.Net3())
+
+
+layers = model.PrintLayer()
+layer = 0
+
+model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+
+
+
+
+
+# traindata, testdata = Loader.MNIST(batchsize)
+# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
+# traindata, testdata = Loader.Cifar10Mono(batchsize)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
+
+
+def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
+    netmodel.eval()
+    sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
+
+    layer = layer-1
+
+    # layerout = []
+    #     layerint = []
+    #     def getnet(self, input, output):
+    #         layerout.append(output)
+    #         layerint.append(input)
+    # handle = netmodel.features[layer].register_forward_hook(getnet)
+    # netmodel.ForwardLayer(data,layer)
+    # output = layerout[0][:,:,:,:]
+    # handle.remove()
+
+
+    for batch_idx, (data, target) in enumerate(DataSet):
+        data = utils.SetDevice(data)
+        target = utils.SetDevice(target)
+        output = netmodel.ForwardLayer(data,layer)
+        output = SearchLayer(output)
+        data.detach()
+        target.detach()
+        
+        output = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
+        sample = torch.cat((sample,output),1)
+        if Interation > 0 and batch_idx >= (Interation-1):
+            break
+
+    sample_mean=torch.mean(sample,dim=1,keepdim=True)
+    dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
+    dat2 = (sample - sample_mean)/dat1
+    dat2 = torch.mean(dat2 * dat2,dim=1)
+    return dat2.cpu().detach().numpy()
+
+def ConvKernelToImage(model, layer, foldname):
+    if not os.path.exists(foldname):
+        os.mkdir(foldname)
+    a2 = model.features[layer].weight.data
+    a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
+    for i in range(a2.shape[0]):
+        d = a2[i]
+        dmin = np.min(d)
+        dmax = np.max(d)
+        d = (d - dmin)*255.0/(dmax-dmin)
+        d = d.astype(int)
+        cv2.imwrite(foldname+"/"+str(i)+".png", d)
+
+
+def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
+    tl = model.features[layer]
+    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
+                         tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
+    newlayer = utils.SetDevice(newlayer)
+
+    newchannels = tl.out_channels * SaveChannelRatio
+    newweightshape = list(newlayer.weight.data.shape)
+    
+    minactive = np.empty((0))
+    minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
+
+    for i in range(NumSearch):
+        newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
+        newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
+
+        score = GetSample(model, layer, newlayer, dataloader, Interation)
+
+        minactive = np.append(minactive, score)
+        minweight = np.concatenate((minweight, newweight))
+
+        index = minactive.argsort()
+        minactive = minactive[index[0:newchannels]]
+        minweight = minweight[index[0:newchannels]]
+        print("search random :" + str(i))
+        if i % (NumSearch/10) == 0:
+            tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
+            utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
+    
+    tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
+    return minweight
+
+
+weight = UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
+np.save("weight.npy", weight)
+
+ConvKernelToImage(model, layer, CurrentPath+"image")
+utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
+print("save model sucess")
+
+
+
+
+
+
+weight =  np.load("weight.npy")

FilterEvaluator/EvaluatorGrad.py

@@ -0,0 +1,153 @@
+from __future__ import print_function
+import os
+import sys
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+import random
+import cv2
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+sys.path.append(CurrentPath+'../tools')
+sys.path.append(CurrentPath+'../')
+
+import Model as Model
+from tools import utils, Train, Loader
+
+batchsize = 128
+
+
+# model = utils.SetDevice(Model.Net5Grad35())
+# model = utils.SetDevice(Model.Net31535())
+model = utils.SetDevice(Model.Net3Grad335())
+# model = utils.SetDevice(Model.Net3())
+
+
+layers = model.PrintLayer()
+layer = 0
+
+model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+
+
+
+
+
+# traindata, testdata = Loader.MNIST(batchsize)
+# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
+# traindata, testdata = Loader.Cifar10Mono(batchsize)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
+
+
+
+
+
+
+
+
+
+
+# k = np.ones((3,3)).astype("float32")*1.0
+# d = np.ones((3,3)).astype("float32")*3.0
+# l = np.ones((3,3)).astype("float32")*3.5
+
+# k = torch.from_numpy(k)
+# d = torch.from_numpy(d)
+# l = torch.from_numpy(l)
+
+# k.requires_grad=True
+
+# optimizer = optim.SGD([k], lr=0.1)
+# optimizer.zero_grad()
+
+
+# output = k + d
+
+# loss = F.l1_loss(output, l)
+# loss.backward()
+# optimizer.step()
+
+
+
+def ConvKernelToImage(numpydate, foldname):
+    if not os.path.exists(foldname):
+        os.mkdir(foldname)
+    a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
+    for i in range(a2.shape[0]):
+        d = a2[i]
+        dmin = np.min(d)
+        dmax = np.max(d)
+        d = (d - dmin)*255.0/(dmax-dmin)
+        d = d.astype(int)
+        cv2.imwrite(foldname+"/"+str(i)+".png", d)
+
+
+
+def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
+    netmodel.eval()
+    sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
+    layer = layer-1
+    SearchLayer.weight.data.requires_grad=True
+    optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
+    for e in range(Epoch):
+        for batch_idx, (data, target) in enumerate(DataSet):
+            optimizer.zero_grad()
+            data = utils.SetDevice(data)
+            target = utils.SetDevice(target)
+            output = netmodel.ForwardLayer(data,layer)
+            output = SearchLayer(output)
+            sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
+            sample_mean=torch.mean(sample,dim=1,keepdim=True)
+            dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
+            dat2 = (sample - sample_mean)/dat1
+            dat2 = torch.mean(dat2 * dat2,dim=1)
+            label = dat2*0.5
+            loss = F.l1_loss(dat2, label)
+            loss.backward()
+            optimizer.step()
+        print("  epoch :" + str(e))
+    return SearchLayer.weight.data.cpu().detach().numpy()
+
+def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
+    tl = model.features[layer]
+    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
+                         tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
+    newlayer = utils.SetDevice(newlayer)
+
+    newweightshape = list(newlayer.weight.data.shape)
+    
+    minactive = np.empty((0))
+    minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
+    for i in range(NumTrain):
+        newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
+        newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
+        newweight = TrainLayer(model, layer, newlayer, dataloader, 5)
+        minweight = np.concatenate((minweight, newweight))
+
+        ConvKernelToImage(minweight, CurrentPath+"image")
+
+        print("search :" + str(i))
+
+    return minweight
+
+
+weight = UnsuperLearnTrainWeight(model, layer, traindata)
+np.save("WeightTrain.npy", weight)
+
+# ConvKernelToImage(model, layer, CurrentPath+"image")
+# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
+print("save model sucess")