Add UnsuperLearnTrainWeight to train new weight kernel

FilterEvaluator/Evaluator.py

@@ -53,32 +53,8 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
 # traindata, testdata = Loader.Cifar10Mono(batchsize)
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True)
 
-def ConvKernelToImage(model, layer, foldname):
-    if not os.path.exists(foldname):
-        os.mkdir(foldname)
-    a2 = model.features[layer].weight.data
-    a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
-    for i in range(a2.shape[0]):
-        d = a2[i]
-        dmin = np.min(d)
-        dmax = np.max(d)
-        d = (d - dmin)*255.0/(dmax-dmin)
-        d = d.astype(int)
-        cv2.imwrite(foldname+"/"+str(i)+".png", d)
-
-def ConvKernelToImage(numpydate, foldname):
-    if not os.path.exists(foldname):
-        os.mkdir(foldname)
-    a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
-    for i in range(a2.shape[0]):
-        d = a2[i]
-        dmin = np.min(d)
-        dmax = np.max(d)
-        d = (d - dmin)*255.0/(dmax-dmin)
-        d = d.astype(int)
-        cv2.imwrite(foldname+"/"+str(i)+".png", d)
 
 
 def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
@@ -175,7 +151,7 @@ def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
         print("  epoch :" + str(e))
     return SearchLayer.weight.data.cpu().detach().numpy()
 
-def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
+def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=20):
     tl = model.features[layer]
     newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
                          tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
@@ -203,6 +179,6 @@ np.save("WeightTrain.npy", weight)
 
 
 
-ConvKernelToImage(weight, CurrentPath+"image")
+utils.NumpyToImage(weight, CurrentPath+"image")
 # utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
 print("save model sucess")

FilterEvaluator/EvaluatorEntropy.py

@@ -47,16 +47,24 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
 
 
 
+
 # traindata, testdata = Loader.MNIST(batchsize)
 # traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
 # traindata, testdata = Loader.Cifar10Mono(batchsize)
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True)
 
 
-def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
+
+
+
+
+
+
+
+def GetScore(netmodel,layer,SearchLayer,DataSet,Interation=-1):
     netmodel.eval()
     sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
 
@@ -92,21 +100,7 @@ def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
     dat2 = torch.mean(dat2 * dat2,dim=1)
     return dat2.cpu().detach().numpy()
 
-def ConvKernelToImage(model, layer, foldname):
+def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
-    if not os.path.exists(foldname):
-        os.mkdir(foldname)
-    a2 = model.features[layer].weight.data
-    a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
-    for i in range(a2.shape[0]):
-        d = a2[i]
-        dmin = np.min(d)
-        dmax = np.max(d)
-        d = (d - dmin)*255.0/(dmax-dmin)
-        d = d.astype(int)
-        cv2.imwrite(foldname+"/"+str(i)+".png", d)
-
-
-def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChannelRatio=500, SearchChannelRatio=1, Interation=10):
     tl = model.features[layer]
     newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * SearchChannelRatio, tl.kernel_size,
                          tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
@@ -122,7 +116,7 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChanne
         newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
         newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
 
-        score = GetSample(model, layer, newlayer, dataloader, Interation)
+        score = GetScore(model, layer, newlayer, dataloader, Interation)
 
         minactive = np.append(minactive, score)
         minweight = np.concatenate((minweight, newweight))
@@ -139,16 +133,77 @@ def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch=10000, SaveChanne
     return minweight
 
 
-weight = UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
+def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
-np.save("weight.npy", weight)
+    netmodel.eval()
+    sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
+    layer = layer-1
+    SearchLayer.weight.data.requires_grad=True
+    optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
+    for e in range(Epoch):
+        for batch_idx, (data, target) in enumerate(DataSet):
+            optimizer.zero_grad()
+            data = utils.SetDevice(data)
+            target = utils.SetDevice(target)
+            output = netmodel.ForwardLayer(data,layer)
+            output = SearchLayer(output)
+            sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
+            sample_mean=torch.mean(sample,dim=1,keepdim=True)
+            dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
+            dat2 = (sample - sample_mean)/dat1
+            dat2 = torch.mean(dat2 * dat2,dim=1)
+            label = dat2*0.5
+            loss = F.l1_loss(dat2, label)
+            loss.backward()
+            optimizer.step()
+        print("  epoch :" + str(e))
+    return SearchLayer.weight.data.cpu().detach().numpy()
 
-ConvKernelToImage(model, layer, CurrentPath+"image")
+def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=20):
-utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
+    tl = model.features[layer]
+    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
+                         tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
+    newlayer = utils.SetDevice(newlayer)
+    newweightshape = list(newlayer.weight.data.shape)
+    minactive = np.empty((0))
+    minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
+    for i in range(NumTrain):
+        newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
+        newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
+        newweight = TrainLayer(model, layer, newlayer, dataloader, Epoch)
+        minweight = np.concatenate((minweight, newweight))
+        print("search :" + str(i))
+    return minweight
+
+
+
+
+
+
+
+
+weight = np.load(CurrentPath+"WeightSearch.npy")
+# weight = np.zeros((990,3,3));
+# weight[:,1]=100
+# weight[:,:,1]=100
+utils.NumpyToImage(weight,CurrentPath+"image")
+
+a=0
+
+
+
+
+
+
+
+
+# weight = UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
+# np.save("WeightSearch.npy", weight)
+
+weight = UnsuperLearnTrainWeight(model, layer, traindata)
+np.save("WeightTrain.npy", weight)
+
+
+
+ConvKernelToImage(weight, CurrentPath+"image")
+# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
 print("save model sucess")
-
-
-
-
-
-
-weight =  np.load("weight.npy")

FilterEvaluator/EvaluatorGrad.py

@@ -1,153 +0,0 @@
-from __future__ import print_function
-import os
-import sys
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-import torch.optim as optim
-import torchvision
-from torchvision import datasets, transforms
-import torchvision.models as models
-import matplotlib.pyplot as plt
-import numpy as np
-from torch.utils.data import Dataset, DataLoader
-from PIL import Image
-import random
-import cv2
-
-
-CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
-print("Current Path :" + CurrentPath)
-
-sys.path.append(CurrentPath+'../tools')
-sys.path.append(CurrentPath+'../')
-
-import Model as Model
-from tools import utils, Train, Loader
-
-batchsize = 128
-
-
-# model = utils.SetDevice(Model.Net5Grad35())
-# model = utils.SetDevice(Model.Net31535())
-model = utils.SetDevice(Model.Net3Grad335())
-# model = utils.SetDevice(Model.Net3())
-
-
-layers = model.PrintLayer()
-layer = 0
-
-model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
-
-
-
-
-
-# traindata, testdata = Loader.MNIST(batchsize)
-# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
-# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
-# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
-# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
-# traindata, testdata = Loader.Cifar10Mono(batchsize)
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
-
-
-
-
-
-
-
-
-
-
-# k = np.ones((3,3)).astype("float32")*1.0
-# d = np.ones((3,3)).astype("float32")*3.0
-# l = np.ones((3,3)).astype("float32")*3.5
-
-# k = torch.from_numpy(k)
-# d = torch.from_numpy(d)
-# l = torch.from_numpy(l)
-
-# k.requires_grad=True
-
-# optimizer = optim.SGD([k], lr=0.1)
-# optimizer.zero_grad()
-
-
-# output = k + d
-
-# loss = F.l1_loss(output, l)
-# loss.backward()
-# optimizer.step()
-
-
-
-def ConvKernelToImage(numpydate, foldname):
-    if not os.path.exists(foldname):
-        os.mkdir(foldname)
-    a2 = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
-    for i in range(a2.shape[0]):
-        d = a2[i]
-        dmin = np.min(d)
-        dmax = np.max(d)
-        d = (d - dmin)*255.0/(dmax-dmin)
-        d = d.astype(int)
-        cv2.imwrite(foldname+"/"+str(i)+".png", d)
-
-
-
-def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
-    netmodel.eval()
-    sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
-    layer = layer-1
-    SearchLayer.weight.data.requires_grad=True
-    optimizer = optim.SGD(SearchLayer.parameters(), lr=0.1)
-    for e in range(Epoch):
-        for batch_idx, (data, target) in enumerate(DataSet):
-            optimizer.zero_grad()
-            data = utils.SetDevice(data)
-            target = utils.SetDevice(target)
-            output = netmodel.ForwardLayer(data,layer)
-            output = SearchLayer(output)
-            sample = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
-            sample_mean=torch.mean(sample,dim=1,keepdim=True)
-            dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
-            dat2 = (sample - sample_mean)/dat1
-            dat2 = torch.mean(dat2 * dat2,dim=1)
-            label = dat2*0.5
-            loss = F.l1_loss(dat2, label)
-            loss.backward()
-            optimizer.step()
-        print("  epoch :" + str(e))
-    return SearchLayer.weight.data.cpu().detach().numpy()
-
-def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannelRatio=1, Epoch=100):
-    tl = model.features[layer]
-    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * TrainChannelRatio, tl.kernel_size,
-                         tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
-    newlayer = utils.SetDevice(newlayer)
-
-    newweightshape = list(newlayer.weight.data.shape)
-    
-    minactive = np.empty((0))
-    minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
-    for i in range(NumTrain):
-        newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
-        newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
-        newweight = TrainLayer(model, layer, newlayer, dataloader, 5)
-        minweight = np.concatenate((minweight, newweight))
-
-        ConvKernelToImage(minweight, CurrentPath+"image")
-
-        print("search :" + str(i))
-
-    return minweight
-
-
-weight = UnsuperLearnTrainWeight(model, layer, traindata)
-np.save("WeightTrain.npy", weight)
-
-# ConvKernelToImage(model, layer, CurrentPath+"image")
-# utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
-print("save model sucess")

tools/utils.py

@@ -149,3 +149,43 @@ def SetCUDAVISIBLEDEVICES(deviceid):
     print("Use GPU IDs :" + str(tuple(deviceid))[1:-1])
 
 
+
+
+def ConvKernelToImage(model, layer, foldname):
+    if not os.path.exists(foldname):
+        os.mkdir(foldname)
+    a2 = model.features[layer].weight.data
+    a2 = a2.cpu().detach().numpy().reshape((-1, a2.shape[-2], a2.shape[-1]))
+    for i in range(a2.shape[0]):
+        d = a2[i]
+        dmin = np.min(d)
+        dmax = np.max(d)
+        d = (d - dmin)*255.0/(dmax-dmin)
+        d = d.astype(int)
+        cv2.imwrite(foldname+"/"+str(i)+".png", d)
+
+def NumpyToImage(numpydate, foldname ,maxImageWidth = 128,maxImageHeight = 128):
+    if not os.path.exists(foldname):
+        os.mkdir(foldname)
+    numpydatemin = np.min(numpydate)
+    numpydatemax = np.max(numpydate)
+    numpydate = (numpydate - numpydatemin)*255.0/(numpydatemax-numpydatemin)
+    data = numpydate.reshape((-1, numpydate.shape[-2], numpydate.shape[-1]))
+    datashape = data.shape
+
+    newdata = np.ones((datashape[0],datashape[1]+1,datashape[2]+1))*numpydatemin
+    newdata[:, 0:datashape[1], 0:datashape[2]]=data
+    datashape = newdata.shape
+    
+    imagecols = int(maxImageWidth/datashape[2])
+    imagerows = int(maxImageHeight/datashape[1])
+    stepimages = imagecols*imagerows
+    for i in range(0,datashape[0],stepimages):
+        d = np.zeros((stepimages,datashape[1],datashape[2]))
+        left = newdata[i:min(i+stepimages,datashape[0])]
+        d[0:left.shape[0]]=left
+        d=np.reshape(d, (imagerows, imagecols, datashape[1], datashape[2]))
+        d=np.swapaxes(d, 1, 2)
+        d=np.reshape(d, (imagerows*datashape[1],imagecols*datashape[2]))
+        d = d.astype(int)
+        cv2.imwrite(foldname+"/"+str(i)+"-"+str(i+stepimages)+".png", d)