start to creat 8 from 4000

FilterEvaluator/Evaluator.py

@@ -38,6 +38,8 @@ model = utils.SetDevice(Model.Net3Grad335())
 # model = utils.SetDevice(Model.Net3())
 
 
+layers = model.PrintLayer()
+layer = 0
 
 model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
 
@@ -54,28 +56,34 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
 traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
 
 
-def GetSample(netmodel,layer,dataloader,iteration=-1):
+def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1):
     netmodel.eval()
-    sample = utils.SetDevice(torch.empty((8,0)))
+    sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
-    for batch_idx, (data, target) in enumerate(dataloader):
+
+    layer = layer-1
+
+    # layerout = []
+    #     layerint = []
+    #     def getnet(self, input, output):
+    #         layerout.append(output)
+    #         layerint.append(input)
+    # handle = netmodel.features[layer].register_forward_hook(getnet)
+    # netmodel.ForwardLayer(data,layer)
+    # output = layerout[0][:,:,:,:]
+    # handle.remove()
+
+
+    for batch_idx, (data, target) in enumerate(DataSet):
         data = utils.SetDevice(data)
         target = utils.SetDevice(target)
-        layerout = []
+        output = netmodel.ForwardLayer(data,layer)
-        layerint = []
+        output = SearchLayer(output)
-        def getnet(self, input, output):
-            layerout.append(output)
-            layerint.append(input)
-        handle = netmodel.features[layer].register_forward_hook(getnet)
-        netmodel.forwardLayer(data,layer=layer)
-        output = layerout[0][:,:,:,:]
-
-        handle.remove()
         data.detach()
         target.detach()
         
-        output = torch.reshape(output.transpose(0,1),(8,-1))
+        output = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1))
         sample = torch.cat((sample,output),1)
-        if iteration > 0 and batch_idx >= (iteration-1):
+        if Interation > 0 and batch_idx >= (Interation-1):
             break
 
     sample_mean=torch.mean(sample,dim=1,keepdim=True)
@@ -104,47 +112,41 @@ def GetRandomSocre(netmodel,layer,dataloader,iteration=-1):
 
     score = GetSample(netmodel,0,dataloader,iteration)
     return np.array(score), newweight
-  
 
 
-ConvKernelToImage(model, 0, CurrentPath+"image")
+def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelRatio=1,Interation=10):
-
+    tl = model.features[layer]
-
+    newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * ChannelRatio, tl.kernel_size,
-
+                         tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode)
-
+    newlayer = utils.SetDevice(newlayer)
-
-layer = 0
-
-weightshape = list(model.features[layer].weight.data.shape)
-channels = weightshape[0]
-weightshape[0] = 0
-minactive = np.empty((0))
-minweight = np.empty(weightshape)
-
-for i in range(300000):
-    score,weight = GetRandomSocre(model,layer,traindata,iteration=10)
-
-    minactive = np.append(minactive, score)
-    minweight = np.concatenate((minweight, weight))
-
-    index = minactive.argsort()
-    minactive = minactive[index[0:channels]]
-    minweight = minweight[index[0:channels]]
-
-    print("search random :" + str(i))
-    if i % 10000 == 0:
-        utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
 
+    newchannels = tl.out_channels * ChannelRatio
+    newweightshape = list(newlayer.weight.data.shape)
     
-# for i in range(minweight.shape[0]):
+    minactive = np.empty((0))
-#     a2 = minweight[i].reshape((minweight.shape[2],minweight.shape[3]))
+    minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]])
-#     a2min = np.min(a2)
+
-#     a2max = np.max(a2)
+    for i in range(NumSearch):
-#     a2 = (a2 - a2min)*255.0/(a2max-a2min)
+        newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32")
-#     a2 = a2.astype(int)
+        newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight))
-#     cv2.imwrite(CurrentPath+"/image/c"+str(i)+"_"+str(minactive[i])+".png",a2)
+
+        score = GetSample(model, layer, newlayer, dataloader, Interation)
+
+        minactive = np.append(minactive, score)
+        minweight = np.concatenate((minweight, newweight))
+
+        index = minactive.argsort()
+        minactive = minactive[index[0:newchannels]]
+        minweight = minweight[index[0:newchannels]]
+        print("search random :" + str(i))
+        if i % (NumSearch/10) == 0:
+            tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
+            utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
+    tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
 
 
-model.features[layer].weight.data=utils.SetDevice(torch.from_numpy(minweight))
+UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, ChannelRatio=8, Interation=10)
+
+ConvKernelToImage(model, layer, CurrentPath+"image")
 utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
 print("save model sucess")

FilterEvaluator/TrainNetwork.py

@@ -31,6 +31,12 @@ batchsize = 128
 
 
 
+# model = utils.SetDevice(Model.Net535())
+# model = utils.SetDevice(Model.Net5Grad35())
+# model = utils.SetDevice(Model.Net31535())
+# optimizer = optim.Adam(model.parameters(), lr=0.01)
+
+
 # traindata, testdata = Loader.MNIST(batchsize, num_workers=4)
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal")
@@ -39,13 +45,7 @@ batchsize = 128
 traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500)
 
 
-
+# WebVisual.InitVisdom()
-
-
-
-
-
-WebVisual.InitVisdom()
 window = WebVisual.LineWin()
 lineNoPre = WebVisual.Line("NoPre")
 linePretrain = WebVisual.Line("Pretrain")
@@ -55,42 +55,26 @@ linePretrain = WebVisual.Line("Pretrain")
 
 
 
-# model = utils.SetDevice(Model.Net535())
+
-# model = utils.SetDevice(Model.Net5Grad35())
-# model = utils.SetDevice(Model.Net31535())
 # model = utils.SetDevice(Model.Net3335()) 
 model = utils.SetDevice(Model.Net3Grad335())
-
+model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
-
-
-# model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
-
-
 
 optimizer = optim.SGD(model.parameters(), lr=0.1)
-# optimizer = optim.Adam(model.parameters(), lr=0.01)
+for i in range(1000):
-
-
-
-
-model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
-for i in range(2000):
     Train.train(model,traindata,optimizer,epoch=i)
     window.AppendData(linePretrain,Train.test(model,testdata))
 
 
-
 model = utils.SetDevice(Model.Net3335())
+model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+
 optimizer = optim.SGD(model.parameters(), lr=0.1)
-for i in range(2000):
+for i in range(1000):
     Train.train(model, traindata, optimizer, epoch=i)
     window.AppendData(lineNoPre, Train.test(model, testdata))
 
 
 
 
-
-
-
-
 # utils.SaveModel(model,CurrentPath+"/checkpointSGDPreTrain.pkl")

tools/UniModule.py

@@ -20,7 +20,17 @@ class ModuleBase(nn.Module):
       if b != None:
         self.features[layer].bias.requires_grad = requiregrad
         
-    def forwardLayer(self, x, layer=0):
+    def ForwardLayer(self, x, layer=0):
+        if layer < 0:
+            return x
         layers = self.features[0:layer+1]
         x = layers(x)
-
+        return x
+    
+    def PrintLayer(self):
+        names = []
+        i = 0
+        for l in self.features:
+            names.append(str(i)+' : '+str(l))
+            i = i+1
+        return names