Try deep search ,and test in Minist DataSet

FilterEvaluator/Evaluator.py

@@ -29,42 +29,40 @@ import EvaluatorUnsuper
 
 
 batchsize = 128
+layer = 0
+
 
 # model = utils.SetDevice(Model.Net5Grad35())
 # model = utils.SetDevice(Model.Net31535())
-model = utils.SetDevice(Model.Net3Grad335())
+# model = utils.SetDevice(Model.Net3Grad335())
+model = utils.SetDevice(Model.Net3Grad334())
 # model = utils.SetDevice(Model.Net3())
+model.PrintLayer()
 
 
 
 
 
-layers = model.PrintLayer()
+traindata, testdata = Loader.MNIST(batchsize)
-layer = 0
-# model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
-
-
-
-# traindata, testdata = Loader.MNIST(batchsize)
 # traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
 # traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
 # traindata, testdata = Loader.Cifar10Mono(batchsize)
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=False)
+# traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=False)
 
 
 
 
 
-# weight,active = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=1000000, SearchChannelRatio=32, Interation=5)
+# weight,active = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=500000, SearchChannelRatio=32, Interation=5)
 # np.save("WeightSearch.npy", weight)
 # weight = np.load(CurrentPath+"WeightSearch.npy")
 # utils.NumpyToImage(weight, CurrentPath+"image",title="SearchWeight")
-weight = np.load(CurrentPath+"WeightSearch.npy")
+# weight = np.load(CurrentPath+"WeightSearch.npy")
-weight = weight[0:256]
+# weight = weight[0:256]
-bestweight,index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,32,400000)
+# bestweight,index = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,32,4000000)
-np.save(CurrentPath+"bestweightSearch.npy", bestweight)
+# np.save(CurrentPath+"bestweightSearch.npy", bestweight)
 bestweight = np.load(CurrentPath+"bestweightSearch.npy")
 utils.NumpyToImage(bestweight, CurrentPath+"image",title="SearchWerightBest")
 EvaluatorUnsuper.SetModelConvWeight(model,layer,bestweight)

FilterEvaluator/Model.py

@@ -60,6 +60,35 @@ class Net3335(UniModule.ModuleBase):
         x = x.view(-1, 1*10)
         return F.log_softmax(x, dim=1)
 
+class Net3334(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net3334, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=3,bias=False,padding=1),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=4,bias=False)]
+        self.features = nn.Sequential(*layers)
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
+class Net3Grad334(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net3Grad334, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=3,bias=False,padding=1),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=4,bias=False)]
+        self.features = nn.Sequential(*layers)
+        self.SetConvRequiresGrad(0,False)
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
 class Net3335BN(UniModule.ModuleBase):
     def __init__(self):
         super(Net3335BN, self).__init__()

FilterEvaluator/TrainNetwork.py

@@ -39,13 +39,14 @@ batchsize = 128
 # model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
 
 
-# traindata, testdata = Loader.MNIST(batchsize, num_workers=4)
+traindata, testdata = Loader.MNIST(batchsize, num_workers=4)
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra")
-# traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500)
+# traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=135)
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=0)
+# traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=0)
+
 
 
 WebVisual.InitVisdom()
@@ -62,26 +63,26 @@ linePretrainTrain = WebVisual.Line(window, "PretrainTrain")
 # model = utils.SetDevice(Model.Net3Grad335())
 # model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
 # optimizer = optim.SGD(model.parameters(), lr=0.1)
-# Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain)
+# Train.TrainEpochs(model,traindata,optimizer,testdata,3000,15,linePretrainTrain)
  
 
-model = utils.SetDevice(Model.Net3335BN())
+#model = utils.SetDevice(Model.Net3335BN())
+## model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
+#optimizer = optim.SGD(model.parameters(), lr=0.1)
+#Train.TrainEpochs(model,traindata,optimizer,testdata,3000,15,lineNoPreBN)
+
+
+model = utils.SetDevice(Model.Net3334())
 # model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
 optimizer = optim.SGD(model.parameters(), lr=0.1)
-Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPreBN)
+Train.TrainEpochs(model,traindata,optimizer,testdata,3000,15,lineNoPre)
 
 
-model = utils.SetDevice(Model.Net3335())
+
-# model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
+model = utils.SetDevice(Model.Net3Grad334())
+model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
 optimizer = optim.SGD(model.parameters(), lr=0.1)
-Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre)
+Train.TrainEpochs(model,traindata,optimizer,testdata,3000,15,linePretrainSearch)
-
-
-
-# model = utils.SetDevice(Model.Net3Grad335())
-# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
-# optimizer = optim.SGD(model.parameters(), lr=0.1)
-# Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
 
 
 

tools/Loader.py

@@ -8,21 +8,26 @@ from torch.utils.data import Dataset, DataLoader, Subset
 import random
 
 
-def MNIST(batchsize=8, num_workers=0, shuffle=False):
+def MNIST(batchsize=8, num_workers=0, shuffle=False, trainsize=0):
   CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
-  train_loader = torch.utils.data.DataLoader(
+  traindata = datasets.MNIST(root=CurrentPath+'../Dataset/', train=True, download=True,
-      datasets.MNIST(root=CurrentPath+'../Dataset/', train=True, download=True,
                              transform=transforms.Compose([
                                  transforms.ColorJitter(0.2, 0.2),
                                  transforms.RandomRotation(30),
                                  transforms.RandomResizedCrop(28),
-                         transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),
+                                 transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]))
+  if trainsize == 0:
+    trainsize = traindata.data.shape[0]
+  train_loader = torch.utils.data.DataLoader(
+      Subset(traindata, range(0, trainsize)),
       batch_size=batchsize, shuffle=shuffle, num_workers=num_workers, drop_last=True)
   test_loader = torch.utils.data.DataLoader(
       datasets.MNIST(root=CurrentPath+'../Dataset/', train=False, transform=transforms.Compose([
           transforms.ToTensor(),
           transforms.Normalize((0.1307,), (0.3081,))
       ])), batch_size=batchsize, shuffle=shuffle, num_workers=num_workers, drop_last=True)
+  train_loader.batch_sampler.sampler.num_samples = 50000
+  test_loader.batch_sampler.sampler.num_samples = 10000
   return train_loader, test_loader
 
 def Cifar10(batchsize=8, num_workers=0, shuffle=False):

tools/Train.py

@@ -14,14 +14,13 @@ from tqdm import tqdm
 
 def train(model, train_loader, optimizer, epoch=0):
     model.train()
-    batchsize = int(train_loader.sampler.num_samples /
+    totalsize = train_loader.batch_sampler.sampler.num_samples
-                    train_loader.batch_size / 5)+1
+    batchsize = int(totalsize / train_loader.batch_size / 5)+1
-    pbar = tqdm(total=train_loader.sampler.num_samples)
+    pbar = tqdm(totalsize)
     for batch_idx, (data, target) in enumerate(train_loader):
         data = utils.SetDevice(data)
         target = utils.SetDevice(target)
         optimizer.zero_grad()
-
         output = model(data)
         loss = F.nll_loss(output, target)
         loss.backward()
@@ -30,15 +29,8 @@ def train(model, train_loader, optimizer, epoch=0):
         pbar.update(train_loader.batch_size)
         if batch_idx % batchsize == 0 and batch_idx > 0:
             pbar.set_description("Loss:"+str(loss.item()))
-            # print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'
-            #       .format(epoch, batch_idx * len(data),
-            #               len(train_loader.dataset),
-            #               100. * batch_idx /
-            #               len(train_loader),
-            #               loss.item()))
     pbar.close()
 
-
 def test(model, test_loader):
     with torch.no_grad():
         model.eval()

tools/WebVisual.py

@@ -17,7 +17,7 @@ def InitVisdom():
     print("Connect http://localhost:8097")
 
 
-class Line():
+class Line(object):
     def __init__(self, windows, name, size):
         super(Line, self).__init__()
         self.name = name
@@ -40,7 +40,7 @@ class Line():
         self.win.AppendData(self, y, x)
 
 
-class LineWin():
+class LineWin(object):
     def __init__(self):
         super(LineWin, self).__init__()
         self.viz = Visdom()