Refine find best 8

FilterEvaluator/Evaluator.py

@@ -60,7 +60,7 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
 # weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
 # np.save("WeightSearch.npy", weight)
 
-# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata)
+# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata, NumTrain=5000)
 # np.save("WeightTrain.npy", weight)
 
 
@@ -72,20 +72,21 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
 # utils.NumpyToImage(bestweight, CurrentPath+"image")
 
 
-# weight = np.load(CurrentPath+"WeightTrain.npy")
+weight = np.load(CurrentPath+"WeightTrain.npy")
-# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
+bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(
-# np.save(CurrentPath+"bestweightTrain.npy", bestweight)
+    model, layer, weight, traindata, databatchs=128, interation=100000)
-# utils.NumpyToImage(bestweight, CurrentPath+"image")
+np.save(CurrentPath+"bestweightTrain.npy", bestweight)
+utils.NumpyToImage(bestweight, CurrentPath+"image")
 
 
 
-weight = np.load(CurrentPath+"bestweightSearch.npy")
+# weight = np.load(CurrentPath+"bestweightSearch.npy")
-EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
+# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
-utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
+# utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
 
-weight = np.load(CurrentPath+"bestweightTrain.npy")
+# weight = np.load(CurrentPath+"bestweightTrain.npy")
-EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
+# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
-utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
+# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
 
 
 

FilterEvaluator/EvaluatorUnsuper.py

@@ -14,6 +14,8 @@ from torch.utils.data import Dataset, DataLoader
 from PIL import Image
 import random
 import cv2
+from tqdm import tqdm
+
 
 CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
 sys.path.append(CurrentPath+'../tools')
@@ -91,7 +93,6 @@ def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChan
     tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
     return minweight
 
-
 def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
     netmodel.eval()
     sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
@@ -133,8 +134,8 @@ def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannel
         print("search :" + str(i))
     return minweight
 
-
 def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=128,interation=10000):
+    interationbar = tqdm(total=interation)    
     weight = weight.astype("float32")
     netmodel.eval()
     tl = netmodel.features[layer]
@@ -155,28 +156,26 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1
     for i in range(outchannels):
         shift.append(1<<i)
     shift = utils.SetDevice(torch.from_numpy(np.array(shift).astype("uint8")))
+    datasnetout = []
+    for d in datas:
+        datasnetout.append(netmodel.ForwardLayer(d,forwardlayer))
+    datasnetout = torch.cat(datasnetout)
     for i in range(len(indexs)):
         newlayer.weight.data=utils.SetDevice(torch.from_numpy(weight[indexs[i]]))
-        outputs = []
+        outputs = newlayer(datasnetout).transpose(0,1)
-        for d in datas:
+        outputs = outputs.reshape(tl.out_channels,-1).transpose(0,1)
-            outputs.append(newlayer(netmodel.ForwardLayer(d,forwardlayer)))
+        sums = ((outputs > outputs.mean(0)) * shift).sum(1).type(torch.float32)
-        outputs = torch.cat(outputs).transpose(0,1)
-        outputs = outputs.reshape(outputs.shape[0],-1)
-        mean = outputs.mean(1)
-        outputs = outputs.transpose(0,1)
-        posi = outputs > mean
-        posi = posi*shift
-        sums = posi.sum(1).type(torch.float32)
         histc = sums.histc(256,0,255).type(torch.float32)
         histc = histc[histc>0]
         histc = histc/histc.sum()
-        entropys.append((histc.log2()*histc).sum())
+        entropy = (histc.log2()*histc).sum()
-        print("search index : " + str(i))
+        entropys.append(entropy.detach().cpu().numpy())
+        interationbar.update(1)
     argmin = np.argmin(entropys)
     bestweight = weight[indexs[argmin]]
+    interationbar.close()
     return bestweight
 
-
 def SetModelConvWeight(model, layer, weight):
     w = utils.SetDevice(torch.from_numpy(weight))
     model.features[layer].weight.data = w

FilterEvaluator/TrainNetwork.py

@@ -15,6 +15,7 @@ from PIL import Image
 import random
 import cv2
 from visdom import Visdom
+from tqdm import tqdm
 
 
 CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
@@ -36,45 +37,44 @@ batchsize = 128
 # model = utils.SetDevice(Model.Net31535())
 # optimizer = optim.Adam(model.parameters(), lr=0.01)
 
+# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
+
 
 # traindata, testdata = Loader.MNIST(batchsize, num_workers=4)
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine")
 # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra")
-traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140)
 
 
-# WebVisual.InitVisdom()
+WebVisual.InitVisdom()
 window = WebVisual.LineWin()
-lineNoPre = WebVisual.Line("NoPre")
+lineNoPre = WebVisual.Line(window, "NoPre")
-linePretrain = WebVisual.Line("Pretrain")
+linePretrainSearch = WebVisual.Line(window, "PretrainSearch")
+linePretrainTrain = WebVisual.Line(window, "PretrainTrain")
 
 
 
 
 
-
-
-# model = utils.SetDevice(Model.Net3335()) 
 model = utils.SetDevice(Model.Net3Grad335())
-model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
-
 optimizer = optim.SGD(model.parameters(), lr=0.1)
-for i in range(1000):
+Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain)
-    Train.train(model,traindata,optimizer,epoch=i)
-    window.AppendData(linePretrain,Train.test(model,testdata))
-
-
 
 
 model = utils.SetDevice(Model.Net3335())
-model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
-
 optimizer = optim.SGD(model.parameters(), lr=0.1)
-for i in range(1000):
+Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre)
-    Train.train(model, traindata, optimizer, epoch=i)
+
-    window.AppendData(lineNoPre, Train.test(model, testdata))
+
+model = utils.SetDevice(Model.Net3Grad335())
+model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
+optimizer = optim.SGD(model.parameters(), lr=0.1)
+Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
+
 
 
 

tools/Train.py

@@ -9,12 +9,14 @@ import torch.nn as nn
 import torch
 import os
 import utils as utils
+from tqdm import tqdm
 
 
 def train(model, train_loader, optimizer, epoch=0):
     model.train()
     batchsize = int(train_loader.sampler.num_samples /
                     train_loader.batch_size / 5)+1
+    pbar = tqdm(total=train_loader.sampler.num_samples)
     for batch_idx, (data, target) in enumerate(train_loader):
         data = utils.SetDevice(data)
         target = utils.SetDevice(target)
@@ -25,13 +27,16 @@ def train(model, train_loader, optimizer, epoch=0):
         loss.backward()
         optimizer.step()
 
+        pbar.update(train_loader.batch_size)
         if batch_idx % batchsize == 0 and batch_idx > 0:
-            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'
+            pbar.set_description("Loss:"+str(loss.item()))
-                  .format(epoch, batch_idx * len(data),
+            # print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'
-                          len(train_loader.dataset),
+            #       .format(epoch, batch_idx * len(data),
-                          100. * batch_idx /
+            #               len(train_loader.dataset),
-                          len(train_loader),
+            #               100. * batch_idx /
-                          loss.item()))
+            #               len(train_loader),
+            #               loss.item()))
+    pbar.close()
 
 
 def test(model, test_loader):
@@ -43,15 +48,23 @@ def test(model, test_loader):
             data = utils.SetDevice(data)
             target = utils.SetDevice(target)
             output = model(data)
-
             # sum up batch loss
             test_loss += F.nll_loss(output, target, reduction='sum').item()
             # get the index of the max log-probability
             pred = output.max(1, keepdim=True)[1]
             correct += pred.eq(target.view_as(pred)).sum().item()
-
         test_loss /= len(test_loader.dataset)
         accu = 100. * correct / len(test_loader.dataset)
-        print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'
+        # print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'
-              .format(test_loss, correct, len(test_loader.dataset), accu))
+        #       .format(test_loss, correct, len(test_loader.dataset), accu))
         return accu
+
+
+def TrainEpochs(model, traindata, optimizer, testdata, epoch=100,testepoch=10, line=None):
+    epochbar = tqdm(total=epoch)
+    for i in range(epoch):
+        train(model, traindata, optimizer, epoch=i)
+        if line and i % testepoch == 0 and i > 0:
+            line.AppendData(test(model, testdata))
+        epochbar.update(1)
+    epochbar.close()

tools/WebVisual.py

@@ -1,27 +1,43 @@
 from visdom import Visdom
 import random
 import numpy as np
-
+import os
+import time
 
 
 def InitVisdom():
-    viz = Visdom()
+    # nohub python3 -m visdom.server > visdom.server.log 2 > &1
-    assert viz.check_connection()
+    # subprocess.Popen("python3 -m visdom.server > visdom.server.log 2>&1", shell=True)
-    viz.close()
+
+    os.popen("python3 -m visdom.server > visdom.server.log 2>&1")
+    time.sleep(3)  # to wait vindom.server started
+    # viz = Visdom()
+    # assert viz.check_connection()
+    # viz.close()
+    print("Connect http://localhost:8097")
+
 
 class Line():
-    def __init__(self,name,size):
+    def __init__(self, windows, name, size):
         super(Line, self).__init__()
-        self.name=name
+        self.name = name
-        self.size=size
+        self.size = size
-    def __init__(self):
+        self.win = windows
+
+    def __init__(self, windows):
         super(Line, self).__init__()
-        self.name=str(random.random())
+        self.name = str(random.random())
-        self.size=0
+        self.size = 0
-    def __init__(self,name):
+        self.win = windows
+
+    def __init__(self, windows, name):
         super(Line, self).__init__()
-        self.name=name
+        self.name = name
-        self.size=0
+        self.size = 0
+        self.win = windows
+
+    def AppendData(self, y):
+        self.win.AppendData(self, y)
 
 class LineWin():
     def __init__(self):
@@ -35,23 +51,23 @@ class LineWin():
         )
         self.data = np.array([])
 
-    def AppendLine(self,y):
+    def AppendLine(self, y):
-        if not isinstance(y,list):
+        if not isinstance(y, list):
             y = [y]
         linename = str(random.random())
         self.viz.line(
             X=np.array(range(len(y))),
             Y=np.array(y),
             opts=dict(markercolor=np.array([50]),
-                    markersymbol='dot',),
+                      markersymbol='dot',),
             win=self.win,
             update="new",
             name=linename,
         )
-        return Line(linename,len(y))
+        return Line(linename, len(y))
 
-    def AppendData(self,line, y):
+    def AppendData(self, line, y):
-        if not isinstance(y,list):
+        if not isinstance(y, list):
             y = [y]
         self.viz.line(
             X=np.array(range(line.size, line.size+len(y))),
@@ -61,4 +77,3 @@ class LineWin():
             name=line.name
         )
         line.size = line.size + len(y)
-

visdom.server.log

@@ -0,0 +1,25 @@
+nohup: ignoring input
+/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops.
+  ioloop.install()  # Needs to happen before any tornado imports!
+Traceback (most recent call last):
+  File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main
+    "__main__", mod_spec)
+  File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 85, in _run_code
+    exec(code, run_globals)
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1308, in <module>
+    download_scripts_and_run()
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1304, in download_scripts_and_run
+    main()
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1299, in main
+    print_func=print_func, user_credential=user_credential)
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1219, in start_server
+    app.listen(port, max_buffer_size=1024 ** 3)
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/web.py", line 2042, in listen
+    server.listen(port, address)
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/tcpserver.py", line 143, in listen
+    sockets = bind_sockets(port, address=address)
+  File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets
+    sock.bind(sockaddr)
+OSError: [Errno 98] Address already in use
+Checking for scripts.
+It's Alive!