diff --git a/FilterEvaluator/Evaluator.py b/FilterEvaluator/Evaluator.py index 55a2490..b4e5ee6 100644 --- a/FilterEvaluator/Evaluator.py +++ b/FilterEvaluator/Evaluator.py @@ -60,7 +60,7 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) # weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10) # np.save("WeightSearch.npy", weight) -# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata) +# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata, NumTrain=5000) # np.save("WeightTrain.npy", weight) @@ -72,20 +72,21 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) # utils.NumpyToImage(bestweight, CurrentPath+"image") -# weight = np.load(CurrentPath+"WeightTrain.npy") -# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000) -# np.save(CurrentPath+"bestweightTrain.npy", bestweight) -# utils.NumpyToImage(bestweight, CurrentPath+"image") +weight = np.load(CurrentPath+"WeightTrain.npy") +bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight( + model, layer, weight, traindata, databatchs=128, interation=100000) +np.save(CurrentPath+"bestweightTrain.npy", bestweight) +utils.NumpyToImage(bestweight, CurrentPath+"image") -weight = np.load(CurrentPath+"bestweightSearch.npy") -EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) -utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl") +# weight = np.load(CurrentPath+"bestweightSearch.npy") +# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) +# utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl") -weight = np.load(CurrentPath+"bestweightTrain.npy") -EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) -utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl") +# weight = np.load(CurrentPath+"bestweightTrain.npy") +# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) +# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl") diff --git a/FilterEvaluator/EvaluatorUnsuper.py b/FilterEvaluator/EvaluatorUnsuper.py index 3235ecc..a25bcfa 100644 --- a/FilterEvaluator/EvaluatorUnsuper.py +++ b/FilterEvaluator/EvaluatorUnsuper.py @@ -14,6 +14,8 @@ from torch.utils.data import Dataset, DataLoader from PIL import Image import random import cv2 +from tqdm import tqdm + CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" sys.path.append(CurrentPath+'../tools') @@ -91,7 +93,6 @@ def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChan tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels])) return minweight - def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100): netmodel.eval() sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0))) @@ -133,8 +134,8 @@ def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannel print("search :" + str(i)) return minweight - def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=128,interation=10000): + interationbar = tqdm(total=interation) weight = weight.astype("float32") netmodel.eval() tl = netmodel.features[layer] @@ -155,28 +156,26 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1 for i in range(outchannels): shift.append(1< mean - posi = posi*shift - sums = posi.sum(1).type(torch.float32) + outputs = newlayer(datasnetout).transpose(0,1) + outputs = outputs.reshape(tl.out_channels,-1).transpose(0,1) + sums = ((outputs > outputs.mean(0)) * shift).sum(1).type(torch.float32) histc = sums.histc(256,0,255).type(torch.float32) histc = histc[histc>0] histc = histc/histc.sum() - entropys.append((histc.log2()*histc).sum()) - print("search index : " + str(i)) + entropy = (histc.log2()*histc).sum() + entropys.append(entropy.detach().cpu().numpy()) + interationbar.update(1) argmin = np.argmin(entropys) bestweight = weight[indexs[argmin]] + interationbar.close() return bestweight - def SetModelConvWeight(model, layer, weight): w = utils.SetDevice(torch.from_numpy(weight)) - model.features[layer].weight.data = w \ No newline at end of file + model.features[layer].weight.data = w diff --git a/FilterEvaluator/TrainNetwork.py b/FilterEvaluator/TrainNetwork.py index ab7c68b..d50903a 100644 --- a/FilterEvaluator/TrainNetwork.py +++ b/FilterEvaluator/TrainNetwork.py @@ -15,6 +15,7 @@ from PIL import Image import random import cv2 from visdom import Visdom +from tqdm import tqdm CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" @@ -36,45 +37,44 @@ batchsize = 128 # model = utils.SetDevice(Model.Net31535()) # optimizer = optim.Adam(model.parameters(), lr=0.01) +# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl") + # traindata, testdata = Loader.MNIST(batchsize, num_workers=4) # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra") -traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500) +traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140) -# WebVisual.InitVisdom() +WebVisual.InitVisdom() window = WebVisual.LineWin() -lineNoPre = WebVisual.Line("NoPre") -linePretrain = WebVisual.Line("Pretrain") +lineNoPre = WebVisual.Line(window, "NoPre") +linePretrainSearch = WebVisual.Line(window, "PretrainSearch") +linePretrainTrain = WebVisual.Line(window, "PretrainTrain") - - -# model = utils.SetDevice(Model.Net3335()) model = utils.SetDevice(Model.Net3Grad335()) -model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") - +model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl") optimizer = optim.SGD(model.parameters(), lr=0.1) -for i in range(1000): - Train.train(model,traindata,optimizer,epoch=i) - window.AppendData(linePretrain,Train.test(model,testdata)) - - +Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain) model = utils.SetDevice(Model.Net3335()) -model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") - +model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl") optimizer = optim.SGD(model.parameters(), lr=0.1) -for i in range(1000): - Train.train(model, traindata, optimizer, epoch=i) - window.AppendData(lineNoPre, Train.test(model, testdata)) +Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre) + + +model = utils.SetDevice(Model.Net3Grad335()) +model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl") +optimizer = optim.SGD(model.parameters(), lr=0.1) +Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch) + diff --git a/FilterEvaluator/visdom.server.log b/FilterEvaluator/visdom.server.log new file mode 100644 index 0000000..35d93f2 Binary files /dev/null and b/FilterEvaluator/visdom.server.log differ diff --git a/tools/Train.py b/tools/Train.py index 17b3bd0..628d1cb 100644 --- a/tools/Train.py +++ b/tools/Train.py @@ -9,12 +9,14 @@ import torch.nn as nn import torch import os import utils as utils +from tqdm import tqdm def train(model, train_loader, optimizer, epoch=0): model.train() batchsize = int(train_loader.sampler.num_samples / train_loader.batch_size / 5)+1 + pbar = tqdm(total=train_loader.sampler.num_samples) for batch_idx, (data, target) in enumerate(train_loader): data = utils.SetDevice(data) target = utils.SetDevice(target) @@ -25,13 +27,16 @@ def train(model, train_loader, optimizer, epoch=0): loss.backward() optimizer.step() + pbar.update(train_loader.batch_size) if batch_idx % batchsize == 0 and batch_idx > 0: - print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}' - .format(epoch, batch_idx * len(data), - len(train_loader.dataset), - 100. * batch_idx / - len(train_loader), - loss.item())) + pbar.set_description("Loss:"+str(loss.item())) + # print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}' + # .format(epoch, batch_idx * len(data), + # len(train_loader.dataset), + # 100. * batch_idx / + # len(train_loader), + # loss.item())) + pbar.close() def test(model, test_loader): @@ -43,15 +48,23 @@ def test(model, test_loader): data = utils.SetDevice(data) target = utils.SetDevice(target) output = model(data) - # sum up batch loss test_loss += F.nll_loss(output, target, reduction='sum').item() # get the index of the max log-probability pred = output.max(1, keepdim=True)[1] correct += pred.eq(target.view_as(pred)).sum().item() - test_loss /= len(test_loader.dataset) accu = 100. * correct / len(test_loader.dataset) - print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n' - .format(test_loss, correct, len(test_loader.dataset), accu)) + # print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n' + # .format(test_loss, correct, len(test_loader.dataset), accu)) return accu + + +def TrainEpochs(model, traindata, optimizer, testdata, epoch=100,testepoch=10, line=None): + epochbar = tqdm(total=epoch) + for i in range(epoch): + train(model, traindata, optimizer, epoch=i) + if line and i % testepoch == 0 and i > 0: + line.AppendData(test(model, testdata)) + epochbar.update(1) + epochbar.close() diff --git a/tools/WebVisual.py b/tools/WebVisual.py index 31bbe32..7ca9870 100644 --- a/tools/WebVisual.py +++ b/tools/WebVisual.py @@ -1,27 +1,43 @@ from visdom import Visdom import random import numpy as np - +import os +import time def InitVisdom(): - viz = Visdom() - assert viz.check_connection() - viz.close() + # nohub python3 -m visdom.server > visdom.server.log 2 > &1 + # subprocess.Popen("python3 -m visdom.server > visdom.server.log 2>&1", shell=True) + + os.popen("python3 -m visdom.server > visdom.server.log 2>&1") + time.sleep(3) # to wait vindom.server started + # viz = Visdom() + # assert viz.check_connection() + # viz.close() + print("Connect http://localhost:8097") + class Line(): - def __init__(self,name,size): + def __init__(self, windows, name, size): super(Line, self).__init__() - self.name=name - self.size=size - def __init__(self): + self.name = name + self.size = size + self.win = windows + + def __init__(self, windows): super(Line, self).__init__() - self.name=str(random.random()) - self.size=0 - def __init__(self,name): + self.name = str(random.random()) + self.size = 0 + self.win = windows + + def __init__(self, windows, name): super(Line, self).__init__() - self.name=name - self.size=0 + self.name = name + self.size = 0 + self.win = windows + + def AppendData(self, y): + self.win.AppendData(self, y) class LineWin(): def __init__(self): @@ -34,24 +50,24 @@ class LineWin(): name=self.name ) self.data = np.array([]) - - def AppendLine(self,y): - if not isinstance(y,list): + + def AppendLine(self, y): + if not isinstance(y, list): y = [y] linename = str(random.random()) self.viz.line( X=np.array(range(len(y))), Y=np.array(y), opts=dict(markercolor=np.array([50]), - markersymbol='dot',), + markersymbol='dot',), win=self.win, update="new", name=linename, ) - return Line(linename,len(y)) + return Line(linename, len(y)) - def AppendData(self,line, y): - if not isinstance(y,list): + def AppendData(self, line, y): + if not isinstance(y, list): y = [y] self.viz.line( X=np.array(range(line.size, line.size+len(y))), @@ -61,4 +77,3 @@ class LineWin(): name=line.name ) line.size = line.size + len(y) - diff --git a/visdom.server.log b/visdom.server.log new file mode 100644 index 0000000..9bc7cb7 --- /dev/null +++ b/visdom.server.log @@ -0,0 +1,25 @@ +nohup: ignoring input +/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops. + ioloop.install() # Needs to happen before any tornado imports! +Traceback (most recent call last): + File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main + "__main__", mod_spec) + File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 85, in _run_code + exec(code, run_globals) + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1308, in + download_scripts_and_run() + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1304, in download_scripts_and_run + main() + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1299, in main + print_func=print_func, user_credential=user_credential) + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1219, in start_server + app.listen(port, max_buffer_size=1024 ** 3) + File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/web.py", line 2042, in listen + server.listen(port, address) + File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/tcpserver.py", line 143, in listen + sockets = bind_sockets(port, address=address) + File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets + sock.bind(sockaddr) +OSError: [Errno 98] Address already in use +Checking for scripts. +It's Alive!