From b0246d2a9516993fc65404f81d9d935e7fc1e190 Mon Sep 17 00:00:00 2001 From: colin Date: Tue, 17 Sep 2019 23:20:24 +0800 Subject: [PATCH] Refine find best 8 --- FilterEvaluator/Evaluator.py | 23 +++++------ FilterEvaluator/EvaluatorUnsuper.py | 31 ++++++++------- FilterEvaluator/TrainNetwork.py | 38 +++++++++---------- FilterEvaluator/visdom.server.log | Bin 0 -> 3818261 bytes tools/Train.py | 33 +++++++++++----- tools/WebVisual.py | 57 ++++++++++++++++++---------- visdom.server.log | 25 ++++++++++++ 7 files changed, 130 insertions(+), 77 deletions(-) create mode 100644 FilterEvaluator/visdom.server.log create mode 100644 visdom.server.log diff --git a/FilterEvaluator/Evaluator.py b/FilterEvaluator/Evaluator.py index 55a2490..b4e5ee6 100644 --- a/FilterEvaluator/Evaluator.py +++ b/FilterEvaluator/Evaluator.py @@ -60,7 +60,7 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) # weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10) # np.save("WeightSearch.npy", weight) -# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata) +# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata, NumTrain=5000) # np.save("WeightTrain.npy", weight) @@ -72,20 +72,21 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) # utils.NumpyToImage(bestweight, CurrentPath+"image") -# weight = np.load(CurrentPath+"WeightTrain.npy") -# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000) -# np.save(CurrentPath+"bestweightTrain.npy", bestweight) -# utils.NumpyToImage(bestweight, CurrentPath+"image") +weight = np.load(CurrentPath+"WeightTrain.npy") +bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight( + model, layer, weight, traindata, databatchs=128, interation=100000) +np.save(CurrentPath+"bestweightTrain.npy", bestweight) +utils.NumpyToImage(bestweight, CurrentPath+"image") -weight = np.load(CurrentPath+"bestweightSearch.npy") -EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) -utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl") +# weight = np.load(CurrentPath+"bestweightSearch.npy") +# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) +# utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl") -weight = np.load(CurrentPath+"bestweightTrain.npy") -EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) -utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl") +# weight = np.load(CurrentPath+"bestweightTrain.npy") +# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight) +# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl") diff --git a/FilterEvaluator/EvaluatorUnsuper.py b/FilterEvaluator/EvaluatorUnsuper.py index 3235ecc..a25bcfa 100644 --- a/FilterEvaluator/EvaluatorUnsuper.py +++ b/FilterEvaluator/EvaluatorUnsuper.py @@ -14,6 +14,8 @@ from torch.utils.data import Dataset, DataLoader from PIL import Image import random import cv2 +from tqdm import tqdm + CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" sys.path.append(CurrentPath+'../tools') @@ -91,7 +93,6 @@ def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChan tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels])) return minweight - def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100): netmodel.eval() sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0))) @@ -133,8 +134,8 @@ def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannel print("search :" + str(i)) return minweight - def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=128,interation=10000): + interationbar = tqdm(total=interation) weight = weight.astype("float32") netmodel.eval() tl = netmodel.features[layer] @@ -155,28 +156,26 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1 for i in range(outchannels): shift.append(1< mean - posi = posi*shift - sums = posi.sum(1).type(torch.float32) + outputs = newlayer(datasnetout).transpose(0,1) + outputs = outputs.reshape(tl.out_channels,-1).transpose(0,1) + sums = ((outputs > outputs.mean(0)) * shift).sum(1).type(torch.float32) histc = sums.histc(256,0,255).type(torch.float32) histc = histc[histc>0] histc = histc/histc.sum() - entropys.append((histc.log2()*histc).sum()) - print("search index : " + str(i)) + entropy = (histc.log2()*histc).sum() + entropys.append(entropy.detach().cpu().numpy()) + interationbar.update(1) argmin = np.argmin(entropys) bestweight = weight[indexs[argmin]] + interationbar.close() return bestweight - def SetModelConvWeight(model, layer, weight): w = utils.SetDevice(torch.from_numpy(weight)) - model.features[layer].weight.data = w \ No newline at end of file + model.features[layer].weight.data = w diff --git a/FilterEvaluator/TrainNetwork.py b/FilterEvaluator/TrainNetwork.py index ab7c68b..d50903a 100644 --- a/FilterEvaluator/TrainNetwork.py +++ b/FilterEvaluator/TrainNetwork.py @@ -15,6 +15,7 @@ from PIL import Image import random import cv2 from visdom import Visdom +from tqdm import tqdm CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/" @@ -36,45 +37,44 @@ batchsize = 128 # model = utils.SetDevice(Model.Net31535()) # optimizer = optim.Adam(model.parameters(), lr=0.01) +# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl") + # traindata, testdata = Loader.MNIST(batchsize, num_workers=4) # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra") -traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500) +traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140) -# WebVisual.InitVisdom() +WebVisual.InitVisdom() window = WebVisual.LineWin() -lineNoPre = WebVisual.Line("NoPre") -linePretrain = WebVisual.Line("Pretrain") +lineNoPre = WebVisual.Line(window, "NoPre") +linePretrainSearch = WebVisual.Line(window, "PretrainSearch") +linePretrainTrain = WebVisual.Line(window, "PretrainTrain") - - -# model = utils.SetDevice(Model.Net3335()) model = utils.SetDevice(Model.Net3Grad335()) -model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") - +model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl") optimizer = optim.SGD(model.parameters(), lr=0.1) -for i in range(1000): - Train.train(model,traindata,optimizer,epoch=i) - window.AppendData(linePretrain,Train.test(model,testdata)) - - +Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain) model = utils.SetDevice(Model.Net3335()) -model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") - +model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl") optimizer = optim.SGD(model.parameters(), lr=0.1) -for i in range(1000): - Train.train(model, traindata, optimizer, epoch=i) - window.AppendData(lineNoPre, Train.test(model, testdata)) +Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre) + + +model = utils.SetDevice(Model.Net3Grad335()) +model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl") +optimizer = optim.SGD(model.parameters(), lr=0.1) +Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch) + diff --git a/FilterEvaluator/visdom.server.log b/FilterEvaluator/visdom.server.log new file mode 100644 index 0000000000000000000000000000000000000000..35d93f293761078238c73dcfc794d8bfb2c213f2 GIT binary patch literal 3818261 zcmeF)U2hvlmH^Ow_OIy4KCC^+QlhADtYLyubuAz%B*?O}1^0rEV@s$@cvE zZArG|*xfG8y$*=Mp5(z&qEc0zx^?S=O?u~5dr|Mq+Ve%TGi|1`wwX^SJLik%JIi;Q zS8X#H9qp_aoBCgu)7k6k%X+S6urkh3E{CT=+ z7R}4k>TehSJF2hhW^>-Q%h95}{-au~tNHt%t>@LEsh01$pR2}4qxWAm?VIXNyL!E@ z-Yhn+x*vWqy*%Gkn|9Sq=WR7@=GAolu9+>`>h71XM~|xNwt~^3S#PH2=Z~LM)jwB1 z)%AScm91V)m&>}Tp4TthRo(rPcXz*Vaj|SyoAqxV{k)pa>gQcAtH&4ZdQ)}%>Dpe+ zx?fhEPuH6#r(GrAFV5@go1F)stuCA8yN>6ZZ>!F>uEr;mZ?F4%*8PvOi*|l_UY||Z zXBX2&)BWD6`sVEB>$9`2K=;4T*2{YKBvs?_!F`3Zc3$69Q@^cekFUS`wz_V}%k%d6 z^nCsIRrKX}xa;s__xPp(|90Ek-`&(dZ{IZM?X>IPY_(W!)@R*-UXwijm71`B(}bZ~ zZ<=&H1iw!8dnYG1)!z;BampM3wT(Cm$G z+t$sN)#mJWHs5e`Gi7wXSa0g)@%7B;qH+56?D^%37hN0Ii@()R$Gdy`)o*`WO)~9z zKc9Er)XzVfBzwF2dpBaY4Y+B~`$Y4LCQaw_RlQDy?S}?zX3M)&b{D$)w{!L5R1)9W>Qgw5+b1_{sUFp4}(Qfze_(`=pn(Pmh z9Pf^AtGimP&(`hib-nqd=-%;%qN}#uoIYFDO+Bx=AD_1u)%y}JR^8W&sg?JbAD6lYJ^Z_d;?vRx zM<1H|Nl&i_@@Dj3Os^KRwizv(m!Ac~-e@u&D1R3{_1jI|tQT#w-uZF5dG%x6T#o*_ z{`WsU-5VW^#-IJl$!Idvp`l;--)(#GrC&K79qtXZ@;+SOESj_W?PC2|EDZMWzU<|4 zKHb!x#L)g-wD!o}PLaVn?<0M%Y=50Ylk2{%gLUqIDH3km+gs=1Bw-t@^L~;I`V97S zFIeY(CWh{k!eD!QZ8QCxl*gsF&i$e3_kb80oVxdAd+R*djmJ=Ut|a;yt}~GnCczjw z%;bcj>|PkpOupR<_H%z|&AxBzU``0nmGT@LtTUX4GBH%nVg2(nVY?rVp+wwHg8h8J z8aOu0&Wz4P-hF+Zd0%5k}^gK-<(Q>XR` z!5B)dpXK$TcRY{E9$~mleJX1L# zy1y-#8=fn{H9Ip`qT`v__l5UE3EOa;ALO9*XWwGIxyIxF3q{HB)h0j)nd_ zRL);tJ%%!S>S)~lquEZ%XS&|;Jj~oNmSdr}&diz}o}Xo(!MJ_E+S}XDqw-zzV4cyp z&766|b(Yuq;eKu*hNAai2U|D`4nA*kf5-FHZBzT(;JUJn9G{vi+lbp>od?^ziiaMu4L}sg7+pz<+cvyq3AtzD&Gd@%6{Tp7Tud4%4=_L+@2)zZS;O9b8qs1XCQm~ zd6cm&@5g)V%+NUio;qq&GDJNNqEi9%$4AI)ftW3a=F3rjOM;CezH^P z^M|5yCDYF>>`k_^egw@9&+(O)@Y|CrX*Uz`XxZO$~D*FuQ+j1=Q z+Gfs3(X}jN8@)3=&fIH8W2jtics$Fo&^w-~_r3?~JTAxHU_T%5`d)88PZDR|@V!Z* z&gf@wY$M;M_SC`sQ0AG!=<9+fTR4|xo^6huSGsdqd0r3B&)EIYxckiA#AhQ0V<`1H zZ#jqc=AqR6P_)i+x#96F-=X#P^8tB!u+H#4ArZG**;AL>+iRPNq3|=m<@@cgKAxEv ziarx@T#onQewOE1|9F5ruq;lEbxXrxBBHYjNydI39@I6@O%v)}6zinn+DbKUsI#cUsaId+I7|P_na30!1 z3}v3R3-9~NxnyvDMxVdUJP#IqKX2-Fq+m`+y|=b}M(@oDsl8@-&)8pQ=6)#J&vILb z<2IE0y8DDgPI$oQ>-5`}*M`Ac}Vdq{?fLeL`k$5@}mGSCe|Jqiob4x0&$_=7h{0M)ZBIP+rU%Ike^otbApVt*f9^7;63EcEs>bKe&}+n49{VD1aoS&qHII>Tp&a_kM)8JeHF zqs-k~c+JkNE75)q)j8SSum14e&(%)#czm)q8Xq2a?*SPd9Y3iiqruI^cz1kTXx+|U z*PG9Z?vC!ya;vu8oIYFDO+Bxg`c3t|$ct5bQ7x9I)h}v&^utfzKReyDt7bZHN7LD? zUawE@|5}>5x!SpyE}EVSAMk?x7fOHe 0: - print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}' - .format(epoch, batch_idx * len(data), - len(train_loader.dataset), - 100. * batch_idx / - len(train_loader), - loss.item())) + pbar.set_description("Loss:"+str(loss.item())) + # print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}' + # .format(epoch, batch_idx * len(data), + # len(train_loader.dataset), + # 100. * batch_idx / + # len(train_loader), + # loss.item())) + pbar.close() def test(model, test_loader): @@ -43,15 +48,23 @@ def test(model, test_loader): data = utils.SetDevice(data) target = utils.SetDevice(target) output = model(data) - # sum up batch loss test_loss += F.nll_loss(output, target, reduction='sum').item() # get the index of the max log-probability pred = output.max(1, keepdim=True)[1] correct += pred.eq(target.view_as(pred)).sum().item() - test_loss /= len(test_loader.dataset) accu = 100. * correct / len(test_loader.dataset) - print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n' - .format(test_loss, correct, len(test_loader.dataset), accu)) + # print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n' + # .format(test_loss, correct, len(test_loader.dataset), accu)) return accu + + +def TrainEpochs(model, traindata, optimizer, testdata, epoch=100,testepoch=10, line=None): + epochbar = tqdm(total=epoch) + for i in range(epoch): + train(model, traindata, optimizer, epoch=i) + if line and i % testepoch == 0 and i > 0: + line.AppendData(test(model, testdata)) + epochbar.update(1) + epochbar.close() diff --git a/tools/WebVisual.py b/tools/WebVisual.py index 31bbe32..7ca9870 100644 --- a/tools/WebVisual.py +++ b/tools/WebVisual.py @@ -1,27 +1,43 @@ from visdom import Visdom import random import numpy as np - +import os +import time def InitVisdom(): - viz = Visdom() - assert viz.check_connection() - viz.close() + # nohub python3 -m visdom.server > visdom.server.log 2 > &1 + # subprocess.Popen("python3 -m visdom.server > visdom.server.log 2>&1", shell=True) + + os.popen("python3 -m visdom.server > visdom.server.log 2>&1") + time.sleep(3) # to wait vindom.server started + # viz = Visdom() + # assert viz.check_connection() + # viz.close() + print("Connect http://localhost:8097") + class Line(): - def __init__(self,name,size): + def __init__(self, windows, name, size): super(Line, self).__init__() - self.name=name - self.size=size - def __init__(self): + self.name = name + self.size = size + self.win = windows + + def __init__(self, windows): super(Line, self).__init__() - self.name=str(random.random()) - self.size=0 - def __init__(self,name): + self.name = str(random.random()) + self.size = 0 + self.win = windows + + def __init__(self, windows, name): super(Line, self).__init__() - self.name=name - self.size=0 + self.name = name + self.size = 0 + self.win = windows + + def AppendData(self, y): + self.win.AppendData(self, y) class LineWin(): def __init__(self): @@ -34,24 +50,24 @@ class LineWin(): name=self.name ) self.data = np.array([]) - - def AppendLine(self,y): - if not isinstance(y,list): + + def AppendLine(self, y): + if not isinstance(y, list): y = [y] linename = str(random.random()) self.viz.line( X=np.array(range(len(y))), Y=np.array(y), opts=dict(markercolor=np.array([50]), - markersymbol='dot',), + markersymbol='dot',), win=self.win, update="new", name=linename, ) - return Line(linename,len(y)) + return Line(linename, len(y)) - def AppendData(self,line, y): - if not isinstance(y,list): + def AppendData(self, line, y): + if not isinstance(y, list): y = [y] self.viz.line( X=np.array(range(line.size, line.size+len(y))), @@ -61,4 +77,3 @@ class LineWin(): name=line.name ) line.size = line.size + len(y) - diff --git a/visdom.server.log b/visdom.server.log new file mode 100644 index 0000000..9bc7cb7 --- /dev/null +++ b/visdom.server.log @@ -0,0 +1,25 @@ +nohup: ignoring input +/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops. + ioloop.install() # Needs to happen before any tornado imports! +Traceback (most recent call last): + File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main + "__main__", mod_spec) + File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 85, in _run_code + exec(code, run_globals) + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1308, in + download_scripts_and_run() + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1304, in download_scripts_and_run + main() + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1299, in main + print_func=print_func, user_credential=user_credential) + File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1219, in start_server + app.listen(port, max_buffer_size=1024 ** 3) + File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/web.py", line 2042, in listen + server.listen(port, address) + File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/tcpserver.py", line 143, in listen + sockets = bind_sockets(port, address=address) + File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets + sock.bind(sockaddr) +OSError: [Errno 98] Address already in use +Checking for scripts. +It's Alive!