Refine find best 8
This commit is contained in:
colin
parent
f858ae1821
commit
b0246d2a95
|
|
@ -60,7 +60,7 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
|
|||
# weight = EvaluatorUnsuper.UnsuperLearnSearchWeight(model, layer, traindata, NumSearch=100000, SearchChannelRatio=8, Interation=10)
|
||||
# np.save("WeightSearch.npy", weight)
|
||||
|
||||
# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata)
|
||||
# weight = EvaluatorUnsuper.UnsuperLearnTrainWeight(model, layer, traindata, NumTrain=5000)
|
||||
# np.save("WeightTrain.npy", weight)
|
||||
|
||||
|
||||
|
|
@ -72,20 +72,21 @@ traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True)
|
|||
# utils.NumpyToImage(bestweight, CurrentPath+"image")
|
||||
|
||||
|
||||
# weight = np.load(CurrentPath+"WeightTrain.npy")
|
||||
# bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(model,layer,weight,traindata,128,100000)
|
||||
# np.save(CurrentPath+"bestweightTrain.npy", bestweight)
|
||||
# utils.NumpyToImage(bestweight, CurrentPath+"image")
|
||||
weight = np.load(CurrentPath+"WeightTrain.npy")
|
||||
bestweight = EvaluatorUnsuper.UnsuperLearnFindBestWeight(
|
||||
model, layer, weight, traindata, databatchs=128, interation=100000)
|
||||
np.save(CurrentPath+"bestweightTrain.npy", bestweight)
|
||||
utils.NumpyToImage(bestweight, CurrentPath+"image")
|
||||
|
||||
|
||||
|
||||
weight = np.load(CurrentPath+"bestweightSearch.npy")
|
||||
EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
|
||||
# weight = np.load(CurrentPath+"bestweightSearch.npy")
|
||||
# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
# utils.SaveModel(model,CurrentPath+"/checkpointSearch.pkl")
|
||||
|
||||
weight = np.load(CurrentPath+"bestweightTrain.npy")
|
||||
EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
|
||||
# weight = np.load(CurrentPath+"bestweightTrain.npy")
|
||||
# EvaluatorUnsuper.SetModelConvWeight(model,layer,weight)
|
||||
# utils.SaveModel(model,CurrentPath+"/checkpointTrain.pkl")
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ from torch.utils.data import Dataset, DataLoader
|
|||
from PIL import Image
|
||||
import random
|
||||
import cv2
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
|
||||
sys.path.append(CurrentPath+'../tools')
|
||||
|
|
@ -91,7 +93,6 @@ def UnsuperLearnSearchWeight(model, layer, dataloader, NumSearch=10000, SaveChan
|
|||
tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels]))
|
||||
return minweight
|
||||
|
||||
|
||||
def TrainLayer(netmodel, layer, SearchLayer, DataSet, Epoch=100):
|
||||
netmodel.eval()
|
||||
sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0)))
|
||||
|
|
@ -133,8 +134,8 @@ def UnsuperLearnTrainWeight(model, layer, dataloader, NumTrain=500, TrainChannel
|
|||
print("search :" + str(i))
|
||||
return minweight
|
||||
|
||||
|
||||
def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=128,interation=10000):
|
||||
interationbar = tqdm(total=interation)
|
||||
weight = weight.astype("float32")
|
||||
netmodel.eval()
|
||||
tl = netmodel.features[layer]
|
||||
|
|
@ -155,28 +156,26 @@ def UnsuperLearnFindBestWeight(netmodel, layer, weight, dataloader, databatchs=1
|
|||
for i in range(outchannels):
|
||||
shift.append(1<<i)
|
||||
shift = utils.SetDevice(torch.from_numpy(np.array(shift).astype("uint8")))
|
||||
datasnetout = []
|
||||
for d in datas:
|
||||
datasnetout.append(netmodel.ForwardLayer(d,forwardlayer))
|
||||
datasnetout = torch.cat(datasnetout)
|
||||
for i in range(len(indexs)):
|
||||
newlayer.weight.data=utils.SetDevice(torch.from_numpy(weight[indexs[i]]))
|
||||
outputs = []
|
||||
for d in datas:
|
||||
outputs.append(newlayer(netmodel.ForwardLayer(d,forwardlayer)))
|
||||
outputs = torch.cat(outputs).transpose(0,1)
|
||||
outputs = outputs.reshape(outputs.shape[0],-1)
|
||||
mean = outputs.mean(1)
|
||||
outputs = outputs.transpose(0,1)
|
||||
posi = outputs > mean
|
||||
posi = posi*shift
|
||||
sums = posi.sum(1).type(torch.float32)
|
||||
outputs = newlayer(datasnetout).transpose(0,1)
|
||||
outputs = outputs.reshape(tl.out_channels,-1).transpose(0,1)
|
||||
sums = ((outputs > outputs.mean(0)) * shift).sum(1).type(torch.float32)
|
||||
histc = sums.histc(256,0,255).type(torch.float32)
|
||||
histc = histc[histc>0]
|
||||
histc = histc/histc.sum()
|
||||
entropys.append((histc.log2()*histc).sum())
|
||||
print("search index : " + str(i))
|
||||
entropy = (histc.log2()*histc).sum()
|
||||
entropys.append(entropy.detach().cpu().numpy())
|
||||
interationbar.update(1)
|
||||
argmin = np.argmin(entropys)
|
||||
bestweight = weight[indexs[argmin]]
|
||||
interationbar.close()
|
||||
return bestweight
|
||||
|
||||
|
||||
def SetModelConvWeight(model, layer, weight):
|
||||
w = utils.SetDevice(torch.from_numpy(weight))
|
||||
model.features[layer].weight.data = w
|
||||
model.features[layer].weight.data = w
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from PIL import Image
|
|||
import random
|
||||
import cv2
|
||||
from visdom import Visdom
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
|
||||
|
|
@ -36,45 +37,44 @@ batchsize = 128
|
|||
# model = utils.SetDevice(Model.Net31535())
|
||||
# optimizer = optim.Adam(model.parameters(), lr=0.01)
|
||||
|
||||
# model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
|
||||
|
||||
|
||||
# traindata, testdata = Loader.MNIST(batchsize, num_workers=4)
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalOneLine")
|
||||
# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="VerticalZebra")
|
||||
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500)
|
||||
traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=140)
|
||||
|
||||
|
||||
# WebVisual.InitVisdom()
|
||||
WebVisual.InitVisdom()
|
||||
window = WebVisual.LineWin()
|
||||
lineNoPre = WebVisual.Line("NoPre")
|
||||
linePretrain = WebVisual.Line("Pretrain")
|
||||
lineNoPre = WebVisual.Line(window, "NoPre")
|
||||
linePretrainSearch = WebVisual.Line(window, "PretrainSearch")
|
||||
linePretrainTrain = WebVisual.Line(window, "PretrainTrain")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# model = utils.SetDevice(Model.Net3335())
|
||||
model = utils.SetDevice(Model.Net3Grad335())
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
|
||||
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
|
||||
optimizer = optim.SGD(model.parameters(), lr=0.1)
|
||||
for i in range(1000):
|
||||
Train.train(model,traindata,optimizer,epoch=i)
|
||||
window.AppendData(linePretrain,Train.test(model,testdata))
|
||||
|
||||
|
||||
Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainTrain)
|
||||
|
||||
|
||||
model = utils.SetDevice(Model.Net3335())
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
|
||||
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpointTrain.pkl")
|
||||
optimizer = optim.SGD(model.parameters(), lr=0.1)
|
||||
for i in range(1000):
|
||||
Train.train(model, traindata, optimizer, epoch=i)
|
||||
window.AppendData(lineNoPre, Train.test(model, testdata))
|
||||
Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,lineNoPre)
|
||||
|
||||
|
||||
model = utils.SetDevice(Model.Net3Grad335())
|
||||
model = utils.LoadModel(model, CurrentPath+"/checkpointSearch.pkl")
|
||||
optimizer = optim.SGD(model.parameters(), lr=0.1)
|
||||
Train.TrainEpochs(model,traindata,optimizer,testdata,3000,30,linePretrainSearch)
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -9,12 +9,14 @@ import torch.nn as nn
|
|||
import torch
|
||||
import os
|
||||
import utils as utils
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def train(model, train_loader, optimizer, epoch=0):
|
||||
model.train()
|
||||
batchsize = int(train_loader.sampler.num_samples /
|
||||
train_loader.batch_size / 5)+1
|
||||
pbar = tqdm(total=train_loader.sampler.num_samples)
|
||||
for batch_idx, (data, target) in enumerate(train_loader):
|
||||
data = utils.SetDevice(data)
|
||||
target = utils.SetDevice(target)
|
||||
|
|
@ -25,13 +27,16 @@ def train(model, train_loader, optimizer, epoch=0):
|
|||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
pbar.update(train_loader.batch_size)
|
||||
if batch_idx % batchsize == 0 and batch_idx > 0:
|
||||
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'
|
||||
.format(epoch, batch_idx * len(data),
|
||||
len(train_loader.dataset),
|
||||
100. * batch_idx /
|
||||
len(train_loader),
|
||||
loss.item()))
|
||||
pbar.set_description("Loss:"+str(loss.item()))
|
||||
# print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'
|
||||
# .format(epoch, batch_idx * len(data),
|
||||
# len(train_loader.dataset),
|
||||
# 100. * batch_idx /
|
||||
# len(train_loader),
|
||||
# loss.item()))
|
||||
pbar.close()
|
||||
|
||||
|
||||
def test(model, test_loader):
|
||||
|
|
@ -43,15 +48,23 @@ def test(model, test_loader):
|
|||
data = utils.SetDevice(data)
|
||||
target = utils.SetDevice(target)
|
||||
output = model(data)
|
||||
|
||||
# sum up batch loss
|
||||
test_loss += F.nll_loss(output, target, reduction='sum').item()
|
||||
# get the index of the max log-probability
|
||||
pred = output.max(1, keepdim=True)[1]
|
||||
correct += pred.eq(target.view_as(pred)).sum().item()
|
||||
|
||||
test_loss /= len(test_loader.dataset)
|
||||
accu = 100. * correct / len(test_loader.dataset)
|
||||
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'
|
||||
.format(test_loss, correct, len(test_loader.dataset), accu))
|
||||
# print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'
|
||||
# .format(test_loss, correct, len(test_loader.dataset), accu))
|
||||
return accu
|
||||
|
||||
|
||||
def TrainEpochs(model, traindata, optimizer, testdata, epoch=100,testepoch=10, line=None):
|
||||
epochbar = tqdm(total=epoch)
|
||||
for i in range(epoch):
|
||||
train(model, traindata, optimizer, epoch=i)
|
||||
if line and i % testepoch == 0 and i > 0:
|
||||
line.AppendData(test(model, testdata))
|
||||
epochbar.update(1)
|
||||
epochbar.close()
|
||||
|
|
|
|||
|
|
@ -1,27 +1,43 @@
|
|||
from visdom import Visdom
|
||||
import random
|
||||
import numpy as np
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
|
||||
def InitVisdom():
|
||||
viz = Visdom()
|
||||
assert viz.check_connection()
|
||||
viz.close()
|
||||
# nohub python3 -m visdom.server > visdom.server.log 2 > &1
|
||||
# subprocess.Popen("python3 -m visdom.server > visdom.server.log 2>&1", shell=True)
|
||||
|
||||
os.popen("python3 -m visdom.server > visdom.server.log 2>&1")
|
||||
time.sleep(3) # to wait vindom.server started
|
||||
# viz = Visdom()
|
||||
# assert viz.check_connection()
|
||||
# viz.close()
|
||||
print("Connect http://localhost:8097")
|
||||
|
||||
|
||||
class Line():
|
||||
def __init__(self,name,size):
|
||||
def __init__(self, windows, name, size):
|
||||
super(Line, self).__init__()
|
||||
self.name=name
|
||||
self.size=size
|
||||
def __init__(self):
|
||||
self.name = name
|
||||
self.size = size
|
||||
self.win = windows
|
||||
|
||||
def __init__(self, windows):
|
||||
super(Line, self).__init__()
|
||||
self.name=str(random.random())
|
||||
self.size=0
|
||||
def __init__(self,name):
|
||||
self.name = str(random.random())
|
||||
self.size = 0
|
||||
self.win = windows
|
||||
|
||||
def __init__(self, windows, name):
|
||||
super(Line, self).__init__()
|
||||
self.name=name
|
||||
self.size=0
|
||||
self.name = name
|
||||
self.size = 0
|
||||
self.win = windows
|
||||
|
||||
def AppendData(self, y):
|
||||
self.win.AppendData(self, y)
|
||||
|
||||
class LineWin():
|
||||
def __init__(self):
|
||||
|
|
@ -34,24 +50,24 @@ class LineWin():
|
|||
name=self.name
|
||||
)
|
||||
self.data = np.array([])
|
||||
|
||||
def AppendLine(self,y):
|
||||
if not isinstance(y,list):
|
||||
|
||||
def AppendLine(self, y):
|
||||
if not isinstance(y, list):
|
||||
y = [y]
|
||||
linename = str(random.random())
|
||||
self.viz.line(
|
||||
X=np.array(range(len(y))),
|
||||
Y=np.array(y),
|
||||
opts=dict(markercolor=np.array([50]),
|
||||
markersymbol='dot',),
|
||||
markersymbol='dot',),
|
||||
win=self.win,
|
||||
update="new",
|
||||
name=linename,
|
||||
)
|
||||
return Line(linename,len(y))
|
||||
return Line(linename, len(y))
|
||||
|
||||
def AppendData(self,line, y):
|
||||
if not isinstance(y,list):
|
||||
def AppendData(self, line, y):
|
||||
if not isinstance(y, list):
|
||||
y = [y]
|
||||
self.viz.line(
|
||||
X=np.array(range(line.size, line.size+len(y))),
|
||||
|
|
@ -61,4 +77,3 @@ class LineWin():
|
|||
name=line.name
|
||||
)
|
||||
line.size = line.size + len(y)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
nohup: ignoring input
|
||||
/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py:30: DeprecationWarning: zmq.eventloop.ioloop is deprecated in pyzmq 17. pyzmq now works with default tornado and asyncio eventloops.
|
||||
ioloop.install() # Needs to happen before any tornado imports!
|
||||
Traceback (most recent call last):
|
||||
File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 193, in _run_module_as_main
|
||||
"__main__", mod_spec)
|
||||
File "/home/colin/anaconda3/lib/python3.7/runpy.py", line 85, in _run_code
|
||||
exec(code, run_globals)
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1308, in <module>
|
||||
download_scripts_and_run()
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1304, in download_scripts_and_run
|
||||
main()
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1299, in main
|
||||
print_func=print_func, user_credential=user_credential)
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/visdom/server.py", line 1219, in start_server
|
||||
app.listen(port, max_buffer_size=1024 ** 3)
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/web.py", line 2042, in listen
|
||||
server.listen(port, address)
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/tcpserver.py", line 143, in listen
|
||||
sockets = bind_sockets(port, address=address)
|
||||
File "/home/colin/anaconda3/lib/python3.7/site-packages/tornado/netutil.py", line 168, in bind_sockets
|
||||
sock.bind(sockaddr)
|
||||
OSError: [Errno 98] Address already in use
|
||||
Checking for scripts.
|
||||
It's Alive!
|
||||
Loading…
Reference in New Issue