From 5220bf6885121a3eaf475c7b8901ceda74f4e86d Mon Sep 17 00:00:00 2001 From: colin Date: Tue, 27 Aug 2019 23:06:24 +0800 Subject: [PATCH] start to creat 8 from 4000 --- FilterEvaluator/Evaluator.py | 104 +++++++++--------- FilterEvaluator/TrainNetwork.py | 42 +++---- FilterEvaluator/checkpoint.pkl | Bin 14349 -> 14040 bytes .../image/c0_1.1933330297470093.png | Bin 69 -> 0 bytes .../image/c1_1.1936779022216797.png | Bin 69 -> 0 bytes .../image/c2_1.193914771080017.png | Bin 69 -> 0 bytes .../image/c3_1.1940335035324097.png | Bin 69 -> 0 bytes .../image/c4_1.1942846775054932.png | Bin 69 -> 0 bytes .../image/c5_1.194313645362854.png | Bin 69 -> 0 bytes .../image/c6_1.1943182945251465.png | Bin 69 -> 0 bytes .../image/c7_1.194318413734436.png | Bin 69 -> 0 bytes tools/UniModule.py | 14 ++- 12 files changed, 78 insertions(+), 82 deletions(-) delete mode 100644 FilterEvaluator/image/c0_1.1933330297470093.png delete mode 100644 FilterEvaluator/image/c1_1.1936779022216797.png delete mode 100644 FilterEvaluator/image/c2_1.193914771080017.png delete mode 100644 FilterEvaluator/image/c3_1.1940335035324097.png delete mode 100644 FilterEvaluator/image/c4_1.1942846775054932.png delete mode 100644 FilterEvaluator/image/c5_1.194313645362854.png delete mode 100644 FilterEvaluator/image/c6_1.1943182945251465.png delete mode 100644 FilterEvaluator/image/c7_1.194318413734436.png diff --git a/FilterEvaluator/Evaluator.py b/FilterEvaluator/Evaluator.py index 0a96e60..3c67c30 100644 --- a/FilterEvaluator/Evaluator.py +++ b/FilterEvaluator/Evaluator.py @@ -38,6 +38,8 @@ model = utils.SetDevice(Model.Net3Grad335()) # model = utils.SetDevice(Model.Net3()) +layers = model.PrintLayer() +layer = 0 model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") @@ -54,28 +56,34 @@ model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=0, shuffle=True) -def GetSample(netmodel,layer,dataloader,iteration=-1): +def GetSample(netmodel,layer,SearchLayer,DataSet,Interation=-1): netmodel.eval() - sample = utils.SetDevice(torch.empty((8,0))) - for batch_idx, (data, target) in enumerate(dataloader): + sample = utils.SetDevice(torch.empty((SearchLayer.out_channels,0))) + + layer = layer-1 + + # layerout = [] + # layerint = [] + # def getnet(self, input, output): + # layerout.append(output) + # layerint.append(input) + # handle = netmodel.features[layer].register_forward_hook(getnet) + # netmodel.ForwardLayer(data,layer) + # output = layerout[0][:,:,:,:] + # handle.remove() + + + for batch_idx, (data, target) in enumerate(DataSet): data = utils.SetDevice(data) target = utils.SetDevice(target) - layerout = [] - layerint = [] - def getnet(self, input, output): - layerout.append(output) - layerint.append(input) - handle = netmodel.features[layer].register_forward_hook(getnet) - netmodel.forwardLayer(data,layer=layer) - output = layerout[0][:,:,:,:] - - handle.remove() + output = netmodel.ForwardLayer(data,layer) + output = SearchLayer(output) data.detach() target.detach() - output = torch.reshape(output.transpose(0,1),(8,-1)) + output = torch.reshape(output.transpose(0,1),(SearchLayer.out_channels,-1)) sample = torch.cat((sample,output),1) - if iteration > 0 and batch_idx >= (iteration-1): + if Interation > 0 and batch_idx >= (Interation-1): break sample_mean=torch.mean(sample,dim=1,keepdim=True) @@ -104,47 +112,41 @@ def GetRandomSocre(netmodel,layer,dataloader,iteration=-1): score = GetSample(netmodel,0,dataloader,iteration) return np.array(score), newweight - -ConvKernelToImage(model, 0, CurrentPath+"image") - - - - - -layer = 0 - -weightshape = list(model.features[layer].weight.data.shape) -channels = weightshape[0] -weightshape[0] = 0 -minactive = np.empty((0)) -minweight = np.empty(weightshape) - -for i in range(300000): - score,weight = GetRandomSocre(model,layer,traindata,iteration=10) - - minactive = np.append(minactive, score) - minweight = np.concatenate((minweight, weight)) - - index = minactive.argsort() - minactive = minactive[index[0:channels]] - minweight = minweight[index[0:channels]] - - print("search random :" + str(i)) - if i % 10000 == 0: - utils.SaveModel(model, CurrentPath+"/checkpoint.pkl") +def UnsuperLearnConvWeight(model, layer, dataloader, NumSearch = 10000, ChannelRatio=1,Interation=10): + tl = model.features[layer] + newlayer = nn.Conv2d(tl.in_channels, tl.out_channels * ChannelRatio, tl.kernel_size, + tl.stride, tl.padding, tl.dilation, tl.groups, tl.bias, tl.padding_mode) + newlayer = utils.SetDevice(newlayer) + newchannels = tl.out_channels * ChannelRatio + newweightshape = list(newlayer.weight.data.shape) -# for i in range(minweight.shape[0]): -# a2 = minweight[i].reshape((minweight.shape[2],minweight.shape[3])) -# a2min = np.min(a2) -# a2max = np.max(a2) -# a2 = (a2 - a2min)*255.0/(a2max-a2min) -# a2 = a2.astype(int) -# cv2.imwrite(CurrentPath+"/image/c"+str(i)+"_"+str(minactive[i])+".png",a2) + minactive = np.empty((0)) + minweight = np.empty([0,newweightshape[1],newweightshape[2],newweightshape[3]]) + + for i in range(NumSearch): + newweight = np.random.uniform(-1.0,1.0,newweightshape).astype("float32") + newlayer.weight.data=utils.SetDevice(torch.from_numpy(newweight)) + + score = GetSample(model, layer, newlayer, dataloader, Interation) + + minactive = np.append(minactive, score) + minweight = np.concatenate((minweight, newweight)) + + index = minactive.argsort() + minactive = minactive[index[0:newchannels]] + minweight = minweight[index[0:newchannels]] + print("search random :" + str(i)) + if i % (NumSearch/10) == 0: + tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels])) + utils.SaveModel(model, CurrentPath+"/checkpoint.pkl") + tl.data=utils.SetDevice(torch.from_numpy(minweight[0:tl.out_channels])) -model.features[layer].weight.data=utils.SetDevice(torch.from_numpy(minweight)) +UnsuperLearnConvWeight(model, layer, traindata, NumSearch=100000, ChannelRatio=8, Interation=10) + +ConvKernelToImage(model, layer, CurrentPath+"image") utils.SaveModel(model,CurrentPath+"/checkpoint.pkl") print("save model sucess") diff --git a/FilterEvaluator/TrainNetwork.py b/FilterEvaluator/TrainNetwork.py index 9ad166b..1d0e74a 100644 --- a/FilterEvaluator/TrainNetwork.py +++ b/FilterEvaluator/TrainNetwork.py @@ -31,6 +31,12 @@ batchsize = 128 +# model = utils.SetDevice(Model.Net535()) +# model = utils.SetDevice(Model.Net5Grad35()) +# model = utils.SetDevice(Model.Net31535()) +# optimizer = optim.Adam(model.parameters(), lr=0.01) + + # traindata, testdata = Loader.MNIST(batchsize, num_workers=4) # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Vertical") # traindata, testdata = Loader.RandomMnist(batchsize, num_workers=4, style="Horizontal") @@ -39,13 +45,7 @@ batchsize = 128 traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=4,shuffle=True,trainsize=500) - - - - - - -WebVisual.InitVisdom() +# WebVisual.InitVisdom() window = WebVisual.LineWin() lineNoPre = WebVisual.Line("NoPre") linePretrain = WebVisual.Line("Pretrain") @@ -55,42 +55,26 @@ linePretrain = WebVisual.Line("Pretrain") -# model = utils.SetDevice(Model.Net535()) -# model = utils.SetDevice(Model.Net5Grad35()) -# model = utils.SetDevice(Model.Net31535()) + # model = utils.SetDevice(Model.Net3335()) model = utils.SetDevice(Model.Net3Grad335()) - - - -# model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") - - +model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") optimizer = optim.SGD(model.parameters(), lr=0.1) -# optimizer = optim.Adam(model.parameters(), lr=0.01) - - - - -model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") -for i in range(2000): +for i in range(1000): Train.train(model,traindata,optimizer,epoch=i) window.AppendData(linePretrain,Train.test(model,testdata)) - model = utils.SetDevice(Model.Net3335()) +model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl") + optimizer = optim.SGD(model.parameters(), lr=0.1) -for i in range(2000): +for i in range(1000): Train.train(model, traindata, optimizer, epoch=i) window.AppendData(lineNoPre, Train.test(model, testdata)) - - - - # utils.SaveModel(model,CurrentPath+"/checkpointSGDPreTrain.pkl") diff --git a/FilterEvaluator/checkpoint.pkl b/FilterEvaluator/checkpoint.pkl index ba7ded64f8d868d655202a7671b9c823e9fa88a4..000c9487269efe777554a3b5dff7e71d3c5d8519 100644 GIT binary patch delta 1028 zcmZvaOH30{6o&htKoL|PwiHEPBEzecp-*Jy77J(`K`apps4zgK5TG79g{Ya71VdC} zVuAv$(1jX}5VZ^AV(x5ph3Ezn)C8g?#<)>5QIa}>BqeF?=H&m*J%9dt&-L?*{ZF*= zteQSA%QbM`LAKi+2QjXlGMUU~Bn<|W(PA`|S;z^S{B*#x4Op#K36YTKik7-N{atLi z0g`I%iaHLGPckxsiIt9$QF4$%Fh?1+RLeQobJe5BcY=lktrJpRiPGJ3ZkF>8x_x@H ze%Rg9?cqS@Q4xeEdb?q@sg$(ZEM`axcig+_xNo~-dYI$>O~(ulGM$h$(XY@3vLns6 zkS5Y#Geb`J_ye`CW28X28{^}oKq^MX!9jv~$>^k73Y51wp5Q==Rp95L(503_we`C> zwu|LhDB1{IOt>Uf26`S!XjhC>4!H+?J>CH@Fmj(ORuZ<}0ZMrwJsWadn5;KI84pH} zVWLu*95C@+tDE6lXw2gzwqb=gPyytuIk*6aQ_bmb8L0T>Za}!Jepib&%K8X9VnxoM?`m7e~bk~H1 zm2o=h?Vz|`T_qNvXhf~0pbx>{RO6qYc|uA@8`#K-M}&) zE-0i`P0wDen=eqI@gwwHcQJl5mtBL#gl069yGwK^XQ}roPlYJ*4(&)ErK18XcM=bdxz&3X6Eds%sW@3oNz zi@z#fu{*pz&+!9>mm2}T@=7^ekbqy(b^FgjR3 zdbfo`Qal=t*;w2D@Q&>f3C2XSt=33Rq)(tz@vuJ5>4w7e#IdQwaeWiVrzK8EC7LCe z7ywJ(M6M@sQl>$p^^!xj)V$SCzlF5fh7q zHbnD*qaP4;|EG&df=prlXUf<0SLn^uc1sfqiH}1cV}WXZ)rwrZXQY!b_SP=Szin$k z@NqmG7wy{;a)^I`4ng*nx(2&D%-wN?QvWVkB{s@>7;u$W=avUO>FO Rg>FFAm%m0`@2;sY`3LP@^V1_mZEM!{qM R&8C6mJzf1=);T3K0RVa(4O9RC diff --git a/FilterEvaluator/image/c1_1.1936779022216797.png b/FilterEvaluator/image/c1_1.1936779022216797.png deleted file mode 100644 index 1e67d792c907a35b2f824c33eaaa69e2a63a4a67..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y@0VEjgy1g@i6pyEiV+e@^VW(FoDM%k?W QuJIswPgg&ebxsLQ0C1KJs{jB1 diff --git a/FilterEvaluator/image/c2_1.193914771080017.png b/FilterEvaluator/image/c2_1.193914771080017.png deleted file mode 100644 index 6cfd3c1e6213937e70ee320be8780a8508f153ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y@0VEjgy1g@i6pyEiV+e@^VRt6>oM(Hrd Q>yjXOPgg&ebxsLQ0BD8`BLDyZ diff --git a/FilterEvaluator/image/c3_1.1940335035324097.png b/FilterEvaluator/image/c3_1.1940335035324097.png deleted file mode 100644 index c530008dd1c099aa2d3df961e7f6d9ec4d1c3217..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y@0VEjgy1g@i6pyEiV+e|eMybMf&j6Cuk RKS7EZJYD@<);T3K0RVcj4G91M diff --git a/FilterEvaluator/image/c4_1.1942846775054932.png b/FilterEvaluator/image/c4_1.1942846775054932.png deleted file mode 100644 index 3f569f2ea1640cb8a2f152e07cbc36cf9469b76f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y@0VEjgy1g@i6pyEiV+e|eMj0{YCjGX*Y Qn!7;qp00i_>zopr0CRQ>pa1{> diff --git a/FilterEvaluator/image/c6_1.1943182945251465.png b/FilterEvaluator/image/c6_1.1943182945251465.png deleted file mode 100644 index 06c9c54161997d46f9fd68b84612b53bfd808e6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y@0VEjgy1g@i6pyEiV+erFup(C R_eK*W@9FC2vd$@?2>`Zu5Ig_? diff --git a/FilterEvaluator/image/c7_1.194318413734436.png b/FilterEvaluator/image/c7_1.194318413734436.png deleted file mode 100644 index b55bd5672ab8a95180d88fece7823d0f84c0ca92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y@0VEjgy1g@i6pyEiV+e@^Veg-C0Mg@nc Q`x`*=p00i_>zopr0Cors82|tP diff --git a/tools/UniModule.py b/tools/UniModule.py index b3447f5..a403f75 100644 --- a/tools/UniModule.py +++ b/tools/UniModule.py @@ -20,7 +20,17 @@ class ModuleBase(nn.Module): if b != None: self.features[layer].bias.requires_grad = requiregrad - def forwardLayer(self, x, layer=0): + def ForwardLayer(self, x, layer=0): + if layer < 0: + return x layers = self.features[0:layer+1] x = layers(x) - + return x + + def PrintLayer(self): + names = [] + i = 0 + for l in self.features: + names.append(str(i)+' : '+str(l)) + i = i+1 + return names