init git

.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+.idea
+.pytest_cache
+*.pyc
+__pycache__/
+Dataset/
+.vscode
+/*/__pycache__

FilterEvaluator/Evaluator.py

@@ -0,0 +1,185 @@
+from __future__ import print_function
+import os
+import sys
+
+# import multiprocessing
+# multiprocessing.set_start_method('spawn', True)
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+import random
+import cv2
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+sys.path.append(CurrentPath+'../tools')
+sys.path.append(CurrentPath+'../')
+
+import Model as Model
+from tools import utils, Train, Loader
+
+batchsize = 128
+
+
+# model = utils.SetDevice(Model.Net5Grad35())
+# model = utils.SetDevice(Model.Net31535())
+model = utils.SetDevice(Model.Net3Grad335())
+# model = utils.SetDevice(Model.Net3())
+
+
+
+# model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+
+
+
+
+
+# a = model.features[0].weight.data
+
+
+
+
+# traindata, testdata = Loader.MNIST(batchsize)
+# traindata, testdata = Loader.RandomMnist(batchsize, style="Vertical")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="Horizontal")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalOneLine")
+# traindata, testdata = Loader.RandomMnist(batchsize, style="VerticalZebra")
+# traindata, testdata = Loader.Cifar10Mono(batchsize)
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=2, shuffle=True)
+
+
+# def GetSample(netmodel,layer,dataloader,iteration=-1):
+#     netmodel.eval()
+#     sample = []
+#     for batch_idx, (data, target) in enumerate(dataloader):
+#         data = utils.SetDevice(data)
+#         target = utils.SetDevice(target)
+#         layerout = []
+#         layerint = []
+#         def getnet(self, input, output):
+#             layerout.append(output)
+#             layerint.append(input)
+#         handle = netmodel.features[layer].register_forward_hook(getnet)
+#         netmodel.forwardLayer(data,layer=layer)
+#         output = layerout[0][:,:,:,:]
+
+#         handle.remove()
+#         data.detach()
+#         target.detach()
+        
+#         sample.append(output.cpu().detach().numpy())
+#         if iteration > 0 and batch_idx >= (iteration-1):
+#             break
+
+#     sample = np.array(sample)
+#     sample = np.swapaxes(sample,2,0)
+#     sample = np.reshape(sample,(sample.shape[0],-1))
+
+#     active = []
+#     for i in range(sample.shape[0]):
+#         data = sample[i]
+#         mean = np.mean(data)
+#         dat1 = np.mean(np.abs(data - mean))
+#         dat2 = (data - mean)/dat1
+#         dat2 = np.mean(dat2*dat2)
+#         active.append(dat2)
+
+#     return active
+
+
+def GetSample(netmodel,layer,dataloader,iteration=-1):
+    netmodel.eval()
+    sample = utils.SetDevice(torch.empty((8,0)))
+    for batch_idx, (data, target) in enumerate(dataloader):
+        data = utils.SetDevice(data)
+        target = utils.SetDevice(target)
+        layerout = []
+        layerint = []
+        def getnet(self, input, output):
+            layerout.append(output)
+            layerint.append(input)
+        handle = netmodel.features[layer].register_forward_hook(getnet)
+        netmodel.forwardLayer(data,layer=layer)
+        output = layerout[0][:,:,:,:]
+
+        handle.remove()
+        data.detach()
+        target.detach()
+        
+        output = torch.reshape(output.transpose(0,1),(8,-1))
+        sample = torch.cat((sample,output),1)
+        if iteration > 0 and batch_idx >= (iteration-1):
+            break
+
+    sample_mean=torch.mean(sample,dim=1,keepdim=True)
+    dat1 = torch.mean(torch.abs(sample - sample_mean),dim=1,keepdim=True)
+    dat2 = (sample - sample_mean)/dat1
+    dat2 = torch.mean(dat2 * dat2,dim=1)
+    return dat2.cpu().detach().numpy()
+
+def weightToImage(weight,filename):
+    a2 = model.features[layear].weight.data[channel]
+    a2 = a2.cpu().detach().numpy().reshape(( a2.shape[-2], a2.shape[-1]))
+
+    a2min = np.min(a2)
+    a2max = np.max(a2)
+    a2 = (a2 - a2min)*255.0/(a2max-a2min)
+    a2 = a2.astype(int)
+    cv2.imwrite(filename,a2)
+
+def GetRandomSocre(netmodel,layer,dataloader,iteration=-1):
+    weightshape = netmodel.features[layer].weight.data.shape
+    newweight = np.random.uniform(-1.0,1.0,weightshape).astype("float32")
+    netmodel.features[layer].weight.data=utils.SetDevice(torch.from_numpy(newweight))
+
+    score = GetSample(netmodel,0,dataloader,iteration)
+    return np.array(score), newweight
+  
+
+
+layer = 0
+
+weightshape = list(model.features[layer].weight.data.shape)
+channels = weightshape[0]
+weightshape[0] = 0
+minactive = np.empty((0))
+minweight = np.empty(weightshape)
+
+for i in range(300000):
+    score,weight = GetRandomSocre(model,layer,traindata,iteration=10)
+
+    minactive = np.append(minactive, score)
+    minweight = np.concatenate((minweight, weight))
+
+    index = minactive.argsort()
+    minactive = minactive[index[0:channels]]
+    minweight = minweight[index[0:channels]]
+
+    print("search random :" + str(i))
+    if i % 10000 == 0:
+        utils.SaveModel(model, CurrentPath+"/checkpoint.pkl")
+
+    
+# for i in range(minweight.shape[0]):
+#     a2 = minweight[i].reshape((minweight.shape[2],minweight.shape[3]))
+#     a2min = np.min(a2)
+#     a2max = np.max(a2)
+#     a2 = (a2 - a2min)*255.0/(a2max-a2min)
+#     a2 = a2.astype(int)
+#     cv2.imwrite(CurrentPath+"/image/c"+str(i)+"_"+str(minactive[i])+".png",a2)
+
+
+model.features[layer].weight.data=utils.SetDevice(torch.from_numpy(minweight))
+utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")
+print("save model sucess")

FilterEvaluator/Model.py

@@ -0,0 +1,95 @@
+from __future__ import print_function
+import os
+import sys
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+sys.path.append(CurrentPath+'../tools')
+sys.path.append(CurrentPath+'../')
+from tools import UniModule
+
+class Net535(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net535, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=5,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=5,bias=False)]
+        self.features = nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
+class Net5Grad35(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net5Grad35, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=5,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=5,bias=False)]
+        self.features = nn.Sequential(*layers)
+        self.SetConvRequiresGrad(0,False)
+
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
+class Net3335(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net3335, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=3,bias=False,padding=1),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=5,bias=False)]
+        self.features = nn.Sequential(*layers)
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
+class Net3Grad335(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net3Grad335, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=3,bias=False,padding=1),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=5,bias=False)]
+        self.features = nn.Sequential(*layers)
+        self.SetConvRequiresGrad(0,False)
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
+class Net31535(UniModule.ModuleBase):
+    def __init__(self):
+        super(Net31535, self).__init__()
+        layers = []
+        layers += [nn.Conv2d(1,  8, kernel_size=[1,3],bias=False,padding=[0,1]),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=5,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  8, kernel_size=3,bias=False),nn.MaxPool2d(kernel_size=2, stride=2),nn.Sigmoid()]
+        layers += [nn.Conv2d(8,  10, kernel_size=5,bias=False)]
+        self.features = nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 1*10)
+        return F.log_softmax(x, dim=1)
+
+    
+
+            

FilterEvaluator/TrainNetwork.py

@@ -0,0 +1,58 @@
+from __future__ import print_function
+import os
+import sys
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+import random
+import cv2
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+sys.path.append(CurrentPath+'../tools')
+sys.path.append(CurrentPath+'../')
+
+import Model
+from tools import utils, Train, Loader
+
+batchsize = 128
+
+
+# model = utils.SetDevice(Model.Net535())
+# model = utils.SetDevice(Model.Net5Grad35())
+# model = utils.SetDevice(Model.Net31535())
+# model = utils.SetDevice(Model.Net3335()) # Test set: Average loss: 1.5294, Accuracy: 4501/10000 (45%)
+model = utils.SetDevice(Model.Net3Grad335())
+
+
+
+
+model = utils.LoadModel(model, CurrentPath+"/checkpoint.pkl")
+
+optimizer = optim.SGD(model.parameters(), lr=0.01)
+
+
+# traindata, testdata = Loader.MNIST(batchsize, num_workers=8)
+# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=8, style="Vertical")
+# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=8, style="Horizontal")
+# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=8, style="VerticalOneLine")
+# traindata, testdata = Loader.RandomMnist(batchsize, num_workers=8, style="VerticalZebra")
+traindata, testdata = Loader.Cifar10Mono(batchsize, num_workers=8)
+
+
+
+for i in range(300):
+    Train.train(model,traindata,optimizer,epoch=i)
+    Train.test(model,testdata)
+
+utils.SaveModel(model,CurrentPath+"/checkpoint.pkl")

Other/GenPrecisionImage.py

@@ -0,0 +1,58 @@
+import matplotlib
+matplotlib.use('Agg')
+
+import sys
+sys.path.append("./coco/PythonAPI/")
+import json
+import time
+import argparse
+import pprint
+
+import numpy as np
+from pycocotools.coco import COCO
+import shutil
+import cv2
+import torch
+import PIL
+import torchvision
+from torchvision import transforms, utils
+import re
+import matplotlib.pyplot as plt
+
+testtxt=open('TestPrecision.txt','r')
+valtxt=open('ValPrecision.txt','r')
+
+test=[]
+for line in testtxt:
+    line = line.strip('\n')
+    line = line.rstrip()
+    #words = line.split()
+    test.append(float(line))
+
+val=[]
+for line in valtxt:
+    line = line.strip('\n')
+    line = line.rstrip()
+    #words = line.split()
+    val.append(float(line))
+
+
+
+
+
+y1 = np.array(test)
+y2 = np.array(val)
+x1=np.arange(len(test))
+x2=np.arange(len(val))
+
+plt.figure()
+
+
+plt.plot(x1, y1, color='red', label='testing')
+plt.plot(x2, y2,  color='green', label='validation')
+plt.title('')
+plt.xlabel('epoch')
+plt.ylabel('precision')
+plt.legend()
+plt.savefig('result.png',dpi=600)
+#plt.show()

Other/InvertCNN.py

@@ -0,0 +1,292 @@
+from __future__ import print_function
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+
+
+# viz=Visdom()
+# viz.delete_env('main')
+
+
+batchsize=128
+
+DATA_FOLDER = os.path.split(os.path.realpath(__file__))[0]+'/Dataset/'
+print("Dataset Path :" + DATA_FOLDER)
+
+class Net(nn.Module):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
+        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
+        self.conv2_drop = nn.Dropout2d()
+        self.fc1 = nn.Linear(320, 50)
+        self.fc2 = nn.Linear(50, 10)
+
+        # Spatial transformer localization-network
+        self.localization = nn.Sequential(
+            nn.Conv2d(1, 8, kernel_size=7),
+            nn.MaxPool2d(2, stride=2),
+            nn.ReLU(True),
+            nn.Conv2d(8, 10, kernel_size=5),
+            nn.MaxPool2d(2, stride=2),
+            nn.ReLU(True)
+        )
+
+        # Regressor for the 3 * 2 affine matrix
+        self.fc_loc = nn.Sequential(
+            nn.Linear(10 * 3 * 3, 32),
+            nn.ReLU(True),
+            nn.Linear(32, 3 * 2)
+        )
+
+        # Initialize the weights/bias with identity transformation
+        self.fc_loc[2].weight.data.zero_()
+        self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
+
+    # Spatial transformer network forward function
+    def stn(self, x):
+        xs = self.localization(x)
+        xs = xs.view(-1, 10 * 3 * 3)
+        theta = self.fc_loc(xs)
+        theta = theta.view(-1, 2, 3)
+
+        grid = F.affine_grid(theta, x.size())
+        x = F.grid_sample(x, grid)
+
+        return x
+
+    def forward(self, x):
+        # transform the input
+        x = self.stn(x)
+
+        # Perform the usual forward pass
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
+        x = x.view(-1, 320)
+        x = F.relu(self.fc1(x))
+        x = F.dropout(x, training=self.training)
+        x = self.fc2(x)
+        return F.log_softmax(x, dim=1)
+
+#99.01
+class NetMnist(nn.Module):
+    def __init__(self):
+        super(NetMnist, self).__init__()
+        channels=32
+        self.conv1_1 = nn.Conv2d(1, channels, kernel_size=5 , padding=0)
+
+        self.conv2_1 = nn.Conv2d(channels, channels, kernel_size=3, padding=0)
+        self.conv2_2 = nn.Conv2d(channels, channels, kernel_size=3, padding=0)
+        # self.conv2_3 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        # self.conv2_4 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+
+        self.conv3_1 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        self.conv3_2 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        self.conv3_3 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        self.conv3_4 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+
+        self.conv3_5 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        self.conv3_6 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        self.conv3_7 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+        self.conv3_8 = nn.Conv2d(channels, channels, kernel_size=5, padding=0)
+
+        self.fc1 = nn.Linear(256, 10)
+
+    def forward(self, x):
+
+        con1 = F.relu(F.max_pool2d(self.conv1_1(x), 2))
+
+        con2_1 = F.relu(F.max_pool2d(self.conv2_1(con1), 2))
+        con2_2 = F.relu(F.max_pool2d(self.conv2_2(con1), 2))
+        # con2_3 = F.relu(F.max_pool2d(self.conv2_3(con1), 2))
+        # con2_4 = F.relu(F.max_pool2d(self.conv2_4(con1), 2))
+
+        con3_1 = F.relu((self.conv3_1(con2_1)))
+        con3_2 = F.relu((self.conv3_2(con2_1)))
+        con3_3 = F.relu((self.conv3_3(con2_1)))
+        con3_4 = F.relu((self.conv3_4(con2_1)))
+
+        con3_5 = F.relu((self.conv3_1(con2_2)))
+        con3_6 = F.relu((self.conv3_2(con2_2)))
+        con3_7 = F.relu((self.conv3_3(con2_2)))
+        con3_8 = F.relu((self.conv3_4(con2_2)))
+
+        cc=[con3_1,con3_2,con3_3,con3_4,con3_5,con3_6,con3_7,con3_8]
+        #cc=[con4_1,con4_2,con4_3,con4_4,con4_5,con4_6,con4_7,con4_8]
+        ccs=torch.cat(cc,1)
+        ccs=ccs.view(-1,256)
+        x = self.fc1(ccs)
+        return x
+        #return F.log_softmax(x, dim=1)
+
+class NetMnistRatio(nn.Module):
+    def __init__(self):
+        super(NetMnistRatio, self).__init__()
+        channels=32
+
+        self.conv1 = nn.Conv2d(1, channels, kernel_size=5)
+        self.conv2 = nn.Conv2d(channels, channels * 2, kernel_size=3)
+        self.conv3 = nn.Conv2d(channels*2, channels * 4, kernel_size=5)
+
+        self.fc1 = nn.Linear(128, 10)
+
+    def forward(self, x):
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2(x), 2))
+        x = F.relu(self.conv3(x))
+
+        x = x.view(-1, 128)
+        x = self.fc1(x)
+        return x
+
+#96.4 train slow
+class NetMnistNormal(nn.Module):
+    def __init__(self):
+        super(NetMnistNormal, self).__init__()
+        channels=128
+
+        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
+        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
+        self.conv2_drop = nn.Dropout2d()
+        self.fc1 = nn.Linear(320, 50)
+        self.fc2 = nn.Linear(50, 10)
+
+    def forward(self, x):
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
+        x = x.view(-1, 320)
+        x = F.relu(self.fc1(x))
+        x = F.dropout(x, training=self.training)
+        x = self.fc2(x)
+        return x
+
+#98.9
+class NetMnistFCNN(nn.Module):
+    def __init__(self):
+        super(NetMnistFCNN, self).__init__()
+        channels=128
+        self.conv1 = nn.Conv2d(1, channels, kernel_size=3 , padding=1)
+
+        self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, padding=0)
+
+        self.conv3 = nn.Conv2d(channels, channels, kernel_size=3, padding=1)
+
+        self.conv4 = nn.Conv2d(channels, 10, kernel_size=3, padding=0)
+
+    def forward(self, x):
+
+        con1 = F.relu(F.max_pool2d(self.conv1(x), 2))
+
+        con2 = F.relu(F.max_pool2d(self.conv2(con1), 2))
+
+        con3 = F.relu(F.max_pool2d((self.conv3(con2)),2))
+
+        con4 = self.conv4(con3)
+
+        x = con4.view(-1,10)
+
+        return x
+
+
+#region loder dataset
+train_loader = torch.utils.data.DataLoader(
+    datasets.MNIST(root=DATA_FOLDER, train=True, download=True,
+                   transform=transforms.Compose([
+                       transforms.ToTensor(),
+                       transforms.Normalize((0.1307,), (0.3081,))
+                   ])), batch_size=batchsize, shuffle=True, num_workers=8)
+# Test dataset
+val_loader = torch.utils.data.DataLoader(
+    datasets.MNIST(root=DATA_FOLDER, train=False, transform=transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.1307,), (0.3081,))
+    ])), batch_size=batchsize, shuffle=True, num_workers=8)
+#endregion loder dataset
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = (NetMnistRatio()).to(device)
+#########################################################
+optimizer = optim.Adadelta(model.parameters(), lr=0.000001)
+lossfunc=torch.nn.CrossEntropyLoss().to(device)
+#lossfunc=torch.nn.NLLLoss()
+
+# gpu_ids=[0,1,2,3]
+# model = torch.nn.DataParallel(model, device_ids = gpu_ids)
+# optimizer = torch.nn.DataParallel(optimizer, device_ids = gpu_ids)
+
+def train(epoch):
+    model.train()
+    correct = 0
+    for batch_idx, (data, target) in enumerate(train_loader):
+        data, target = data.to(device), target.to(device)
+
+        
+
+        output = model(data)
+        loss = lossfunc(output, target)
+        loss.backward()
+        optimizer.step()
+
+        pred = output.max(1, keepdim=True)[1]
+        correct += pred.eq(target.view_as(pred)).sum().item()
+
+        if batch_idx == (len(train_loader) - 2):
+            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset),100. * batch_idx / len(train_loader), loss.item()))
+    lens = len(train_loader.dataset)
+    correct = float(correct) / lens
+    testtxt.write(str(correct)+'\n')
+    print('Test set: val: {:.6f}'.format(correct))
+def val():
+    with torch.no_grad():
+        model.eval()
+        correct = 0
+        for data, target in val_loader:
+            data, target = data.to(device), target.to(device)
+            output = model(data)
+            # get the index of the max log-probability
+            pred = output.max(1, keepdim=True)[1]
+            correct += pred.eq(target.view_as(pred)).sum().item()
+        lens=len(val_loader.dataset)
+        correct = float(correct)/lens
+        valtxt.write(str(correct) + '\n')
+        print('Val  set: val: {:.6f}'.format(correct))
+
+
+testtxt=open('TestPrecision.txt','w')
+valtxt=open('ValPrecision.txt','w')
+
+# optimizer.zero_grad()
+
+for epoch in range(1, 1000):
+    train(epoch)
+    val()
+
+
+
+
+
+# mm = model.conv1(ad)
+# datodis = mm * 256 + 128
+# datodis = datodis.view(64, 1, 24 * 10, 24)
+# imaglis.append(datodis.detach().cpu().numpy()[0:8,:,:,:])
+
+
+#cdsa=np.reshape(np.array(imaglis),newshape=[-1,1,240,24])
+#viz.images(cdsa, win=imagewin, opts=dict(title='Random!', caption='How random.'), nrow=8,padding=2)
+
+
+# Visualize the STN transformation on some input batch
+# visualize_stn()
+
+#plt.ioff()
+#plt.show()
+

Other/UnSuppervise.py

@@ -0,0 +1,122 @@
+from __future__ import print_function
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.utils.data import Dataset, DataLoader
+
+
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+import cv2
+
+
+
+# viz=Visdom()
+# viz.delete_env('main')
+
+
+DATA_FOLDER = os.path.split(os.path.realpath(__file__))[0]+'/Dataset/'
+print("Dataset Path :" + DATA_FOLDER)
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Training dataset
+train_loader = torch.utils.data.DataLoader(
+    datasets.MNIST(root=DATA_FOLDER, train=True, download=True,
+                   transform=transforms.Compose([
+                       transforms.ToTensor(),
+                       #transforms.Normalize((0.1307,), (0.3081,))
+                   ])), batch_size=1, shuffle=True, num_workers=1)
+
+class NetMnist(nn.Module):
+    def __init__(self):
+        super(NetMnist, self).__init__()
+        channels=1
+        self.conv1 = nn.Conv2d(1, channels, kernel_size=3 , padding=0)
+
+    def forward(self, x):
+
+        da = self.conv1.weight.data
+        da = da.view(9)
+        damean=da.mean()
+        da = da - damean
+        daabssum=da.abs().sum()
+        da = da/daabssum
+        da = da.view(1,1,3,3)
+        self.conv1.weight.data = da
+        con1 = self.conv1(x)
+        con1 = con1.abs()
+
+        # con1 = F.sigmoid(F.max_pool2d(self.conv1(x), 2))
+        #
+        # con2 = F.sigmoid(F.max_pool2d(self.conv2(con1), 2))
+        #
+        # con3 = F.sigmoid(F.max_pool2d((self.conv3(con2)),2))
+        #
+        # con4 = F.sigmoid(self.conv4(con3))
+        #
+        # x = con4.view(-1,10)
+
+        return con1
+
+
+model = (NetMnist()).to(device)
+
+
+
+#########################################################
+optimizer = optim.SGD(model.parameters(), lr=0.1)
+#lossfunc=torch.nn.CrossEntropyLoss()
+lossfunc=torch.nn.MSELoss()
+
+
+
+gpu_ids=[0,1,2,3]
+#model = torch.nn.DataParallel(model, device_ids = gpu_ids)
+#optimizer = torch.nn.DataParallel(optimizer, device_ids = gpu_ids)
+
+
+def train(epoch):
+    model.train()
+    for batch_idx, (data, target) in enumerate(train_loader):
+        data, target = data.to(device), target.to(device)
+
+        optimizer.zero_grad()
+
+        output = model(data)
+        target = output + 0.1
+
+        var_no_grad = target.detach()
+        loss = lossfunc(output, var_no_grad)
+
+
+
+
+        loss.backward()
+        optimizer.step()
+
+        if batch_idx % 10 == 0 and batch_idx>0 :
+            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset),100. * batch_idx / len(train_loader), loss.item()))
+
+            da=model.conv1.weight.data
+            da = da.view(9)
+            damean = da.mean()
+            da = da - damean
+            daabssum = da.abs().sum()
+            da = da / daabssum
+            da = da.view(1, 1, 3, 3)
+
+            print(da)
+
+
+
+
+for epoch in range(1, 3000):
+    train(epoch)

Other/UnSupperviseCifar.py

@@ -0,0 +1,170 @@
+from __future__ import print_function
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.utils.data import Dataset, DataLoader
+
+
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+
+
+# viz=Visdom()
+# viz.delete_env('main')
+
+
+DATA_FOLDER = os.path.split(os.path.realpath(__file__))[0]+'/Dataset/'
+print("Dataset Path :" + DATA_FOLDER)
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+#region User Define Radom Dataset
+def default_loader(path):
+    da=np.random.randint(0,255,(1,28,28)).astype("float32")
+    da[0,15:17,15:17]=255
+    return da
+
+class MyDataset(Dataset):
+    def __init__(self,imagepath, transform=None, target_transform=None, loader=default_loader):
+        imgs = []
+        for line in range(10000):
+            imgs.append((imagepath,int(0)))
+        self.imgs = imgs
+        self.transform = transform
+        self.target_transform = target_transform
+        self.loader = loader
+
+    def __getitem__(self, index):
+        fn, label = self.imgs[index]
+        img = self.loader(fn)
+        img = torch.from_numpy(img)
+        return img,label
+
+    def __len__(self):
+        return len(self.imgs)
+
+
+train_data=MyDataset(imagepath="" , transform=transforms.Compose([
+                                                #transforms.Resize(256),
+                                                #transforms.CenterCrop(224),
+                                                # transforms.RandomHorizontalFlip(),
+                                                # transforms.RandomAffine(degrees=30,translate=(0.2,0.2),scale=(0.8,1.2),resample=PIL.Image.BILINEAR,fillcolor=0),
+                                                #transforms.ColorJitter(),
+                                                transforms.ToTensor(),
+                                                #transforms.Normalize(mean = (0.5, 0.5, 0.5), std = (0.5, 0.5, 0.5)),
+                                                ]))
+
+train_loader = torch.utils.data.DataLoader(train_data,
+                                          batch_size=64,
+                                          shuffle=True,#if random data
+                                          drop_last=True,
+                                          num_workers=1,
+                                          #collate_fn = collate_fn
+                                          )
+#endregion
+
+
+
+
+# Training dataset
+train_loader = torch.utils.data.DataLoader(
+    datasets.CIFAR10(root=DATA_FOLDER, train=True, download=True,
+                   transform=transforms.Compose([
+                       transforms.ToTensor(),
+                       #transforms.Normalize((0.1307,), (0.3081,))
+                   ])), batch_size=1, shuffle=True, num_workers=1)
+
+
+class NetMnist(nn.Module):
+    def __init__(self):
+        super(NetMnist, self).__init__()
+        channels=1
+        self.conv1 = nn.Conv2d(3, channels, kernel_size=3 , padding=0)
+
+    def forward(self, x):
+
+        da = self.conv1.weight.data
+        da = da.view(27)
+        damean=da.mean()
+        da = da - damean
+        daabssum=da.abs().sum()
+        da = da/daabssum
+        da = da.view(1,3,3,3)
+        self.conv1.weight.data = da
+        con1 = self.conv1(x)
+        con1 = con1.abs()
+
+        # con1 = F.sigmoid(F.max_pool2d(self.conv1(x), 2))
+        #
+        # con2 = F.sigmoid(F.max_pool2d(self.conv2(con1), 2))
+        #
+        # con3 = F.sigmoid(F.max_pool2d((self.conv3(con2)),2))
+        #
+        # con4 = F.sigmoid(self.conv4(con3))
+        #
+        # x = con4.view(-1,10)
+
+        return con1
+
+
+model = (NetMnist()).to(device)
+
+
+
+#########################################################
+optimizer = optim.SGD(model.parameters(), lr=1)
+#lossfunc=torch.nn.CrossEntropyLoss()
+lossfunc=torch.nn.MSELoss()
+
+
+
+gpu_ids=[0,1,2,3]
+#model = torch.nn.DataParallel(model, device_ids = gpu_ids)
+#optimizer = torch.nn.DataParallel(optimizer, device_ids = gpu_ids)
+
+
+def train(epoch):
+    model.train()
+    for batch_idx, (data, target) in enumerate(train_loader):
+        data, target = data.to(device), target.to(device)
+
+        optimizer.zero_grad()
+
+        output = model(data)
+        target = output + 0.1
+
+        var_no_grad = target.detach()
+        loss = lossfunc(output, var_no_grad)
+
+
+
+
+        loss.backward()
+        optimizer.step()
+
+        if batch_idx % 1 == 0 and batch_idx>0 :
+            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset),100. * batch_idx / len(train_loader), loss.item()))
+
+            da=model.conv1.weight.data
+            da = da.view(27)
+            damean = da.mean()
+            da = da - damean
+            daabssum = da.abs().sum()
+            da = da / daabssum
+            da = da.view(1, 3, 3, 3)
+
+            print(da)
+
+
+
+for epoch in range(1, 3000):
+    train(epoch)

Other/UnSupperviseSelfData.py

@@ -0,0 +1,186 @@
+from __future__ import print_function
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.utils.data import Dataset, DataLoader
+
+
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+
+
+# viz=Visdom()
+# viz.delete_env('main')
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+
+def default_loader(path):
+    da=np.random.randint(0,255,(1,28,28)).astype("float32")
+    da[0,15:17,15:17]=255
+    return da
+
+class MyDataset(Dataset):
+    def __init__(self,imagepath, transform=None, target_transform=None, loader=default_loader):
+        imgs = []
+        for line in range(10000):
+            imgs.append((imagepath,int(0)))
+        self.imgs = imgs
+        self.transform = transform
+        self.target_transform = target_transform
+        self.loader = loader
+
+    def __getitem__(self, index):
+        fn, label = self.imgs[index]
+        img = self.loader(fn)
+        img = torch.from_numpy(img)
+        return img,label
+
+    def __len__(self):
+        return len(self.imgs)
+
+
+train_data=MyDataset(imagepath="" , transform=transforms.Compose([
+                                                #transforms.Resize(256),
+                                                #transforms.CenterCrop(224),
+                                                # transforms.RandomHorizontalFlip(),
+                                                # transforms.RandomAffine(degrees=30,translate=(0.2,0.2),scale=(0.8,1.2),resample=PIL.Image.BILINEAR,fillcolor=0),
+                                                #transforms.ColorJitter(),
+                                                transforms.ToTensor(),
+                                                #transforms.Normalize(mean = (0.5, 0.5, 0.5), std = (0.5, 0.5, 0.5)),
+                                                ]))
+
+train_loader = torch.utils.data.DataLoader(train_data,
+                                          batch_size=64,
+                                          shuffle=True,#if random data
+                                          drop_last=True,
+                                          num_workers=1,
+                                          #collate_fn = collate_fn
+                                          )
+
+# # Training dataset
+# train_loader = torch.utils.data.DataLoader(
+#     datasets.CIFAR10(root='.', train=True, download=True,
+#                    transform=transforms.Compose([
+#                        transforms.ToTensor(),
+#                        #transforms.Normalize((0.1307,), (0.3081,))
+#                    ])), batch_size=128, shuffle=True, num_workers=1)
+# # Test dataset
+# val_loader = torch.utils.data.DataLoader(
+#     datasets.CIFAR10(root='.', train=False, transform=transforms.Compose([
+#         transforms.ToTensor(),
+#         transforms.Normalize((0.1307,), (0.3081,))
+#     ])), batch_size=32, shuffle=True, num_workers=1)
+
+
+class NetMnist(nn.Module):
+    def __init__(self):
+        super(NetMnist, self).__init__()
+        channels=1
+        self.conv1 = nn.Conv2d(1, channels, kernel_size=3 , padding=0)
+
+    def forward(self, x):
+
+        da = self.conv1.weight.data
+        da = da.view(9)
+        damean=da.mean()
+        da = da - damean
+        daabssum=da.abs().sum()
+        da = da/daabssum
+        da = da.view(1,1,3,3)
+        self.conv1.weight.data = da
+        con1 = self.conv1(x)
+        con1 = con1.abs()
+
+        # con1 = F.sigmoid(F.max_pool2d(self.conv1(x), 2))
+        #
+        # con2 = F.sigmoid(F.max_pool2d(self.conv2(con1), 2))
+        #
+        # con3 = F.sigmoid(F.max_pool2d((self.conv3(con2)),2))
+        #
+        # con4 = F.sigmoid(self.conv4(con3))
+        #
+        # x = con4.view(-1,10)
+
+        return con1
+
+
+model = (NetMnist()).to(device)
+
+
+
+#########################################################
+optimizer = optim.SGD(model.parameters(), lr=0.1)
+#lossfunc=torch.nn.CrossEntropyLoss()
+lossfunc=torch.nn.MSELoss()
+
+
+
+gpu_ids=[0,1,2,3]
+#model = torch.nn.DataParallel(model, device_ids = gpu_ids)
+#optimizer = torch.nn.DataParallel(optimizer, device_ids = gpu_ids)
+
+
+def train(epoch):
+    model.train()
+    for batch_idx, (data, target) in enumerate(train_loader):
+        data, target = data.to(device), target.to(device)
+
+        optimizer.zero_grad()
+
+        output = model(data)
+        target = output + 0.1
+
+        var_no_grad = target.detach()
+        loss = lossfunc(output, var_no_grad)
+
+
+
+
+        loss.backward()
+        optimizer.step()
+
+        if batch_idx % 10 == 0 and batch_idx>0 :
+            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset),100. * batch_idx / len(train_loader), loss.item()))
+
+            da=model.conv1.weight.data
+            da = da.view(9)
+            damean = da.mean()
+            da = da - damean
+            daabssum = da.abs().sum()
+            da = da / daabssum
+            da = da.view(1, 1, 3, 3)
+
+            print(da)
+
+def val():
+    with torch.no_grad():
+        model.eval()
+
+        correct = 0
+        for data, target in val_loader:
+            data, target = data.to(device), target.to(device)
+            output = model(data)
+
+            # get the index of the max log-probability
+            pred = output.max(1, keepdim=True)[1]
+            correct += pred.eq(target.view_as(pred)).sum().item()
+        lens=len(val_loader.dataset)
+        correct = float(correct)/lens
+        print('\nTest set: val: {:.6f}\n'.format(correct))
+
+
+
+
+
+for epoch in range(1, 3000):
+    train(epoch)
+
+    #val()

Other/sequence_models_tutorial.py

@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+r"""
+Sequence Models and Long-Short Term Memory Networks
+===================================================
+
+At this point, we have seen various feed-forward networks. That is,
+there is no state maintained by the network at all. This might not be
+the behavior we want. Sequence models are central to NLP: they are
+models where there is some sort of dependence through time between your
+inputs. The classical example of a sequence model is the Hidden Markov
+Model for part-of-speech tagging. Another example is the conditional
+random field.
+
+A recurrent neural network is a network that maintains some kind of
+state. For example, its output could be used as part of the next input,
+so that information can propogate along as the network passes over the
+sequence. In the case of an LSTM, for each element in the sequence,
+there is a corresponding *hidden state* :math:`h_t`, which in principle
+can contain information from arbitrary points earlier in the sequence.
+We can use the hidden state to predict words in a language model,
+part-of-speech tags, and a myriad of other things.
+
+
+LSTM's in Pytorch
+~~~~~~~~~~~~~~~~~
+
+Before getting to the example, note a few things. Pytorch's LSTM expects
+all of its inputs to be 3D tensors. The semantics of the axes of these
+tensors is important. The first axis is the sequence itself, the second
+indexes instances in the mini-batch, and the third indexes elements of
+the input. We haven't discussed mini-batching, so lets just ignore that
+and assume we will always have just 1 dimension on the second axis. If
+we want to run the sequence model over the sentence "The cow jumped",
+our input should look like
+
+.. math::
+
+
+   \begin{bmatrix}
+   \overbrace{q_\text{The}}^\text{row vector} \\
+   q_\text{cow} \\
+   q_\text{jumped}
+   \end{bmatrix}
+
+Except remember there is an additional 2nd dimension with size 1.
+
+In addition, you could go through the sequence one at a time, in which
+case the 1st axis will have size 1 also.
+
+Let's see a quick example.
+"""
+
+# Author: Robert Guthrie
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+
+torch.manual_seed(1)
+
+######################################################################
+# Example: An LSTM for Part-of-Speech Tagging
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# In this section, we will use an LSTM to get part of speech tags. We will
+# not use Viterbi or Forward-Backward or anything like that, but as a
+# (challenging) exercise to the reader, think about how Viterbi could be
+# used after you have seen what is going on.
+#
+# The model is as follows: let our input sentence be
+# :math:`w_1, \dots, w_M`, where :math:`w_i \in V`, our vocab. Also, let
+# :math:`T` be our tag set, and :math:`y_i` the tag of word :math:`w_i`.
+# Denote our prediction of the tag of word :math:`w_i` by
+# :math:`\hat{y}_i`.
+#
+# This is a structure prediction, model, where our output is a sequence
+# :math:`\hat{y}_1, \dots, \hat{y}_M`, where :math:`\hat{y}_i \in T`.
+#
+# To do the prediction, pass an LSTM over the sentence. Denote the hidden
+# state at timestep :math:`i` as :math:`h_i`. Also, assign each tag a
+# unique index (like how we had word\_to\_ix in the word embeddings
+# section). Then our prediction rule for :math:`\hat{y}_i` is
+#
+# .. math::  \hat{y}_i = \text{argmax}_j \  (\log \text{Softmax}(Ah_i + b))_j
+#
+# That is, take the log softmax of the affine map of the hidden state,
+# and the predicted tag is the tag that has the maximum value in this
+# vector. Note this implies immediately that the dimensionality of the
+# target space of :math:`A` is :math:`|T|`.
+#
+#
+# Prepare data:
+
+def prepare_sequence(seq, to_ix):
+    idxs = [to_ix[w] for w in seq]
+    return torch.tensor(idxs, dtype=torch.long)
+
+
+training_data = [
+    ("The dog ate the apple".split(), ["DET", "NN", "V", "DET", "NN"]),
+    ("Everybody read that book".split(), ["NN", "V", "DET", "NN"])
+]
+word_to_ix = {}
+for sent, tags in training_data:
+    for word in sent:
+        if word not in word_to_ix:
+            word_to_ix[word] = len(word_to_ix)
+print(word_to_ix)
+tag_to_ix = {"DET": 0, "NN": 1, "V": 2}
+
+# These will usually be more like 32 or 64 dimensional.
+# We will keep them small, so we can see how the weights change as we train.
+EMBEDDING_DIM = 6
+HIDDEN_DIM = 6
+
+######################################################################
+# Create the model:
+
+
+class LSTMTagger(nn.Module):
+
+    def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size):
+        super(LSTMTagger, self).__init__()
+        self.hidden_dim = hidden_dim
+
+        self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)
+
+        # The LSTM takes word embeddings as inputs, and outputs hidden states
+        # with dimensionality hidden_dim.
+        self.lstm = nn.LSTM(embedding_dim, hidden_dim )
+
+        # The linear layer that maps from hidden state space to tag space
+        self.hidden2tag = nn.Linear(hidden_dim, tagset_size)
+        self.hidden = self.init_hidden()
+
+    def init_hidden(self):
+        # Before we've done anything, we dont have any hidden state.
+        # Refer to the Pytorch documentation to see exactly
+        # why they have this dimensionality.
+        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
+        return (torch.zeros(1, 1, self.hidden_dim),
+                torch.zeros(1, 1, self.hidden_dim))
+
+    def forward(self, sentence):
+        embeds = self.word_embeddings(sentence)
+        lstm_out, self.hidden = self.lstm(
+            embeds.view(len(sentence), 1, -1), self.hidden)
+        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))
+        tag_scores = F.log_softmax(tag_space, dim=1)
+        return tag_scores
+
+######################################################################
+# Train the model:
+
+
+model = LSTMTagger(EMBEDDING_DIM, HIDDEN_DIM, len(word_to_ix), len(tag_to_ix))
+loss_function = nn.NLLLoss()
+optimizer = optim.SGD(model.parameters(), lr=0.1)
+
+# See what the scores are before training
+# Note that element i,j of the output is the score for tag j for word i.
+# Here we don't need to train, so the code is wrapped in torch.no_grad()
+with torch.no_grad():
+    inputs = prepare_sequence(training_data[0][0], word_to_ix)
+    tag_scores = model(inputs)
+    print(tag_scores)
+
+for epoch in range(300):  # again, normally you would NOT do 300 epochs, it is toy data
+    for sentence, tags in training_data:
+        # Step 1. Remember that Pytorch accumulates gradients.
+        # We need to clear them out before each instance
+        model.zero_grad()
+
+        # Also, we need to clear out the hidden state of the LSTM,
+        # detaching it from its history on the last instance.
+        model.hidden = model.init_hidden()
+
+        # Step 2. Get our inputs ready for the network, that is, turn them into
+        # Tensors of word indices.
+        sentence_in = prepare_sequence(sentence, word_to_ix)
+        targets = prepare_sequence(tags, tag_to_ix)
+
+        # Step 3. Run our forward pass.
+        tag_scores = model(sentence_in)
+
+        # Step 4. Compute the loss, gradients, and update the parameters by
+        #  calling optimizer.step()
+        loss = loss_function(tag_scores, targets)
+        loss.backward()
+        optimizer.step()
+    print("epoch:"+str(epoch))
+
+# See what the scores are after training
+with torch.no_grad():
+    inputs = prepare_sequence(training_data[0][0], word_to_ix)
+    tag_scores = model(inputs)
+
+    # The sentence is "the dog ate the apple".  i,j corresponds to score for tag j
+    # for word i. The predicted tag is the maximum scoring tag.
+    # Here, we can see the predicted sequence below is 0 1 2 0 1
+    # since 0 is index of the maximum value of row 1,
+    # 1 is the index of maximum value of row 2, etc.
+    # Which is DET NOUN VERB DET NOUN, the correct sequence!
+    print(tag_scores)
+
+
+######################################################################
+# Exercise: Augmenting the LSTM part-of-speech tagger with character-level features
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# In the example above, each word had an embedding, which served as the
+# inputs to our sequence model. Let's augment the word embeddings with a
+# representation derived from the characters of the word. We expect that
+# this should help significantly, since character-level information like
+# affixes have a large bearing on part-of-speech. For example, words with
+# the affix *-ly* are almost always tagged as adverbs in English.
+#
+# To do this, let :math:`c_w` be the character-level representation of
+# word :math:`w`. Let :math:`x_w` be the word embedding as before. Then
+# the input to our sequence model is the concatenation of :math:`x_w` and
+# :math:`c_w`. So if :math:`x_w` has dimension 5, and :math:`c_w`
+# dimension 3, then our LSTM should accept an input of dimension 8.
+#
+# To get the character level representation, do an LSTM over the
+# characters of a word, and let :math:`c_w` be the final hidden state of
+# this LSTM. Hints:
+#
+# * There are going to be two LSTM's in your new model.
+#   The original one that outputs POS tag scores, and the new one that
+#   outputs a character-level representation of each word.
+# * To do a sequence model over characters, you will have to embed characters.
+#   The character embeddings will be the input to the character LSTM.
+#

Other/transformer_tutorial.py

@@ -0,0 +1,258 @@
+
+from __future__ import print_function
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+
+
+viz=Visdom()
+viz.delete_env('main')
+
+imagewin1=viz.image(np.random.rand(3, 512, 256),opts=dict(title='Random!', caption='How random.'))
+imagewin2=viz.image(np.random.rand(3, 512, 256),opts=dict(title='Random!', caption='How random.'))
+imagewin3=viz.image(np.random.rand(3, 512, 256),opts=dict(title='Random!', caption='How random.'))
+imagewin4=viz.image(np.random.rand(3, 512, 256),opts=dict(title='Random!', caption='How random.'))
+imagewin5=viz.image(np.random.rand(3, 512, 256),opts=dict(title='Random!', caption='How random.'))
+imagewin6=viz.image(np.random.rand(3, 512, 256),opts=dict(title='Random!', caption='How random.'))
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Training dataset
+train_loader = torch.utils.data.DataLoader(
+    datasets.MNIST(root='.', train=True, download=True,
+                   transform=transforms.Compose([
+                       transforms.ToTensor(),
+                       transforms.Normalize((0.1307,), (0.3081,))
+                   ])), batch_size=64, shuffle=True, num_workers=4)
+# Test dataset
+test_loader = torch.utils.data.DataLoader(
+    datasets.MNIST(root='.', train=False, transform=transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.1307,), (0.3081,))
+    ])), batch_size=64, shuffle=True, num_workers=4)
+
+
+
+#modelnet = models.vgg11(pretrained=True)
+
+class Net(nn.Module):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
+        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
+        self.conv2_drop = nn.Dropout2d()
+        self.fc1 = nn.Linear(320, 50)
+        self.fc2 = nn.Linear(50, 10)
+
+        # Spatial transformer localization-network
+        self.localization = nn.Sequential(
+            nn.Conv2d(1, 8, kernel_size=7),
+            nn.MaxPool2d(2, stride=2),
+            nn.ReLU(True),
+            nn.Conv2d(8, 10, kernel_size=5),
+            nn.MaxPool2d(2, stride=2),
+            nn.ReLU(True)
+        )
+
+        # Regressor for the 3 * 2 affine matrix
+        self.fc_loc = nn.Sequential(
+            nn.Linear(10 * 3 * 3, 32),
+            nn.ReLU(True),
+            nn.Linear(32, 3 * 2)
+        )
+
+        # Initialize the weights/bias with identity transformation
+        self.fc_loc[2].weight.data.zero_()
+        self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
+
+    # Spatial transformer network forward function
+    def stn(self, x):
+        xs = self.localization(x)
+        xs = xs.view(-1, 10 * 3 * 3)
+        theta = self.fc_loc(xs)
+        theta = theta.view(-1, 2, 3)
+
+        grid = F.affine_grid(theta, x.size())
+        x = F.grid_sample(x, grid)
+
+        return x
+
+    def forward(self, x):
+        # transform the input
+        x = self.stn(x)
+
+        # Perform the usual forward pass
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
+        x = x.view(-1, 320)
+        x = F.relu(self.fc1(x))
+        x = F.dropout(x, training=self.training)
+        x = self.fc2(x)
+        return F.log_softmax(x, dim=1)
+class NetMnist(nn.Module):
+    def __init__(self):
+        super(NetMnist, self).__init__()
+        self.conv1 = nn.Conv2d(1, 4, kernel_size=5)
+        self.conv2 = nn.Conv2d(4, 8, kernel_size=3)
+        self.conv3 = nn.Conv2d(8, 16, kernel_size=5)
+        self.fc1 = nn.Linear(1*16, 10)
+
+    def forward(self, x):
+
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2(x), 2))
+        x = F.relu(self.conv3(x), 2)
+
+        x = x.view(-1, 1*16)
+        x = F.relu(self.fc1(x))
+
+        return F.log_softmax(x, dim=1)
+
+
+#model = torch.nn.DataParallel(NetMnist()).to(device)
+model = (NetMnist()).to(device)
+
+
+optimizer = optim.SGD(model.parameters(), lr=0.01)
+
+
+
+ad = None;
+
+def train(epoch):
+    model.train()
+    for batch_idx, (data, target) in enumerate(train_loader):
+        data, target = data.to(device), target.to(device)
+        optimizer.zero_grad()
+        global ad
+
+        if ad is None:
+            ad=data
+
+        # nda=data.numpy()
+        # nao=np.append(nda,nda,1)
+        # nao = np.append(nao, nda, 1)
+        # data=torch.tensor(nao)
+
+
+        output = model(data)
+        loss = F.nll_loss(output, target)
+        loss.backward()
+        optimizer.step()
+
+        if batch_idx % 930 == 0 and batch_idx>0 :
+            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data), len(train_loader.dataset),100. * batch_idx / len(train_loader), loss.item()))
+
+
+def test():
+    with torch.no_grad():
+        model.eval()
+        test_loss = 0
+        correct = 0
+        for data, target in test_loader:
+            data, target = data.to(device), target.to(device)
+            output = model(data)
+
+            # sum up batch loss
+            test_loss += F.nll_loss(output, target, size_average=False).item()
+            # get the index of the max log-probability
+            pred = output.max(1, keepdim=True)[1]
+            correct += pred.eq(target.view_as(pred)).sum().item()
+
+        test_loss /= len(test_loader.dataset)
+        print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(test_loss, correct, len(test_loader.dataset),100. * correct / len(test_loader.dataset)))
+
+imaglis1=[]
+imaglis2=[]
+imaglis3=[]
+imaglis4=[]
+imaglis5=[]
+imaglis6=[]
+
+for epoch in range(1, 100 + 1):
+    train(epoch)
+
+
+
+
+
+    mm1 = model.conv1(ad[0:1, :, :, :])
+    datodis = mm1 * 256 + 128
+    dashape = datodis.shape
+    datodis = datodis.view(dashape[1], 1, dashape[2], dashape[3])
+    imaglis1.append(datodis.detach().cpu().numpy())
+
+    imaglisnp = np.array(imaglis1)
+    cdsa = np.reshape(imaglisnp, newshape=[-1, 1, imaglisnp.shape[3], imaglisnp.shape[4]])
+    viz.images(cdsa, win=imagewin1, opts=dict(title='Random!', caption='How random.'), nrow=dashape[1])
+
+
+    mm2 = model.conv2(F.relu(F.max_pool2d(mm1, 2)))
+    datodis = mm2 * 256 + 128
+    dashape = datodis.shape
+    datodis = datodis.view(dashape[1], 1, dashape[2], dashape[3])
+    imaglis2.append(datodis.detach().cpu().numpy())
+
+    imaglisnp = np.array(imaglis2)
+    cdsa = np.reshape(imaglisnp, newshape=[-1, 1, imaglisnp.shape[3], imaglisnp.shape[4]])
+    viz.images(cdsa, win=imagewin2, opts=dict(title='Random!', caption='How random.'), nrow=dashape[1])
+
+
+    mm3 = model.conv3(F.relu(F.max_pool2d(mm2, 2)))
+    datodis = mm3 * 256 + 128
+    dashape = datodis.shape
+    datodis = datodis.view(dashape[1], 1, dashape[2], dashape[3])
+    imaglis3.append(datodis.detach().cpu().numpy())
+
+    imaglisnp = np.array(imaglis3)
+    cdsa = np.reshape(imaglisnp, newshape=[-1, 1, imaglisnp.shape[3], imaglisnp.shape[4]])
+    viz.images(cdsa, win=imagewin3, opts=dict(title='Random!', caption='How random.'), nrow=dashape[1])
+
+
+
+
+
+    mm4 = model.conv1.weight.data
+    datodis = mm4 * 256 + 128
+    dashape = datodis.shape
+    datodis = datodis.view(dashape[1]*dashape[0], 1, dashape[2], dashape[3])
+    imaglis4.append(datodis.detach().cpu().numpy())
+
+    imaglisnp = np.array(imaglis4)
+    cdsa = np.reshape(imaglisnp, newshape=[-1, 1, imaglisnp.shape[3], imaglisnp.shape[4]])
+    viz.images(cdsa, win=imagewin4, opts=dict(title='Random!', caption='How random.'), nrow=dashape[1]*dashape[0])
+
+
+
+
+
+
+
+
+
+    # mm = model.conv1(ad)
+    # datodis = mm * 256 + 128
+    # datodis = datodis.view(64, 1, 24 * 10, 24)
+    # imaglis.append(datodis.detach().cpu().numpy()[0:8,:,:,:])
+
+    test()
+
+
+#cdsa=np.reshape(np.array(imaglis),newshape=[-1,1,240,24])
+#viz.images(cdsa, win=imagewin, opts=dict(title='Random!', caption='How random.'), nrow=8,padding=2)
+
+
+
+# Visualize the STN transformation on some input batch
+# visualize_stn()
+
+#plt.ioff()
+#plt.show()

PaintVgg19/CaffeToPytorch.py

@@ -0,0 +1,4 @@
+#python -m mmdnn.conversion._script.convertToIR -f caffe -d vgg19Caffe -n VGG_ILSVRC_19_layers_deploy.prototxt -w VGG_ILSVRC_19_layers.caffemodel
+#python -m mmdnn.conversion._script.IRToCode -f pytorch --IRModelPath vgg19Caffe.pb --dstModelPath vgg19Pytorch.py --IRWeightPath vgg19Caffe.npy -dw vgg19Pytorch.npy
+
+

PaintVgg19/PaintVgg19.py

@@ -0,0 +1,153 @@
+from __future__ import print_function
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+import cv2
+import os
+import shutil
+
+from vgg19Pytorch import Vgg19Module
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# vgg19=Vgg19Module(CurrentPath+'/vgg19Pytorch.npy')
+vgg19 = torchvision.models.vgg19(True)
+
+if torch.cuda.is_available():
+    vgg19=vgg19.to(device)
+
+
+def readAndPreprocessImage(imagepath):
+    mean = np.array([103.939, 116.779, 123.68])
+    img=cv2.imread(imagepath)
+    img=img.astype('float32')
+    img -= mean
+    img = cv2.resize(img, (224, 224)).transpose((2, 0, 1))
+    img = img[np.newaxis, :, :, :]
+    # cv image read is  by BGR order
+    # and the network Need RGB
+    # the following BGR values should be subtracted: [103.939, 116.779, 123.68].
+    image = torch.from_numpy(img)
+    if torch.cuda.is_available():
+        image = image.cuda()
+    return image
+
+
+imgdir = CurrentPath+'/../Dataset/ILSVRC2012_img_val/'
+allImageData=[]
+allfile=[]
+for dirpath, dirnames, filenames in os.walk(imgdir):
+    for file in filenames[0:1000]:
+        allImageData.append(dirpath+"/"+file)
+        allfile.append(file)
+        
+
+resultdata=[]
+count=0
+
+for imagefile in allImageData:
+    count+=1
+    image=readAndPreprocessImage(imagefile)
+
+    conv1_1_pad     = F.pad(image, (1L, 1L, 1L, 1L))
+    conv1_1         = vgg19.conv1_1(conv1_1_pad)
+    relu1_1         = F.relu(conv1_1)
+    conv1_2_pad     = F.pad(relu1_1, (1L, 1L, 1L, 1L))
+    conv1_2         = vgg19.conv1_2(conv1_2_pad)
+    relu1_2         = F.relu(conv1_2)
+    pool1           = F.max_pool2d(relu1_2, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+    conv2_1_pad     = F.pad(pool1, (1L, 1L, 1L, 1L))
+    conv2_1         = vgg19.conv2_1(conv2_1_pad)
+    relu2_1         = F.relu(conv2_1)
+    conv2_2_pad     = F.pad(relu2_1, (1L, 1L, 1L, 1L))
+    conv2_2         = vgg19.conv2_2(conv2_2_pad)
+    relu2_2         = F.relu(conv2_2)
+    pool2           = F.max_pool2d(relu2_2, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+    conv3_1_pad     = F.pad(pool2, (1L, 1L, 1L, 1L))
+    conv3_1         = vgg19.conv3_1(conv3_1_pad)
+    relu3_1         = F.relu(conv3_1)
+    conv3_2_pad     = F.pad(relu3_1, (1L, 1L, 1L, 1L))
+    conv3_2         = vgg19.conv3_2(conv3_2_pad)
+    relu3_2         = F.relu(conv3_2)
+    conv3_3_pad     = F.pad(relu3_2, (1L, 1L, 1L, 1L))
+    conv3_3         = vgg19.conv3_3(conv3_3_pad)
+    relu3_3         = F.relu(conv3_3)
+    conv3_4_pad     = F.pad(relu3_3, (1L, 1L, 1L, 1L))
+    conv3_4         = vgg19.conv3_4(conv3_4_pad)
+    relu3_4         = F.relu(conv3_4)
+    pool3           = F.max_pool2d(relu3_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+    conv4_1_pad     = F.pad(pool3, (1L, 1L, 1L, 1L))
+    conv4_1         = vgg19.conv4_1(conv4_1_pad)
+    relu4_1         = F.relu(conv4_1)
+    conv4_2_pad     = F.pad(relu4_1, (1L, 1L, 1L, 1L))
+    conv4_2         = vgg19.conv4_2(conv4_2_pad)
+    relu4_2         = F.relu(conv4_2)
+    conv4_3_pad     = F.pad(relu4_2, (1L, 1L, 1L, 1L))
+    conv4_3         = vgg19.conv4_3(conv4_3_pad)
+    relu4_3         = F.relu(conv4_3)
+    conv4_4_pad     = F.pad(relu4_3, (1L, 1L, 1L, 1L))
+    conv4_4         = vgg19.conv4_4(conv4_4_pad)
+    relu4_4         = F.relu(conv4_4)
+    pool4           = F.max_pool2d(relu4_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+    conv5_1_pad     = F.pad(pool4, (1L, 1L, 1L, 1L))
+    conv5_1         = vgg19.conv5_1(conv5_1_pad)
+    relu5_1         = F.relu(conv5_1)
+    conv5_2_pad     = F.pad(relu5_1, (1L, 1L, 1L, 1L))
+    conv5_2         = vgg19.conv5_2(conv5_2_pad)
+    relu5_2         = F.relu(conv5_2)
+    conv5_3_pad     = F.pad(relu5_2, (1L, 1L, 1L, 1L))
+    conv5_3         = vgg19.conv5_3(conv5_3_pad)
+    relu5_3         = F.relu(conv5_3)
+    conv5_4_pad     = F.pad(relu5_3, (1L, 1L, 1L, 1L))
+    conv5_4         = vgg19.conv5_4(conv5_4_pad)
+
+    resultdata.append(conv5_3.detach().cpu().numpy()[0,204,:,:])
+    print("CalImageCount:"+str(count))
+
+
+his=np.histogram(resultdata,256)
+xx=np.array(his[0]).astype('float32')
+yy=np.array(his[1]).astype('float32')
+data=np.empty((2,256))
+data[1,:]=xx
+data[0,:]=yy[0:256]
+data=data.swapaxes(0,1)
+
+
+threvalue=500
+count=0
+for daimage in resultdata:
+    if np.max(daimage)>threvalue:
+        path1 = allImageData[count]
+        ima=cv2.imread(path1)
+        imagewidth=ima.shape[1]
+        imageheight=ima.shape[0]
+
+        datawidth=daimage.shape[1]
+        dataheight=daimage.shape[0]
+
+        widthradio=float(imagewidth)/float(datawidth)
+        heithradio=float(imageheight)/float(dataheight)
+
+        for i in range(dataheight):
+            for j in range(datawidth):
+                if daimage[i,j]>threvalue :
+                    cv2.circle(ima,(int(j*widthradio),int(i*heithradio)),15,(255,0,0),2);
+        path2 = CurrentPath+"/imagePointOut/" + allfile[count]
+        cv2.imwrite(path2,ima)
+    count+=1
+
+
+

PaintVgg19/PaintVgg19Weight.py

@@ -0,0 +1,176 @@
+from __future__ import print_function
+from vgg19Pytorch import Vgg19Module
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from visdom import Visdom
+import cv2
+import os
+import shutil
+
+
+viz = Visdom()
+# use visdom , we must start visdom Host first
+# with python -m visdom.server
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# vgg19 = Vgg19Module(CurrentPath+'/vgg19Pytorch.npy')
+vgg19 = torchvision.models.vgg19(True)
+
+if torch.cuda.is_available():
+    vgg19 = vgg19.to(device)
+
+
+def weightVisual(layer, name=''):
+    mm4 = layer.weight.data
+    datodis = mm4 * 256 + 128
+    dashape = datodis.shape
+    datodis = datodis.view(dashape[1] * dashape[0], 1, dashape[2], dashape[3])
+    imaglisnp = datodis.detach().cpu().numpy()
+    viz.images(imaglisnp, opts=dict(
+        title=name, caption='How random.'), nrow=dashape[1])
+
+
+def imageVisual(data, name='', rownum=8):
+    datodis = data
+    #datodis = datodis * 256 + 128
+    dashape = datodis.shape
+    datodis = datodis.view(dashape[1] * dashape[0], 1, dashape[2], dashape[3])
+    imaglisnp = datodis.detach().cpu().numpy()
+    viz.images(imaglisnp, opts=dict(
+        title=name, caption='How random.'), nrow=rownum)
+
+
+def readAndPreprocessImage(imagepath):
+    mean = np.array([103.939, 116.779, 123.68])
+    img = cv2.imread(imagepath)
+    img = img.astype('float32')
+    img -= mean
+    img = cv2.resize(img, (224, 224)).transpose((2, 0, 1))
+    img = img[np.newaxis, :, :, :]
+    # cv image read is  by BGR order
+    # and the network Need RGB
+    # the following BGR values should be subtracted: [103.939, 116.779, 123.68].
+    image = torch.from_numpy(img)
+    if torch.cuda.is_available():
+        image = image.cuda()
+    return image
+
+
+#region forward image
+image = readAndPreprocessImage(CurrentPath+"dog_resize.jpg")
+imgfor = vgg19.forward(image)
+imgfornumpy = imgfor.cpu().detach().numpy()
+
+words = open(CurrentPath+'synset_words.txt').readlines()
+words = [(w[0], ' '.join(w[1:])) for w in [w.split() for w in words]]
+words = np.asarray(words)
+
+top5 = np.argsort(imgfornumpy)[0][::-1][:5]
+probs = np.sort(imgfornumpy)[0][::-1][:5]
+for w, p in zip(words[top5], probs):
+    print('{}\tprobability:{}'.format(w, p))
+#endregion
+
+# region write image and weight
+image = readAndPreprocessImage(CurrentPath+"dog_resize.jpg")
+conv1_1_pad = F.pad(image, (1L, 1L, 1L, 1L))
+conv1_1 = vgg19.conv1_1(conv1_1_pad)
+relu1_1 = F.relu(conv1_1)
+conv1_2_pad = F.pad(relu1_1, (1L, 1L, 1L, 1L))
+conv1_2 = vgg19.conv1_2(conv1_2_pad)
+relu1_2 = F.relu(conv1_2)
+pool1 = F.max_pool2d(relu1_2, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+conv2_1_pad = F.pad(pool1, (1L, 1L, 1L, 1L))
+conv2_1 = vgg19.conv2_1(conv2_1_pad)
+relu2_1 = F.relu(conv2_1)
+conv2_2_pad = F.pad(relu2_1, (1L, 1L, 1L, 1L))
+conv2_2 = vgg19.conv2_2(conv2_2_pad)
+relu2_2 = F.relu(conv2_2)
+pool2 = F.max_pool2d(relu2_2, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+conv3_1_pad = F.pad(pool2, (1L, 1L, 1L, 1L))
+conv3_1 = vgg19.conv3_1(conv3_1_pad)
+relu3_1 = F.relu(conv3_1)
+conv3_2_pad = F.pad(relu3_1, (1L, 1L, 1L, 1L))
+conv3_2 = vgg19.conv3_2(conv3_2_pad)
+relu3_2 = F.relu(conv3_2)
+conv3_3_pad = F.pad(relu3_2, (1L, 1L, 1L, 1L))
+conv3_3 = vgg19.conv3_3(conv3_3_pad)
+relu3_3 = F.relu(conv3_3)
+conv3_4_pad = F.pad(relu3_3, (1L, 1L, 1L, 1L))
+conv3_4 = vgg19.conv3_4(conv3_4_pad)
+relu3_4 = F.relu(conv3_4)
+pool3 = F.max_pool2d(relu3_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+conv4_1_pad = F.pad(pool3, (1L, 1L, 1L, 1L))
+conv4_1 = vgg19.conv4_1(conv4_1_pad)
+relu4_1 = F.relu(conv4_1)
+conv4_2_pad = F.pad(relu4_1, (1L, 1L, 1L, 1L))
+conv4_2 = vgg19.conv4_2(conv4_2_pad)
+relu4_2 = F.relu(conv4_2)
+conv4_3_pad = F.pad(relu4_2, (1L, 1L, 1L, 1L))
+conv4_3 = vgg19.conv4_3(conv4_3_pad)
+relu4_3 = F.relu(conv4_3)
+conv4_4_pad = F.pad(relu4_3, (1L, 1L, 1L, 1L))
+conv4_4 = vgg19.conv4_4(conv4_4_pad)
+relu4_4 = F.relu(conv4_4)
+pool4 = F.max_pool2d(relu4_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+conv5_1_pad = F.pad(pool4, (1L, 1L, 1L, 1L))
+conv5_1 = vgg19.conv5_1(conv5_1_pad)
+relu5_1 = F.relu(conv5_1)
+conv5_2_pad = F.pad(relu5_1, (1L, 1L, 1L, 1L))
+conv5_2 = vgg19.conv5_2(conv5_2_pad)
+relu5_2 = F.relu(conv5_2)
+conv5_3_pad = F.pad(relu5_2, (1L, 1L, 1L, 1L))
+conv5_3 = vgg19.conv5_3(conv5_3_pad)
+relu5_3 = F.relu(conv5_3)
+conv5_4_pad = F.pad(relu5_3, (1L, 1L, 1L, 1L))
+conv5_4 = vgg19.conv5_4(conv5_4_pad)
+relu5_4 = F.relu(conv5_4)
+pool5 = F.max_pool2d(relu5_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+fc6_0 = pool5.view(pool5.size(0), -1)
+
+imageVisual(conv1_1, "conv1_1", 8)
+imageVisual(conv1_2, 'conv1_2', 8)
+imageVisual(conv2_1, "conv2_1", 16)
+imageVisual(conv2_2, "conv2_2", 16)
+imageVisual(conv3_1, "conv3_1", 16)
+imageVisual(conv3_2, "conv3_2", 16)
+imageVisual(conv3_3, "conv3_3", 16)
+imageVisual(conv3_4, "conv3_4", 16)
+imageVisual(conv4_1, "conv4_1", 32)
+imageVisual(conv4_2, "conv4_2", 32)
+imageVisual(conv4_3, "conv4_3", 32)
+imageVisual(conv4_4, "conv4_4", 32)
+imageVisual(conv5_1, "conv5_1", 32)
+imageVisual(conv5_2, "conv5_2", 32)
+imageVisual(conv5_3, "conv5_3", 32)
+imageVisual(conv5_4, "conv5_4", 32)
+
+weightVisual(vgg19.conv1_1, "conv1_1")
+weightVisual(vgg19.conv1_2, 'conv1_2')
+weightVisual(vgg19.conv2_1, "conv2_1")
+weightVisual(vgg19.conv2_2, "conv2_2")
+weightVisual(vgg19.conv3_1, "conv3_1")
+weightVisual(vgg19.conv3_2, "conv3_2")
+weightVisual(vgg19.conv3_3, "conv3_3")
+weightVisual(vgg19.conv3_4, "conv3_4")
+weightVisual(vgg19.conv4_1, "conv4_1")
+weightVisual(vgg19.conv4_2, "conv4_2")
+weightVisual(vgg19.conv4_3, "conv4_3")
+weightVisual(vgg19.conv4_4, "conv4_4")
+weightVisual(vgg19.conv5_1, "conv5_1")
+weightVisual(vgg19.conv5_2, "conv5_2")
+weightVisual(vgg19.conv5_3, "conv5_3")
+weightVisual(vgg19.conv5_4, "conv5_4")
+# endregion

PaintVgg19/vgg19Pytorch.py

@@ -0,0 +1,133 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+__weights_dict = dict()
+
+def load_weights(weight_file):
+    if weight_file == None:
+        return
+
+    try:
+        weights_dict = np.load(weight_file).item()
+    except:
+        weights_dict = np.load(weight_file, encoding='bytes').item()
+
+    return weights_dict
+
+class Vgg19Module(nn.Module):
+
+    
+    def __init__(self, weight_file):
+        super(Vgg19Module, self).__init__()
+        global __weights_dict
+        __weights_dict = load_weights(weight_file)
+
+        self.conv1_1 = self.__conv(2, name='conv1_1', in_channels=3, out_channels=64, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv1_2 = self.__conv(2, name='conv1_2', in_channels=64, out_channels=64, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv2_1 = self.__conv(2, name='conv2_1', in_channels=64, out_channels=128, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv2_2 = self.__conv(2, name='conv2_2', in_channels=128, out_channels=128, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv3_1 = self.__conv(2, name='conv3_1', in_channels=128, out_channels=256, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv3_2 = self.__conv(2, name='conv3_2', in_channels=256, out_channels=256, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv3_3 = self.__conv(2, name='conv3_3', in_channels=256, out_channels=256, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv3_4 = self.__conv(2, name='conv3_4', in_channels=256, out_channels=256, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv4_1 = self.__conv(2, name='conv4_1', in_channels=256, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv4_2 = self.__conv(2, name='conv4_2', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv4_3 = self.__conv(2, name='conv4_3', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv4_4 = self.__conv(2, name='conv4_4', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv5_1 = self.__conv(2, name='conv5_1', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv5_2 = self.__conv(2, name='conv5_2', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv5_3 = self.__conv(2, name='conv5_3', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.conv5_4 = self.__conv(2, name='conv5_4', in_channels=512, out_channels=512, kernel_size=(3L, 3L), stride=(1L, 1L), groups=1, bias=True)
+        self.fc6_1 = self.__dense(name = 'fc6_1', in_features = 25088, out_features = 4096, bias = True)
+        self.fc7_1 = self.__dense(name = 'fc7_1', in_features = 4096, out_features = 4096, bias = True)
+        self.fc8_1 = self.__dense(name = 'fc8_1', in_features = 4096, out_features = 1000, bias = True)
+
+    def forward(self, x):
+        conv1_1_pad     = F.pad(x, (1L, 1L, 1L, 1L))
+        conv1_1         = self.conv1_1(conv1_1_pad)
+        relu1_1         = F.relu(conv1_1)
+        conv1_2_pad     = F.pad(relu1_1, (1L, 1L, 1L, 1L))
+        conv1_2         = self.conv1_2(conv1_2_pad)
+        relu1_2         = F.relu(conv1_2)
+        pool1           = F.max_pool2d(relu1_2, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+        conv2_1_pad     = F.pad(pool1, (1L, 1L, 1L, 1L))
+        conv2_1         = self.conv2_1(conv2_1_pad)
+        relu2_1         = F.relu(conv2_1)
+        conv2_2_pad     = F.pad(relu2_1, (1L, 1L, 1L, 1L))
+        conv2_2         = self.conv2_2(conv2_2_pad)
+        relu2_2         = F.relu(conv2_2)
+        pool2           = F.max_pool2d(relu2_2, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+        conv3_1_pad     = F.pad(pool2, (1L, 1L, 1L, 1L))
+        conv3_1         = self.conv3_1(conv3_1_pad)
+        relu3_1         = F.relu(conv3_1)
+        conv3_2_pad     = F.pad(relu3_1, (1L, 1L, 1L, 1L))
+        conv3_2         = self.conv3_2(conv3_2_pad)
+        relu3_2         = F.relu(conv3_2)
+        conv3_3_pad     = F.pad(relu3_2, (1L, 1L, 1L, 1L))
+        conv3_3         = self.conv3_3(conv3_3_pad)
+        relu3_3         = F.relu(conv3_3)
+        conv3_4_pad     = F.pad(relu3_3, (1L, 1L, 1L, 1L))
+        conv3_4         = self.conv3_4(conv3_4_pad)
+        relu3_4         = F.relu(conv3_4)
+        pool3           = F.max_pool2d(relu3_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+        conv4_1_pad     = F.pad(pool3, (1L, 1L, 1L, 1L))
+        conv4_1         = self.conv4_1(conv4_1_pad)
+        relu4_1         = F.relu(conv4_1)
+        conv4_2_pad     = F.pad(relu4_1, (1L, 1L, 1L, 1L))
+        conv4_2         = self.conv4_2(conv4_2_pad)
+        relu4_2         = F.relu(conv4_2)
+        conv4_3_pad     = F.pad(relu4_2, (1L, 1L, 1L, 1L))
+        conv4_3         = self.conv4_3(conv4_3_pad)
+        relu4_3         = F.relu(conv4_3)
+        conv4_4_pad     = F.pad(relu4_3, (1L, 1L, 1L, 1L))
+        conv4_4         = self.conv4_4(conv4_4_pad)
+        relu4_4         = F.relu(conv4_4)
+        pool4           = F.max_pool2d(relu4_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+        conv5_1_pad     = F.pad(pool4, (1L, 1L, 1L, 1L))
+        conv5_1         = self.conv5_1(conv5_1_pad)
+        relu5_1         = F.relu(conv5_1)
+        conv5_2_pad     = F.pad(relu5_1, (1L, 1L, 1L, 1L))
+        conv5_2         = self.conv5_2(conv5_2_pad)
+        relu5_2         = F.relu(conv5_2)
+        conv5_3_pad     = F.pad(relu5_2, (1L, 1L, 1L, 1L))
+        conv5_3         = self.conv5_3(conv5_3_pad)
+        relu5_3         = F.relu(conv5_3)
+        conv5_4_pad     = F.pad(relu5_3, (1L, 1L, 1L, 1L))
+        conv5_4         = self.conv5_4(conv5_4_pad)
+        relu5_4         = F.relu(conv5_4)
+        pool5           = F.max_pool2d(relu5_4, kernel_size=(2L, 2L), stride=(2L, 2L), padding=(0L,), ceil_mode=True)
+        fc6_0           = pool5.view(pool5.size(0), -1)
+        fc6_1           = self.fc6_1(fc6_0)
+        relu6           = F.relu(fc6_1)
+        drop6           = F.dropout(input = relu6, p = 0.5, training = self.training, inplace = True)
+        fc7_0           = drop6.view(drop6.size(0), -1)
+        fc7_1           = self.fc7_1(fc7_0)
+        relu7           = F.relu(fc7_1)
+        drop7           = F.dropout(input = relu7, p = 0.5, training = self.training, inplace = True)
+        fc8_0           = drop7.view(drop7.size(0), -1)
+        fc8_1           = self.fc8_1(fc8_0)
+        prob            = F.softmax(fc8_1)
+        return prob
+
+
+    @staticmethod
+    def __conv(dim, name, **kwargs):
+        if   dim == 1:  layer = nn.Conv1d(**kwargs)
+        elif dim == 2:  layer = nn.Conv2d(**kwargs)
+        elif dim == 3:  layer = nn.Conv3d(**kwargs)
+        else:           raise NotImplementedError()
+
+        layer.state_dict()['weight'].copy_(torch.from_numpy(__weights_dict[name]['weights']))
+        if 'bias' in __weights_dict[name]:
+            layer.state_dict()['bias'].copy_(torch.from_numpy(__weights_dict[name]['bias']))
+        return layer
+
+    @staticmethod
+    def __dense(name, **kwargs):
+        layer = nn.Linear(**kwargs)
+        layer.state_dict()['weight'].copy_(torch.from_numpy(__weights_dict[name]['weights']))
+        if 'bias' in __weights_dict[name]:
+            layer.state_dict()['bias'].copy_(torch.from_numpy(__weights_dict[name]['bias']))
+        return layer

RBM/README.md

@@ -0,0 +1,9 @@
+# Restricted Boltzmann Machines (RBMs) in PyTorch
+
+> **Author:** [Gabriel Bianconi](http://www.gabrielbianconi.com/)
+
+## Overview
+
+This project implements Restricted Boltzmann Machines (RBMs) using PyTorch (see `rbm.py`). Our implementation includes momentum, weight decay, L2 regularization, and CD-*k* contrastive divergence. We also provide support for CPU and GPU (CUDA) calculations.
+
+In addition, we provide an example file applying our model to the MNIST dataset (see `mnist_dataset.py`). The example trains an RBM, uses the trained model to extract features from the images, and finally uses a SciPy-based logistic regression for classification. It achieves 92.8% classification accuracy (this is obviously not a cutting-edge model).

RBM/mnist_example.py

@@ -0,0 +1,95 @@
+import os
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+import torch
+import torchvision.datasets
+import torchvision.models
+import torchvision.transforms
+
+from rbm import RBM
+
+
+########## CONFIGURATION ##########
+BATCH_SIZE = 64
+VISIBLE_UNITS = 784  # 28 x 28 images
+HIDDEN_UNITS = 128
+CD_K = 2
+EPOCHS = 100
+
+DATA_FOLDER = os.path.split(os.path.realpath(__file__))[0]+'/../Dataset/'
+print("Dataset Path :" + DATA_FOLDER)
+
+CUDA = torch.cuda.is_available()
+CUDA_DEVICE = 0
+
+if CUDA:
+    torch.cuda.set_device(CUDA_DEVICE)
+
+
+########## LOADING DATASET ##########
+print('Loading dataset...')
+
+train_dataset = torchvision.datasets.MNIST(root=DATA_FOLDER, train=True, transform=torchvision.transforms.ToTensor(), download=True)
+train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE)
+
+test_dataset = torchvision.datasets.MNIST(root=DATA_FOLDER, train=False, transform=torchvision.transforms.ToTensor(), download=True)
+test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE)
+
+
+########## TRAINING RBM ##########
+print('Training RBM...')
+
+rbm = RBM(VISIBLE_UNITS, HIDDEN_UNITS, CD_K, use_cuda=CUDA)
+
+for epoch in range(EPOCHS):
+    epoch_error = 0.0
+
+    for batch, _ in train_loader:
+        batch = batch.view(len(batch), VISIBLE_UNITS)  # flatten input data
+
+        if CUDA:
+            batch = batch.cuda()
+
+        batch_error = rbm.contrastive_divergence(batch)
+
+        epoch_error += batch_error
+
+    print('Epoch Error (epoch=%d): %.4f' % (epoch, epoch_error))
+
+
+########## EXTRACT FEATURES ##########
+print('Extracting features...')
+
+train_features = np.zeros((len(train_dataset), HIDDEN_UNITS))
+train_labels = np.zeros(len(train_dataset))
+test_features = np.zeros((len(test_dataset), HIDDEN_UNITS))
+test_labels = np.zeros(len(test_dataset))
+
+for i, (batch, labels) in enumerate(train_loader):
+    batch = batch.view(len(batch), VISIBLE_UNITS)  # flatten input data
+
+    if CUDA:
+        batch = batch.cuda()
+
+    train_features[i*BATCH_SIZE:i*BATCH_SIZE+len(batch)] = rbm.sample_hidden(batch).cpu().numpy()
+    train_labels[i*BATCH_SIZE:i*BATCH_SIZE+len(batch)] = labels.numpy()
+
+for i, (batch, labels) in enumerate(test_loader):
+    batch = batch.view(len(batch), VISIBLE_UNITS)  # flatten input data
+
+    if CUDA:
+        batch = batch.cuda()
+
+    test_features[i*BATCH_SIZE:i*BATCH_SIZE+len(batch)] = rbm.sample_hidden(batch).cpu().numpy()
+    test_labels[i*BATCH_SIZE:i*BATCH_SIZE+len(batch)] = labels.numpy()
+
+
+########## CLASSIFICATION ##########
+print('Classifying...')
+
+clf = LogisticRegression()
+clf.fit(train_features, train_labels)
+predictions = clf.predict(test_features)
+
+print('Result: %d/%d' % (sum(predictions == test_labels), test_labels.shape[0]))
+

RBM/rbm.py

@@ -0,0 +1,95 @@
+import torch
+
+
+class RBM():
+
+    def __init__(self, num_visible, num_hidden, k, learning_rate=1e-3, momentum_coefficient=0.5, weight_decay=1e-4, use_cuda=True):
+        self.num_visible = num_visible
+        self.num_hidden = num_hidden
+        self.k = k
+        self.learning_rate = learning_rate
+        self.momentum_coefficient = momentum_coefficient
+        self.weight_decay = weight_decay
+        self.use_cuda = use_cuda
+
+        self.weights = torch.randn(num_visible, num_hidden) * 0.1
+        self.visible_bias = torch.ones(num_visible) * 0.5
+        self.hidden_bias = torch.zeros(num_hidden)
+
+        self.weights_momentum = torch.zeros(num_visible, num_hidden)
+        self.visible_bias_momentum = torch.zeros(num_visible)
+        self.hidden_bias_momentum = torch.zeros(num_hidden)
+
+        if self.use_cuda:
+            self.weights = self.weights.cuda()
+            self.visible_bias = self.visible_bias.cuda()
+            self.hidden_bias = self.hidden_bias.cuda()
+
+            self.weights_momentum = self.weights_momentum.cuda()
+            self.visible_bias_momentum = self.visible_bias_momentum.cuda()
+            self.hidden_bias_momentum = self.hidden_bias_momentum.cuda()
+
+    def sample_hidden(self, visible_probabilities):
+        hidden_activations = torch.matmul(visible_probabilities, self.weights) + self.hidden_bias
+        hidden_probabilities = self._sigmoid(hidden_activations)
+        return hidden_probabilities
+
+    def sample_visible(self, hidden_probabilities):
+        visible_activations = torch.matmul(hidden_probabilities, self.weights.t()) + self.visible_bias
+        visible_probabilities = self._sigmoid(visible_activations)
+        return visible_probabilities
+
+    def contrastive_divergence(self, input_data):
+        # Positive phase
+        positive_hidden_probabilities = self.sample_hidden(input_data)
+        positive_hidden_activations = (positive_hidden_probabilities >= self._random_probabilities(self.num_hidden)).float()
+        positive_associations = torch.matmul(input_data.t(), positive_hidden_activations)
+
+        # Negative phase
+        hidden_activations = positive_hidden_activations
+
+        for step in range(self.k):
+            visible_probabilities = self.sample_visible(hidden_activations)
+            hidden_probabilities = self.sample_hidden(visible_probabilities)
+            hidden_activations = (hidden_probabilities >= self._random_probabilities(self.num_hidden)).float()
+
+        negative_visible_probabilities = visible_probabilities
+        negative_hidden_probabilities = hidden_probabilities
+
+        negative_associations = torch.matmul(negative_visible_probabilities.t(), negative_hidden_probabilities)
+
+        # Update parameters
+        self.weights_momentum *= self.momentum_coefficient
+        self.weights_momentum += (positive_associations - negative_associations)
+
+        self.visible_bias_momentum *= self.momentum_coefficient
+        self.visible_bias_momentum += torch.sum(input_data - negative_visible_probabilities, dim=0)
+
+        self.hidden_bias_momentum *= self.momentum_coefficient
+        self.hidden_bias_momentum += torch.sum(positive_hidden_probabilities - negative_hidden_probabilities, dim=0)
+
+        batch_size = input_data.size(0)
+
+        self.weights += self.weights_momentum * self.learning_rate / batch_size
+        self.visible_bias += self.visible_bias_momentum * self.learning_rate / batch_size
+        self.hidden_bias += self.hidden_bias_momentum * self.learning_rate / batch_size
+
+        self.weights -= self.weights * self.weight_decay  # L2 weight decay
+
+        # Compute reconstruction error
+        error = torch.sum((input_data - negative_visible_probabilities)**2)
+
+        return error
+
+    def _sigmoid(self, x):
+        return 1 / (1 + torch.exp(-x))
+
+    def _random_probabilities(self, num):
+        random_probabilities = torch.rand(num)
+
+        if self.use_cuda:
+            random_probabilities = random_probabilities.cuda()
+
+        return random_probabilities
+
+

VisualNetwork/TrainNetwork.py

@@ -0,0 +1,147 @@
+from __future__ import print_function
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+from model import Net
+from torch.utils.data import Dataset, DataLoader
+from PIL import Image
+import random
+import cv2
+
+
+batchsize = 16
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print("device: " + str(device))
+
+
+class ManualDataset(Dataset):
+    def __init__(self, transform=None):
+        self.transform = transform
+
+    def __getitem__(self, index):
+
+        # data = np.random.random_integers(0,255,(28,28))
+        data = np.zeros((28, 28), dtype="float32")
+        data = data.astype("float32")
+
+        radiu = int(random.random()*7+3)
+        basex = random.randint(radiu, 29-radiu)
+        basey = random.randint(radiu, 29-radiu)
+        ifcircle = random.random()
+
+        if ifcircle>0.5:
+            cv2.circle(data,(basex,basey),radiu,255,-1)
+            angle = random.random() * 360
+            M = cv2.getRotationMatrix2D((basex, basey), angle, 1.0)
+            data = cv2.warpAffine(data, M, (28, 28))
+            label = 0
+        else:
+            cv2.rectangle(data,(basex-radiu,basey-radiu),(basex+radiu,basey+radiu),255,-1)
+            angle = random.random() * 360
+            M = cv2.getRotationMatrix2D((basex,basey), angle, 1.0)
+            data = cv2.warpAffine(data, M, (28,28))
+            label = 1
+
+        # cv2.imwrite("test.jpg",data)
+
+        data = (data - 128) / 256.0
+        img = torch.from_numpy(data)
+        img = img.view(1, 28, 28)
+        return img, label
+
+    def __len__(self):
+        return 10000
+
+
+train_loader = torch.utils.data.DataLoader(
+    ManualDataset(transform=transforms.Compose([
+        # transforms.ColorJitter(0.2,0.2),
+        # transforms.RandomRotation(30),
+        # transforms.RandomResizedCrop(28),
+        transforms.ToTensor(),
+        transforms.Normalize((128,), (256,)), ])),
+    batch_size=batchsize, shuffle=True, num_workers=2)
+test_loader = torch.utils.data.DataLoader(
+    ManualDataset(transform=transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((128,), (256,))])),
+    batch_size=batchsize, shuffle=True, num_workers=2)
+
+
+# train_loader = torch.utils.data.DataLoader(
+#         datasets.MNIST(root='.', train=True, download=False,
+#                        transform=transforms.Compose([
+#                            transforms.ColorJitter(0.2,0.2),
+#                            transforms.RandomRotation(30),
+#                            transforms.RandomResizedCrop(28),
+#                            transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))])),
+#                        batch_size=batchsize, shuffle=True, num_workers=4)
+# test_loader = torch.utils.data.DataLoader(
+#         datasets.MNIST(root='.', train=False, transform=transforms.Compose([
+#             transforms.ToTensor(),
+#             transforms.Normalize((0.1307,), (0.3081,))
+#         ])), batch_size=batchsize, shuffle=True, num_workers=4)
+
+
+model = (Net()).to(device)
+optimizer = optim.SGD(model.parameters(), lr=0.01)
+
+
+def train(epoch):
+    model.train()
+    for batch_idx, (data, target) in enumerate(train_loader):
+        data, target = data.to(device), target.to(device)
+        optimizer.zero_grad()
+
+        output = model(data)
+        loss = F.nll_loss(output, target)
+        loss.backward()
+        optimizer.step()
+
+        if batch_idx % 100 == 0 and batch_idx > 0:
+            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch, batch_idx * len(data),
+                                                                           len(train_loader.dataset),
+                                                                           100. * batch_idx / len(train_loader),
+                                                                           loss.item()))
+
+
+def test():
+    with torch.no_grad():
+        model.eval()
+        test_loss = 0
+        correct = 0
+        for data, target in test_loader:
+            data, target = data.to(device), target.to(device)
+            output = model(data)
+
+            # sum up batch loss
+            test_loss += F.nll_loss(output, target, size_average=False).item()
+            # get the index of the max log-probability
+            pred = output.max(1, keepdim=True)[1]
+            correct += pred.eq(target.view_as(pred)).sum().item()
+
+        test_loss /= len(test_loader.dataset)
+        print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(test_loss, correct,
+                                                                                     len(test_loader.dataset),
+                                                                                     100. * correct / len(
+                                                                                         test_loader.dataset)))
+
+
+for epoch in range(1, 15):
+    train(epoch)
+    test()
+
+torch.save(model.state_dict(), CurrentPath+"mnistcnn.pth.tar")

VisualNetwork/VisualNetwork.py

@@ -0,0 +1,133 @@
+from __future__ import print_function
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+import cv2
+from model import Net
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+
+image_out_path=CurrentPath+"/imageout/"
+if not os.path.exists(image_out_path):
+    os.mkdir(image_out_path)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+netmodel = Net()
+
+state_dict = torch.load(CurrentPath+"mnistcnn.pth.tar", map_location='cpu')
+from collections import OrderedDict
+
+new_state_dict = OrderedDict()
+for k, v in state_dict.items():
+    name = k
+    if k[0:7] == "module.":
+        name = k[7:]
+    new_state_dict[name] = v
+netmodel.load_state_dict(new_state_dict)
+netmodel.eval()
+
+
+def visualmodle(initimagefile, netmodel, layer, channel):
+    class Suggest(nn.Module):
+        def __init__(self, initdata=None):
+            super(Suggest, self).__init__()
+
+            # self.weight = nn.Parameter(torch.randn((1,1,28,28)))
+
+            if initdata is not None:
+                self.weight = nn.Parameter(initdata)
+            else:
+                data = np.random.uniform(-1, 1, (1, 1, 28, 28))
+                data = data.astype("float32")
+                data = torch.from_numpy(data)
+                self.weight = nn.Parameter(data)
+
+        def forward(self, x):
+            x = x * self.weight
+            return F.upsample(x, (28, 28), mode='bilinear', align_corners=True)
+
+    netmodel.eval()
+
+    if initimagefile is None:
+        model = Suggest(None)
+    else:
+        img = cv2.imread(initimagefile)
+        b, g, r = cv2.split(img)
+        img = cv2.merge([r, g, b])
+        img = img.astype("float32")
+        img = np.transpose(img, (2, 0, 1))
+        img = torch.from_numpy(img)
+        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        img = img / 256
+        img = normalize(img)
+        model = Suggest(img)
+
+    optimizer = optim.SGD(model.parameters(), lr=1.0)
+    model.train()
+    data = np.ones((1, 1, 28, 28), dtype="float32")
+    data = torch.from_numpy(data)
+
+    criterion = nn.MSELoss()
+    if torch.cuda.is_available():
+        criterion = criterion.cuda()
+        model = model.cuda()
+        netmodel = netmodel.cuda()
+        data = data.cuda()
+
+    for i in range(100):
+        output = model(data)
+
+        netout = []
+        netint = []
+
+        def getnet(self, input, output):
+            netout.append(output)
+            netint.append(input)
+
+        # print(netmodel.features)
+
+        handle = netmodel.features[layer].register_forward_hook(getnet)
+        output = netmodel(output)
+        output = netout[0][0,channel,:,:]
+
+        # output = output.view(1, 1, output.shape[0], output.shape[1])
+        # output = F.max_pool2d(output, netmodel.conv2.kernel_size[0])
+
+        netout = []
+        netint = []
+
+        # output = output.mean()
+        target = output + 256.0
+        target = target.detach()
+        loss = criterion(output, target)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        print('Train Inter:' + str(i) + "  loss:" + str(loss.cpu().detach().numpy()))
+        handle.remove()
+
+    model.eval()
+    output = model(data)
+    out = output.view(28, 28)
+    out = out.cpu().detach().numpy()
+    outmax = out.max()
+    outmin = out.min()
+    out = out * (256.0 / (outmax - outmin)) - outmin * (256.0 / (outmax - outmin))
+    return out
+
+
+
+for i in range(8):
+    out = visualmodle(None, netmodel, 3, i)
+    cv2.imwrite(image_out_path+"/L3_C" + str(i) + ".jpg", out)

VisualNetwork/VisualVgg19.py

@@ -0,0 +1,215 @@
+from __future__ import print_function
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision
+from torchvision import datasets, transforms
+import torchvision.models as models
+import matplotlib.pyplot as plt
+import numpy as np
+import cv2
+
+
+CurrentPath = os.path.split(os.path.realpath(__file__))[0]+"/"
+print("Current Path :" + CurrentPath)
+
+image_out_path=CurrentPath+"/imageoutVgg19/"
+if not os.path.exists(image_out_path):
+    os.mkdir(image_out_path)
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+#region define network myself
+# class Net(nn.Module):
+#     def __init__(self):
+#         super(Net, self).__init__()
+#         self.conv1 = nn.Conv2d(1, 4, kernel_size=5)
+#         self.conv2 = nn.Conv2d(4, 8, kernel_size=3)
+#         self.conv3 = nn.Conv2d(8, 16, kernel_size=5)
+#         self.fc1 = nn.Linear(1*16, 10)
+#
+#     def forward(self, x):
+#
+#         x = F.relu(F.max_pool2d(self.conv1(x), 2))
+#         x = F.relu(F.max_pool2d(self.conv2(x), 2))
+#         x = F.relu(self.conv3(x), 2)
+#
+#         x = x.view(-1, 1*16)
+#         x = F.relu(self.fc1(x))
+#
+#         return F.log_softmax(x, dim=1)
+#
+# netmodel = Net()
+#
+# state_dict = torch.load("mnistcnn.pth.tar", map_location='cpu')
+# from collections import OrderedDict
+# new_state_dict = OrderedDict()
+# for k, v in state_dict.items():
+#     name = k
+#     if k[0:7] == "module.":
+#         name = k[7:]
+#     new_state_dict[name] = v
+# netmodel.load_state_dict(new_state_dict)
+# netmodel.eval()
+
+
+
+# train_loader = torch.utils.data.DataLoader(
+#         datasets.MNIST(root='.', train=True, download=True,
+#                        transform=transforms.Compose([
+#                            transforms.ToTensor(),
+#                            transforms.Normalize((0.1307,), (0.3081,))
+#                        ])), batch_size=1, shuffle=True, num_workers=4)
+# test_loader = torch.utils.data.DataLoader(
+#         datasets.MNIST(root='.', train=False, transform=transforms.Compose([
+#             transforms.ToTensor(),
+#             transforms.Normalize((0.1307,), (0.3081,))
+#         ])), batch_size=1, shuffle=True, num_workers=4)
+#
+# for batch_idx, (data, target) in enumerate(train_loader):
+#     data, target = data.to(device), target.to(device)
+#
+#     output = netmodel(data)
+#     i=0
+
+
+#endregion define network myself
+
+#region define from torchvision models
+netmodel = torchvision.models.vgg19(True)
+netmodel.eval()
+
+
+# img = cv2.imread("t.jpg")
+# b,g,r = cv2.split(img)
+# img = cv2.merge([r,g,b])
+# img = img.astype("float32")
+# img = np.transpose(img,(2,0,1))
+# img = torch.from_numpy(img)
+#
+# normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+#                                  std=[0.229, 0.224, 0.225])
+# img = img/256
+# img = normalize(img)
+#
+# img = img.view(1,3,224,224)
+# out = netmodel(img)
+# out = out.view(-1).detach().numpy()
+# index = np.argmax(out)
+# value = out[index]
+# i=0
+#endregion define from torchvision models
+
+
+def visualmodle(initimagefile,netmodel,layer,channel):
+
+    class Suggest(nn.Module):
+        def __init__(self, initdata=None):
+            super(Suggest, self).__init__()
+
+            # self.weight = nn.Parameter(torch.randn((1,1,28,28)))
+
+            if initdata is not None:
+                self.weight = nn.Parameter(initdata)
+            else:
+                data = np.random.uniform(-1, 1, (1, 3, 224, 224))
+                data = data.astype("float32")
+                data = torch.from_numpy(data)
+                self.weight = nn.Parameter(data)
+
+        def forward(self, x):
+            x = x * self.weight
+            return F.upsample(x, (224, 224), mode='bilinear', align_corners=True)
+
+    netmodel.eval()
+
+    if initimagefile is None:
+        model = Suggest(None)
+    else:
+        img = cv2.imread(initimagefile)
+        b, g, r = cv2.split(img)
+        img = cv2.merge([r, g, b])
+        img = img.astype("float32")
+        img = np.transpose(img, (2, 0, 1))
+        img = torch.from_numpy(img)
+        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        img = img / 256
+        img = normalize(img)
+        model = Suggest(img)
+
+    optimizer = optim.SGD(model.parameters(), lr= 1.0)
+    model.train()
+    data = np.ones((1,3,224,224), dtype="float32")
+    data =  torch.from_numpy(data)
+
+    # target = np.zeros((1),dtype='int64')
+    # target[0]=14
+    # target = torch.from_numpy(target)
+
+    # criterion = nn.CrossEntropyLoss()
+    criterion = nn.MSELoss()
+    if torch.cuda.is_available():
+        criterion = criterion.cuda()
+        model = model.cuda()
+        netmodel = netmodel.cuda()
+        data = data.cuda()
+
+    for i in range(100):
+        output = model(data)
+
+        netout=[]
+        netint=[]
+        def getnet(self, input, output):
+            netout.append(output)
+            netint.append(input)
+        # print(netmodel.features)
+        handle = netmodel.features[layer].register_forward_hook(getnet)
+        output = netmodel(output)
+        output = netout[0][0,channel,:,:]
+        netout=[]
+        netint=[]
+
+        # output = output.mean()
+        target = output+256.0
+        target = target.detach()
+        loss = criterion(output, target)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        print('Train Inter:'+str(i) + "  loss:"+str(loss.cpu().detach().numpy()))
+        handle.remove()
+
+    # model = model.cpu()
+    # netmodel = netmodel.cpu()
+    # data = data.cpu()
+
+    model.eval()
+    output = model(data)
+    out = output.view(3,224,224)
+    out = out.cpu().detach().numpy()
+    outmax = out[0].max()
+    outmin = out[0].min()
+    out[0] = out[0] * (256.0/(outmax-outmin)) - outmin * (256.0/(outmax-outmin))
+    outmax = out[1].max()
+    outmin = out[1].min()
+    out[1] = out[1] * (256.0/(outmax-outmin)) - outmin * (256.0/(outmax-outmin))
+    outmax = out[2].max()
+    outmin = out[2].min()
+    out[2] = out[2] * (256.0/(outmax-outmin)) - outmin * (256.0/(outmax-outmin))
+    out = np.transpose(out,(1,2,0))
+    b,g,r = cv2.split(out)
+    out = cv2.merge([r,g,b])
+    out = out*2
+    out = out- (128*(2-1))
+    return out
+
+
+# 128  7
+# 512  30
+for i in range(512):
+    out = visualmodle(None,netmodel,36,i)
+    cv2.imwrite(image_out_path+"L36_C"+str(i)+".jpg",out)
+i=0