diff --git a/unsuper/Readme.md b/unsuper/Readme.md index d614bde..cb7a678 100644 --- a/unsuper/Readme.md +++ b/unsuper/Readme.md @@ -21,12 +21,20 @@ array([[4.3121886e+00, 3.2778070e-07, 8.9907879e-01, 2.1270120e+00, 1. 重复的权重,虽然权重看起来都一样,但是有稍微的不同,不是完全一样 2. 3x3太小了导致了样本的信噪比太低,大部分的样本切出来都是0 2. 5x5的时候会有网格状重复 + 1. 通过比较小的loss不断得训练(epoch很大),最终还是会变成所有kernel都一样的“网格布” + 1. 每个kernel都是数值很低的,很接近的结果 3. 7x7的时候边框区域问题 -## 发现的原因 +## 问题的调试 1. 几个卷积核都是重复的,网格状的 - 1. grad太大,learning rate 太大 - 2. grad太小,训练的epoch不合适 + 1. grad太大,learning rate 太大,grad太小,训练的epoch不合适 + 2. 网格状,就是为了尽量降低最终输出的绝对值,降低loss + 1. 形成4个一组,一共2组,的形式,交替成为最大值,把另外一组的输出降低,最后都输出最低的绝对值 + 2. 需要类似batchnormal的方式对各个conv核心之间进行归一化 + 3. 采用label不改变原来sample output的 abs均值的限制 + 1. 生成2个分别交替充当最大值的极端(只有2个像素不是0)的卷积核,其他的卷积核都是输出接近0的网格 + 4. 多个卷积核之间需要有差异,同一个卷积核的不同样本输入也要有差异,卷积核的分布要有要求 + 5. 每个卷积核尽量平摊权重到所有像素,而不是集中一个像素?提高鲁棒性? ## 可能的策略 1. 每个卷积核的改变权重(grad)能量守恒 diff --git a/unsuper/minist.py b/unsuper/minist.py index 78eb13b..858c29c 100644 --- a/unsuper/minist.py +++ b/unsuper/minist.py @@ -117,7 +117,7 @@ for epoch in range(epochs): images = images.to(device) # images = torch.ones((1, 1, 5, 5), device=device) - # type = random.randint(0, 10) + # type = random.randint(0, 7) # if type == 0: # rand = random.randint(0, 4) # images[:, :, rand, :] = images[:, :, rand, :] * 0.5 @@ -130,32 +130,45 @@ for epoch in range(epochs): # images[:, :, 2, 2] = images[:, :, 2, 2] * 0.5 # images[:, :, 3, 3] = images[:, :, 3, 3] * 0.5 # images[:, :, 4, 4] = images[:, :, 4, 4] * 0.5 + # if type == 3: + # randx = random.randint(0, 2) + # randy = random.randint(0, 2) + # images[:, :, randx, randy] = images[:, :, randx, randy] * 0.5 + # images[:, :, randx, randy + 1] = images[:, :, randx, randy + 1] * 0.5 + # images[:, :, randx, randy - 1] = images[:, :, randx, randy - 1] * 0.5 + # images[:, :, randx + 1, randy] = images[:, :, randx + 1, randy] * 0.5 + # images[:, :, randx - 1, randy] = images[:, :, randx - 1, randy] * 0.5 outputs = model.forward_unsuper(images) - outputs = outputs.permute(0, 2, 3, 1) # 64 8 24 24 -> 64 24 24 8 sample = outputs.reshape(-1, outputs.shape[3]) # -> 36864 8 + + + # sample = outputs.reshape(-1, 8,24*24) # -> 36864 8 + # sample = torch.mean(sample,dim=2) # -> 36864 8 + abs = torch.abs(sample).detach() max, max_index = torch.max(abs, dim=1) mean = torch.mean(abs, dim=1) - mean = torch.expand_copy(mean.reshape(-1, 1), sample.shape) - max = torch.expand_copy(max.reshape(-1, 1), sample.shape) + mean = torch.expand_copy(mean.reshape(-1, 1), abs.shape) + max = torch.expand_copy(max.reshape(-1, 1), abs.shape) + ratio = torch.pow(abs / mean, 2) - all = range(0, sample.shape[0]) - # ratio_max = abs / mean - # ratio_nor = (max - abs) / max - # ratio_nor[all, max_index] = ratio_max[all, max_index].clone() - ratio_nor = torch.pow(abs / mean, 4) - ratio_nor = torch.where(torch.isnan(ratio_nor), 1.0, ratio_nor) - label = sample * ratio_nor + ratio = torch.where(torch.isnan(ratio), 0.0, ratio) + label = ratio * abs + label_mean = torch.expand_copy(torch.mean(label, dim=1).reshape(-1, 1), abs.shape) + label = label - label_mean + mean + sample = torch.abs(sample) - loss = F.l1_loss(sample, label) + sample_nz = sample[abs > 0] + label_nz = label[abs > 0] + loss = F.l1_loss(sample_nz, label_nz) model.conv1.weight.grad = None loss.backward() # if epoch >= (epochs - 1): # continue - model.conv1.weight.data = model.conv1.weight.data - model.conv1.weight.grad * 0.001 + model.conv1.weight.data = model.conv1.weight.data - model.conv1.weight.grad * 0.1 model.conv1.weight.data = model.normal_conv1_weight() if (i + 1) % 100 == 0: @@ -168,10 +181,7 @@ w = model.conv1.weight.data show.DumpTensorToImage(w.view(-1, w.shape[2], w.shape[3]), "conv1_weight_update.png", Value2Log=True) # model.conv1.weight.data = torch.rand(model.conv1.weight.data.shape, device=device) - -# loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False) -# images, labels = next(iter(loader)) -# images = images.to(device) +# model.conv2.weight.data = torch.ones(model.conv2.weight.data.shape, device=device) # Train the model model.conv1.weight.requires_grad = False