From df05002c90572631011610e73738c6ce81b6772d Mon Sep 17 00:00:00 2001
From: Colin <colin>
Date: Sun, 3 Nov 2024 15:15:33 +0800
Subject: [PATCH] Update unsuper minist.

---
 unsuper/Readme.md | 14 +++++++++++---
 unsuper/minist.py | 44 +++++++++++++++++++++++++++-----------------
 2 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/unsuper/Readme.md b/unsuper/Readme.md
index d614bde..cb7a678 100644
--- a/unsuper/Readme.md
+++ b/unsuper/Readme.md
@@ -21,12 +21,20 @@ array([[4.3121886e+00, 3.2778070e-07, 8.9907879e-01, 2.1270120e+00,
    1. 重复的权重，虽然权重看起来都一样，但是有稍微的不同，不是完全一样
    2. 3x3太小了导致了样本的信噪比太低，大部分的样本切出来都是0
 2. 5x5的时候会有网格状重复
+   1. 通过比较小的loss不断得训练（epoch很大），最终还是会变成所有kernel都一样的“网格布”
+      1. 每个kernel都是数值很低的，很接近的结果
 3. 7x7的时候边框区域问题
 
-## 发现的原因
+## 问题的调试
 1. 几个卷积核都是重复的，网格状的
-   1. grad太大，learning rate 太大
-   2. grad太小，训练的epoch不合适
+   1. grad太大，learning rate 太大，grad太小，训练的epoch不合适
+   2. 网格状，就是为了尽量降低最终输出的绝对值，降低loss
+      1. 形成4个一组，一共2组，的形式，交替成为最大值，把另外一组的输出降低，最后都输出最低的绝对值
+      2. 需要类似batchnormal的方式对各个conv核心之间进行归一化
+   3. 采用label不改变原来sample output的 abs均值的限制
+      1. 生成2个分别交替充当最大值的极端（只有2个像素不是0）的卷积核，其他的卷积核都是输出接近0的网格
+   4. 多个卷积核之间需要有差异，同一个卷积核的不同样本输入也要有差异，卷积核的分布要有要求
+   5. 每个卷积核尽量平摊权重到所有像素，而不是集中一个像素？提高鲁棒性？
 
 ## 可能的策略
 1. 每个卷积核的改变权重(grad)能量守恒
diff --git a/unsuper/minist.py b/unsuper/minist.py
index 78eb13b..858c29c 100644
--- a/unsuper/minist.py
+++ b/unsuper/minist.py
@@ -117,7 +117,7 @@ for epoch in range(epochs):
         images = images.to(device)
 
         # images = torch.ones((1, 1, 5, 5), device=device)
-        # type = random.randint(0, 10)
+        # type = random.randint(0, 7)
         # if type == 0:
         #     rand = random.randint(0, 4)
         #     images[:, :, rand, :] = images[:, :, rand, :] * 0.5
@@ -130,32 +130,45 @@ for epoch in range(epochs):
         #     images[:, :, 2, 2] = images[:, :, 2, 2] * 0.5
         #     images[:, :, 3, 3] = images[:, :, 3, 3] * 0.5
         #     images[:, :, 4, 4] = images[:, :, 4, 4] * 0.5
+        # if type == 3:
+        #     randx = random.randint(0, 2)
+        #     randy = random.randint(0, 2)
+        #     images[:, :, randx, randy] = images[:, :, randx, randy] * 0.5
+        #     images[:, :, randx, randy + 1] = images[:, :, randx, randy + 1] * 0.5
+        #     images[:, :, randx, randy - 1] = images[:, :, randx, randy - 1] * 0.5
+        #     images[:, :, randx + 1, randy] = images[:, :, randx + 1, randy] * 0.5
+        #     images[:, :, randx - 1, randy] = images[:, :, randx - 1, randy] * 0.5
 
         outputs = model.forward_unsuper(images)
-
         outputs = outputs.permute(0, 2, 3, 1)  # 64 8 24 24 -> 64 24 24 8
         sample = outputs.reshape(-1, outputs.shape[3])  # -> 36864 8
+
+
+        # sample = outputs.reshape(-1, 8,24*24)  # -> 36864 8
+        # sample = torch.mean(sample,dim=2)  # -> 36864 8
+
         abs = torch.abs(sample).detach()
         max, max_index = torch.max(abs, dim=1)
         mean = torch.mean(abs, dim=1)
-        mean = torch.expand_copy(mean.reshape(-1, 1), sample.shape)
-        max = torch.expand_copy(max.reshape(-1, 1), sample.shape)
+        mean = torch.expand_copy(mean.reshape(-1, 1), abs.shape)
+        max = torch.expand_copy(max.reshape(-1, 1), abs.shape)
+        ratio = torch.pow(abs / mean, 2)
 
-        all = range(0, sample.shape[0])
-        # ratio_max = abs / mean
-        # ratio_nor = (max - abs) / max
-        # ratio_nor[all, max_index] = ratio_max[all, max_index].clone()
-        ratio_nor = torch.pow(abs / mean, 4)
-        ratio_nor = torch.where(torch.isnan(ratio_nor), 1.0, ratio_nor)
-        label = sample * ratio_nor
+        ratio = torch.where(torch.isnan(ratio), 0.0, ratio)
+        label = ratio * abs
+        label_mean = torch.expand_copy(torch.mean(label, dim=1).reshape(-1, 1), abs.shape)
+        label = label - label_mean + mean
+        sample = torch.abs(sample)
 
-        loss = F.l1_loss(sample, label)
+        sample_nz = sample[abs > 0]
+        label_nz = label[abs > 0]
+        loss = F.l1_loss(sample_nz, label_nz)
         model.conv1.weight.grad = None
         loss.backward()
 
         # if epoch >= (epochs - 1):
         #     continue
-        model.conv1.weight.data = model.conv1.weight.data - model.conv1.weight.grad * 0.001
+        model.conv1.weight.data = model.conv1.weight.data - model.conv1.weight.grad * 0.1
         model.conv1.weight.data = model.normal_conv1_weight()
 
         if (i + 1) % 100 == 0:
@@ -168,10 +181,7 @@ w = model.conv1.weight.data
 show.DumpTensorToImage(w.view(-1, w.shape[2], w.shape[3]), "conv1_weight_update.png", Value2Log=True)
 
 # model.conv1.weight.data = torch.rand(model.conv1.weight.data.shape, device=device)
-
-# loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False)
-# images, labels = next(iter(loader))
-# images = images.to(device)
+# model.conv2.weight.data = torch.ones(model.conv2.weight.data.shape, device=device)
 
 # Train the model
 model.conv1.weight.requires_grad = False