01-svhn-pytorch

.gitignore

@@ -0,0 +1,4 @@
+*.swp
+*.pyc
+*.sh
+02-feature-inversion-pytorch/experiment

01-svhn-pytorch/dataset.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# =======================================
+# File Name : dataset.py
+# Purpose : generate samples for train and test
+# Creation Date : 2019-03-24 10:35
+# Last Modified :
+# Created By : niuyazhe
+# =======================================
+
+import os
+import torch
+import numpy as np
+from PIL import Image
+from scipy import io as scio
+from torch.utils.data import Dataset
+from torchvision import transforms
+
+
+class SvhnDataset(Dataset):
+    def __init__(self, root, train, transform=None, use_extra_data=True):
+        self.root = root
+        if train:
+            self.data = [os.path.join(root, 'train_32x32.mat')]
+            if use_extra_data:
+                self.data.append(os.path.join(root, 'extra_32x32.mat'))
+        else:
+            self.data = [os.path.join(root, 'test_32x32.mat')]
+
+        self.datas_list, self.labels_list = [], []
+        for f in self.data:
+            samples = scio.loadmat(f)
+            self.datas_list.append(samples['X'])
+            self.labels_list.append(samples['y'])
+        self.datas_np = np.concatenate(self.datas_list, axis=3)
+        self.datas_np = self.datas_np.transpose(3, 0, 1, 2)
+        self.labels_np = np.concatenate(self.labels_list, axis=0)
+
+        if transform is None:
+            raise ValueError
+        else:
+            self.transform = transform
+
+    def __getitem__(self, index):
+        img = self.datas_np[index]
+        label = self.labels_np[index]
+        label[0] = label[0] % 10
+        img = self.transform(Image.fromarray(np.uint8(img)))
+        return img.float(), torch.from_numpy(label).long()
+
+    def __len__(self):
+        return self.datas_np.shape[0]
+
+
+if __name__ == "__main__":
+    root = '~/data/'
+    transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
+    ])
+    dataset = SvhnDataset(root, train=True, transform=transform)
+    img, label = dataset[0]
+    print(img.shape)
+    print(img.mean())

01-svhn-pytorch/loss.py

@@ -0,0 +1,65 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class MaxLoss(nn.Module):
+    max_type_list = ['softmax', 'abs-max', 'square-max', 'plus-one-abs-max', 'non-negative-max']
+
+    def __init__(self, max_type):
+        super(MaxLoss, self).__init__()
+        assert max_type in self. max_type_list
+        self.max_type = max_type
+        self.NLLLoss = nn.NLLLoss()
+
+    def _onehot(self, x, dim_num):
+        one_hot_code = torch.zeros(x.size()[0], dim_num)
+        one_hot_code.scatter_(1, x, 1)
+        if torch.cuda.is_available():
+            one_hot_code = one_hot_code.cuda()
+        return one_hot_code.long()
+
+    def forward(self, inputs, target):
+        if self.max_type == 'softmax':
+            pred = F.softmax(inputs, dim=1)
+            pred = torch.log(pred)
+        elif self.max_type == 'abs-max':
+            raise NotImplementedError
+        elif self.max_type == 'square-max':
+            raise NotImplementedError
+        elif self.max_type == 'plus-one-abs-max':
+            raise NotImplementedError
+        elif self.max_type == 'non-negative-max':
+            raise NotImplementedError
+
+        return self.NLLLoss(pred, target)
+
+
+class MSELoss(nn.Module):
+    def __init__(self):
+        super(MSELoss, self).__init__()
+        self.mse_loss = nn.MSELoss()
+
+    def _onehot(self, x, dim_num):
+        one_hot_code = torch.zeros(x.size()[0], dim_num)
+        one_hot_code.scatter_(1, x, 1)
+        if torch.cuda.is_available():
+            one_hot_code = one_hot_code.cuda()
+        return one_hot_code.float()
+
+    def forward(self, inputs, target):
+        raise NotImplementedError
+
+
+class LpNorm(nn.Module):
+    def __init__(self, p=2, factor=1e-5):
+        super(LpNorm, self).__init__()
+        self.p = p
+        self.factor = factor
+
+    def forward(self, net):
+        total_norm = []
+        for item in net.parameters():
+            total_norm.append(torch.norm(item.data, p=self.p))
+        reg = torch.sum(torch.stack(total_norm, dim=0), dim=0)
+        return self.factor * reg

01-svhn-pytorch/model.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# =======================================
+# File Name : model.py
+# Purpose : build a compute graph
+# Creation Date : 2019-03-23 14:02
+# Last Modified :
+# Created By : niuyazhe
+# =======================================
+
+
+import math
+import numbers
+import torch
+import torch.nn as nn
+
+
+class LpPool2d(nn.Module):
+    def __init__(self, p, kernel_size, stride, padding=0):
+        super(LpPool2d, self).__init__()
+        self.p = p
+        self.kernel_size = kernel_size
+        self.padding = padding
+        self.stride = stride
+        raise NotImplementedError
+
+    def forward(self, x):
+        raise NotImplementedError
+
+
+class Model(nn.Module):
+    def __init__(self, in_channels=3, num_classes=10, pool='normal'):
+        super(Model, self).__init__()
+        self.in_channels = in_channels
+        self.num_classes = num_classes
+        self.pool = pool
+        if self.pool != 'normal' and not isinstance(self.pool, numbers.Integral):
+            raise ValueError
+        self.build()
+        self.init()
+
+    def _conv_layer(self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True, activation=nn.ReLU(), use_bn=True):
+        layers = []
+        layers.append(nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias))
+        if use_bn:
+            layers.append(nn.BatchNorm2d(out_channels))
+        if activation is not None:
+            layers.append(activation)
+        return nn.Sequential(*layers)
+
+    def _pool_layer(self, kernel_size, stride, padding=0, mode='MAX'):
+        if isinstance(self.pool, numbers.Integral):
+            return LpPool2d(self.pool, kernel_size, stride, padding)
+
+        mode_list = ['MAX', 'AVG']
+        assert(mode in mode_list or isinstance(mode, numbers.Integral))
+        if mode == 'MAX':
+            return nn.MaxPool2d(kernel_size, stride, padding)
+        elif mode == 'AVG':
+            return nn.AvgPool2d(kernel_size, stride, padding)
+
+    def _fc_layer(self, in_channels, out_channels, dropout=0.5):
+        layers = []
+        layers.append(nn.Linear(in_channels, out_channels))
+        if dropout != 0:
+            layers.append(nn.Dropout2d(p=dropout))
+        return nn.Sequential(*layers)
+
+    def init(self, scale_factor=1.0, mode='FAN_IN'):
+        mode_list = ['FAN_IN']
+        assert(mode in mode_list)
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                if mode == 'FAN_IN':
+                    n = m.kernel_size[0] * m.kernel_size[1] * m.in_channels
+                    m.weight.data.normal_(0, math.sqrt(scale_factor / n))
+                elif mode == 'FAN_OUT':
+                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                    m.weight.data.normal_(0, math.sqrt(scale_factor / n))
+                m.bias.data.zero_()
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+
+    def build(self):
+        self.conv1 = self._conv_layer(self.in_channels, 16, 3, 1, 1)
+        self.mp1 = self._pool_layer(kernel_size=2, stride=2, mode='MAX')
+
+        self.conv21 = self._conv_layer(16, 32, 3, 1, 1)
+        self.conv22 = self._conv_layer(32, 32, 3, 1, 1)
+        self.mp2 = self._pool_layer(kernel_size=2, stride=2, mode='MAX')
+
+        self.conv31 = self._conv_layer(32, 64, 3, 1, 1)
+        self.conv32 = self._conv_layer(64, 64, 3, 1, 1)
+        self.mp3 = self._pool_layer(kernel_size=2, stride=2, mode='MAX')
+
+        self.conv41 = self._conv_layer(64, 128, 3, 1, 1)
+        self.conv42 = self._conv_layer(128, 128, 3, 1, 1)
+        self.ap4 = self._pool_layer(kernel_size=4, stride=4, mode='AVG')
+
+        self.fc1 = self._fc_layer(128, self.num_classes, dropout=0)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.mp1(x)
+        x = self.conv21(x)
+        x = self.conv22(x)
+        x = self.mp2(x)
+        x = self.conv31(x)
+        x = self.conv32(x)
+        x = self.mp3(x)
+        x = self.conv41(x)
+        x = self.conv42(x)
+        x = self.ap4(x)
+
+        x = self.fc1(x.view(-1, 128))
+
+        return x
+
+
+def test_model():
+    net = Model()
+    inputs = torch.randn(4, 3, 32, 32)
+    output = net(inputs)
+    print(output.shape)
+
+
+def test_LpPool2d():
+    pool = LpPool2d(p=2, kernel_size=2, stride=2, padding=0)
+    pool.cuda()
+    inputs = torch.randn(4, 3, 32, 32).cuda()
+    output = pool(inputs)
+    print(output.shape)
+
+
+if __name__ == "__main__":
+    test_model()
+    test_LpPool2d()