= 1.1 for using 'torch.utils.tensorboard' or turn off the option in " \
+ "the 'config.json' file."
+ logger.warning(message)
+
+ self.step = 0
+ self.mode = ''
+
+ self.tb_writer_ftns = {
+ 'add_scalar', 'add_scalars', 'add_image', 'add_images', 'add_audio',
+ 'add_text', 'add_histogram', 'add_pr_curve', 'add_embedding'
+ }
+ self.tag_mode_exceptions = {'add_histogram', 'add_embedding'}
+
+ self.timer = Timer()
+
+ def set_step(self, step, mode='train'):
+ self.mode = mode
+ self.step = step
+ if step == 0:
+ self.timer.reset()
+ else:
+ duration = self.timer.check()
+ self.add_scalar('steps_per_sec', 1 / duration)
+
+ def __getattr__(self, name):
+ """
+ If visualization is configured to use:
+ return add_data() methods of tensorboard with additional information (step, tag) added.
+ Otherwise:
+ return a blank function handle that does nothing
+ """
+ if name in self.tb_writer_ftns:
+ add_data = getattr(self.writer, name, None)
+
+ def wrapper(tag, data, *args, **kwargs):
+ if add_data is not None:
+ # add mode(train/valid) tag
+ if name not in self.tag_mode_exceptions:
+ tag = '{}/{}'.format(tag, self.mode)
+ add_data(tag, data, self.step, *args, **kwargs)
+ return wrapper
+ else:
+ # default action for returning methods defined in this class, set_step() for instance.
+ try:
+ attr = object.__getattr__(name)
+ except AttributeError:
+ raise AttributeError("type object '{}' has no attribute '{}'".format(self.selected_module, name))
+ return attr
diff --git a/ELR_plus/model/InceptionResNetV2.py b/ELR_plus/model/InceptionResNetV2.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0dbcb93ef6132fafcae4e7fd0dcece0d24d8f89
--- /dev/null
+++ b/ELR_plus/model/InceptionResNetV2.py
@@ -0,0 +1,314 @@
+from __future__ import print_function, division, absolute_import
+import torch
+import torch.nn as nn
+import os
+import sys
+
+
+class BasicConv2d(nn.Module):
+
+ def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):
+ super(BasicConv2d, self).__init__()
+ self.conv = nn.Conv2d(in_planes, out_planes,
+ kernel_size=kernel_size, stride=stride,
+ padding=padding, bias=False) # verify bias false
+ self.bn = nn.BatchNorm2d(out_planes,
+ eps=0.001, # value found in tensorflow
+ momentum=0.1, # default pytorch value
+ affine=True)
+ self.relu = nn.ReLU(inplace=False)
+
+ def forward(self, x):
+ x = self.conv(x)
+ x = self.bn(x)
+ x = self.relu(x)
+ return x
+
+
+class Mixed_5b(nn.Module):
+
+ def __init__(self):
+ super(Mixed_5b, self).__init__()
+
+ self.branch0 = BasicConv2d(192, 96, kernel_size=1, stride=1)
+
+ self.branch1 = nn.Sequential(
+ BasicConv2d(192, 48, kernel_size=1, stride=1),
+ BasicConv2d(48, 64, kernel_size=5, stride=1, padding=2)
+ )
+
+ self.branch2 = nn.Sequential(
+ BasicConv2d(192, 64, kernel_size=1, stride=1),
+ BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),
+ BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)
+ )
+
+ self.branch3 = nn.Sequential(
+ nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),
+ BasicConv2d(192, 64, kernel_size=1, stride=1)
+ )
+
+ def forward(self, x):
+ x0 = self.branch0(x)
+ x1 = self.branch1(x)
+ x2 = self.branch2(x)
+ x3 = self.branch3(x)
+ out = torch.cat((x0, x1, x2, x3), 1)
+ return out
+
+
+class Block35(nn.Module):
+
+ def __init__(self, scale=1.0):
+ super(Block35, self).__init__()
+
+ self.scale = scale
+
+ self.branch0 = BasicConv2d(320, 32, kernel_size=1, stride=1)
+
+ self.branch1 = nn.Sequential(
+ BasicConv2d(320, 32, kernel_size=1, stride=1),
+ BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1)
+ )
+
+ self.branch2 = nn.Sequential(
+ BasicConv2d(320, 32, kernel_size=1, stride=1),
+ BasicConv2d(32, 48, kernel_size=3, stride=1, padding=1),
+ BasicConv2d(48, 64, kernel_size=3, stride=1, padding=1)
+ )
+
+ self.conv2d = nn.Conv2d(128, 320, kernel_size=1, stride=1)
+ self.relu = nn.ReLU(inplace=False)
+
+ def forward(self, x):
+ x0 = self.branch0(x)
+ x1 = self.branch1(x)
+ x2 = self.branch2(x)
+ out = torch.cat((x0, x1, x2), 1)
+ out = self.conv2d(out)
+ out = out * self.scale + x
+ out = self.relu(out)
+ return out
+
+
+class Mixed_6a(nn.Module):
+
+ def __init__(self):
+ super(Mixed_6a, self).__init__()
+
+ self.branch0 = BasicConv2d(320, 384, kernel_size=3, stride=2)
+
+ self.branch1 = nn.Sequential(
+ BasicConv2d(320, 256, kernel_size=1, stride=1),
+ BasicConv2d(256, 256, kernel_size=3, stride=1, padding=1),
+ BasicConv2d(256, 384, kernel_size=3, stride=2)
+ )
+
+ self.branch2 = nn.MaxPool2d(3, stride=2)
+
+ def forward(self, x):
+ x0 = self.branch0(x)
+ x1 = self.branch1(x)
+ x2 = self.branch2(x)
+ out = torch.cat((x0, x1, x2), 1)
+ return out
+
+
+class Block17(nn.Module):
+
+ def __init__(self, scale=1.0):
+ super(Block17, self).__init__()
+
+ self.scale = scale
+
+ self.branch0 = BasicConv2d(1088, 192, kernel_size=1, stride=1)
+
+ self.branch1 = nn.Sequential(
+ BasicConv2d(1088, 128, kernel_size=1, stride=1),
+ BasicConv2d(128, 160, kernel_size=(1,7), stride=1, padding=(0,3)),
+ BasicConv2d(160, 192, kernel_size=(7,1), stride=1, padding=(3,0))
+ )
+
+ self.conv2d = nn.Conv2d(384, 1088, kernel_size=1, stride=1)
+ self.relu = nn.ReLU(inplace=False)
+
+ def forward(self, x):
+ x0 = self.branch0(x)
+ x1 = self.branch1(x)
+ out = torch.cat((x0, x1), 1)
+ out = self.conv2d(out)
+ out = out * self.scale + x
+ out = self.relu(out)
+ return out
+
+
+class Mixed_7a(nn.Module):
+
+ def __init__(self):
+ super(Mixed_7a, self).__init__()
+
+ self.branch0 = nn.Sequential(
+ BasicConv2d(1088, 256, kernel_size=1, stride=1),
+ BasicConv2d(256, 384, kernel_size=3, stride=2)
+ )
+
+ self.branch1 = nn.Sequential(
+ BasicConv2d(1088, 256, kernel_size=1, stride=1),
+ BasicConv2d(256, 288, kernel_size=3, stride=2)
+ )
+
+ self.branch2 = nn.Sequential(
+ BasicConv2d(1088, 256, kernel_size=1, stride=1),
+ BasicConv2d(256, 288, kernel_size=3, stride=1, padding=1),
+ BasicConv2d(288, 320, kernel_size=3, stride=2)
+ )
+
+ self.branch3 = nn.MaxPool2d(3, stride=2)
+
+ def forward(self, x):
+ x0 = self.branch0(x)
+ x1 = self.branch1(x)
+ x2 = self.branch2(x)
+ x3 = self.branch3(x)
+ out = torch.cat((x0, x1, x2, x3), 1)
+ return out
+
+
+class Block8(nn.Module):
+
+ def __init__(self, scale=1.0, noReLU=False):
+ super(Block8, self).__init__()
+
+ self.scale = scale
+ self.noReLU = noReLU
+
+ self.branch0 = BasicConv2d(2080, 192, kernel_size=1, stride=1)
+
+ self.branch1 = nn.Sequential(
+ BasicConv2d(2080, 192, kernel_size=1, stride=1),
+ BasicConv2d(192, 224, kernel_size=(1,3), stride=1, padding=(0,1)),
+ BasicConv2d(224, 256, kernel_size=(3,1), stride=1, padding=(1,0))
+ )
+
+ self.conv2d = nn.Conv2d(448, 2080, kernel_size=1, stride=1)
+ if not self.noReLU:
+ self.relu = nn.ReLU(inplace=False)
+
+ def forward(self, x):
+ x0 = self.branch0(x)
+ x1 = self.branch1(x)
+ out = torch.cat((x0, x1), 1)
+ out = self.conv2d(out)
+ out = out * self.scale + x
+ if not self.noReLU:
+ out = self.relu(out)
+ return out
+
+
+class InceptionResNetV2(nn.Module):
+
+ def __init__(self, num_classes=50):
+ super(InceptionResNetV2, self).__init__()
+ # Special attributs
+ self.input_space = None
+ self.input_size = (299, 299, 3)
+ self.mean = None
+ self.std = None
+ # Modules
+ self.conv2d_1a = BasicConv2d(3, 32, kernel_size=3, stride=2)
+ self.conv2d_2a = BasicConv2d(32, 32, kernel_size=3, stride=1)
+ self.conv2d_2b = BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1)
+ self.maxpool_3a = nn.MaxPool2d(3, stride=2)
+ self.conv2d_3b = BasicConv2d(64, 80, kernel_size=1, stride=1)
+ self.conv2d_4a = BasicConv2d(80, 192, kernel_size=3, stride=1)
+ self.maxpool_5a = nn.MaxPool2d(3, stride=2)
+ self.mixed_5b = Mixed_5b()
+ self.repeat = nn.Sequential(
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17),
+ Block35(scale=0.17)
+ )
+ self.mixed_6a = Mixed_6a()
+ self.repeat_1 = nn.Sequential(
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10),
+ Block17(scale=0.10)
+ )
+ self.mixed_7a = Mixed_7a()
+ self.repeat_2 = nn.Sequential(
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20),
+ Block8(scale=0.20)
+ )
+ self.block8 = Block8(noReLU=True)
+ self.conv2d_7b = BasicConv2d(2080, 1536, kernel_size=1, stride=1)
+ self.avgpool_1a = nn.AdaptiveAvgPool2d((1, 1))#nn.AvgPool2d(8, count_include_pad=False)
+ self.last_linear = nn.Linear(1536, num_classes)
+
+
+ def features(self, input):
+ x = self.conv2d_1a(input)
+ x = self.conv2d_2a(x)
+ x = self.conv2d_2b(x)
+ x = self.maxpool_3a(x)
+ x = self.conv2d_3b(x)
+ x = self.conv2d_4a(x)
+ x = self.maxpool_5a(x)
+ x = self.mixed_5b(x)
+ x = self.repeat(x)
+ x = self.mixed_6a(x)
+ x = self.repeat_1(x)
+ x = self.mixed_7a(x)
+ x = self.repeat_2(x)
+ x = self.block8(x)
+ x = self.conv2d_7b(x)
+ return x
+
+ def logits(self, features):
+ x = self.avgpool_1a(features)
+ x = x.view(x.size(0), -1)
+ out = self.last_linear(x)
+ return out
+
+
+ def forward(self, input):
+ x = self.features(input)
+ out = self.logits(x)
+ return out
+
+
+def test():
+ net = InceptionResNetV2().cuda()
+ y = net(torch.randn(1,3,227,227).cuda())
+ print(y.size())
+#test()
\ No newline at end of file
diff --git a/ELR_plus/model/PreResNet.py b/ELR_plus/model/PreResNet.py
new file mode 100644
index 0000000000000000000000000000000000000000..117c4b428bd510b33b619d7a9cc5ec42e96db3d6
--- /dev/null
+++ b/ELR_plus/model/PreResNet.py
@@ -0,0 +1,181 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.autograd import Variable
+
+
+def conv3x3(in_planes, out_planes, stride=1):
+ return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)
+
+
+class BasicBlock(nn.Module):
+ expansion = 1
+
+ def __init__(self, in_planes, planes, stride=1):
+ super(BasicBlock, self).__init__()
+ self.conv1 = conv3x3(in_planes, planes, stride)
+ self.bn1 = nn.BatchNorm2d(planes)
+ self.conv2 = conv3x3(planes, planes)
+ self.bn2 = nn.BatchNorm2d(planes)
+
+ self.shortcut = nn.Sequential()
+ if stride != 1 or in_planes != self.expansion*planes:
+ self.shortcut = nn.Sequential(
+ nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
+ nn.BatchNorm2d(self.expansion*planes)
+ )
+
+ def forward(self, x):
+ out = F.relu(self.bn1(self.conv1(x)))
+ out = self.bn2(self.conv2(out))
+ out += self.shortcut(x)
+ out = F.relu(out)
+ return out
+
+
+class PreActBlock(nn.Module):
+ '''Pre-activation version of the BasicBlock.'''
+ expansion = 1
+
+ def __init__(self, in_planes, planes, stride=1):
+ super(PreActBlock, self).__init__()
+ self.bn1 = nn.BatchNorm2d(in_planes)
+ self.conv1 = conv3x3(in_planes, planes, stride)
+ self.bn2 = nn.BatchNorm2d(planes)
+ self.conv2 = conv3x3(planes, planes)
+
+ self.shortcut = nn.Sequential()
+ if stride != 1 or in_planes != self.expansion*planes:
+ self.shortcut = nn.Sequential(
+ nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False)
+ )
+
+ def forward(self, x):
+ out = F.relu(self.bn1(x))
+ shortcut = self.shortcut(out)
+ out = self.conv1(out)
+ out = self.conv2(F.relu(self.bn2(out)))
+ out += shortcut
+ return out
+
+
+class Bottleneck(nn.Module):
+ expansion = 4
+
+ def __init__(self, in_planes, planes, stride=1):
+ super(Bottleneck, self).__init__()
+ self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
+ self.bn1 = nn.BatchNorm2d(planes)
+ self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+ self.bn2 = nn.BatchNorm2d(planes)
+ self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)
+ self.bn3 = nn.BatchNorm2d(self.expansion*planes)
+
+ self.shortcut = nn.Sequential()
+ if stride != 1 or in_planes != self.expansion*planes:
+ self.shortcut = nn.Sequential(
+ nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
+ nn.BatchNorm2d(self.expansion*planes)
+ )
+
+ def forward(self, x):
+ out = F.relu(self.bn1(self.conv1(x)))
+ out = F.relu(self.bn2(self.conv2(out)))
+ out = self.bn3(self.conv3(out))
+ out += self.shortcut(x)
+ out = F.relu(out)
+ return out
+
+
+class PreActBottleneck(nn.Module):
+ '''Pre-activation version of the original Bottleneck module.'''
+ expansion = 4
+
+ def __init__(self, in_planes, planes, stride=1):
+ super(PreActBottleneck, self).__init__()
+ self.bn1 = nn.BatchNorm2d(in_planes)
+ self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
+ self.bn2 = nn.BatchNorm2d(planes)
+ self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+ self.bn3 = nn.BatchNorm2d(planes)
+ self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)
+
+ self.shortcut = nn.Sequential()
+ if stride != 1 or in_planes != self.expansion*planes:
+ self.shortcut = nn.Sequential(
+ nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False)
+ )
+
+ def forward(self, x):
+ out = F.relu(self.bn1(x))
+ shortcut = self.shortcut(out)
+ out = self.conv1(out)
+ out = self.conv2(F.relu(self.bn2(out)))
+ out = self.conv3(F.relu(self.bn3(out)))
+ out += shortcut
+ return out
+
+
+class PreActResNet(nn.Module):
+ def __init__(self, block, num_blocks, num_classes=10):
+ super(PreActResNet, self).__init__()
+ self.in_planes = 64
+
+ self.conv1 = conv3x3(3,64)
+ self.bn1 = nn.BatchNorm2d(64)
+ self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
+ self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
+ self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
+ self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
+ self.linear = nn.Linear(512*block.expansion, num_classes)
+
+ def _make_layer(self, block, planes, num_blocks, stride):
+ strides = [stride] + [1]*(num_blocks-1)
+ layers = []
+ for stride in strides:
+ layers.append(block(self.in_planes, planes, stride))
+ self.in_planes = planes * block.expansion
+ return nn.Sequential(*layers)
+
+ def forward(self, x, lin=0, lout=5):
+ out = x
+ if lin < 1 and lout > -1:
+ out = self.conv1(out)
+ out = self.bn1(out)
+ out = F.relu(out)
+ if lin < 2 and lout > 0:
+ out = self.layer1(out)
+ if lin < 3 and lout > 1:
+ out = self.layer2(out)
+ if lin < 4 and lout > 2:
+ out = self.layer3(out)
+ if lin < 5 and lout > 3:
+ out = self.layer4(out)
+ if lout > 4:
+ out = F.avg_pool2d(out, 4)
+ out = out.view(out.size(0), -1)
+ out_final = self.linear(out)
+ return out_final
+
+
+def PreActResNet18(num_classes=10):
+ return PreActResNet(PreActBlock, [2,2,2,2], num_classes=num_classes)
+
+def PreActResNet34(num_classes=10):
+ return PreActResNet(BasicBlock, [3,4,6,3], num_classes=num_classes)
+
+def PreActResNet50(num_classes=10):
+ return PreActResNet(Bottleneck, [3,4,6,3], num_classes=num_classes)
+
+def PreActResNet101(num_classes=10):
+ return PreActResNet(Bottleneck, [3,4,23,3], num_classes=num_classes)
+
+def PreActResNet152(num_classes=10):
+ return PreActResNet(Bottleneck, [3,8,36,3], num_classes=num_classes)
+
+
+def test():
+ net = PreActResNet18()
+ y = net(Variable(torch.randn(1,3,32,32)))
+ print(y.size())
diff --git a/ELR_plus/model/ResNet_Zoo.py b/ELR_plus/model/ResNet_Zoo.py
new file mode 100644
index 0000000000000000000000000000000000000000..34cf692ea9d57423a21f11a9e0cdcb91d1f5f122
--- /dev/null
+++ b/ELR_plus/model/ResNet_Zoo.py
@@ -0,0 +1,121 @@
+'''ResNet in PyTorch.
+For Pre-activation ResNet, see 'preact_resnet.py'.
+Reference:
+[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+ Deep Residual Learning for Image Recognition. arXiv:1512.03385
+'''
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class BasicBlock(nn.Module):
+ expansion = 1
+
+ def __init__(self, in_planes, planes, stride=1):
+ super(BasicBlock, self).__init__()
+ self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+ self.bn1 = nn.BatchNorm2d(planes)
+ self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
+ self.bn2 = nn.BatchNorm2d(planes)
+
+ self.shortcut = nn.Sequential()
+ if stride != 1 or in_planes != self.expansion*planes:
+ self.shortcut = nn.Sequential(
+ nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
+ nn.BatchNorm2d(self.expansion*planes)
+ )
+
+ def forward(self, x):
+ out = F.relu(self.bn1(self.conv1(x)))
+ out = self.bn2(self.conv2(out))
+ out += self.shortcut(x)
+ out = F.relu(out)
+ return out
+
+
+
+class Bottleneck(nn.Module):
+ expansion = 4
+
+ def __init__(self, in_planes, planes, stride=1):
+ super(Bottleneck, self).__init__()
+ self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
+ self.bn1 = nn.BatchNorm2d(planes)
+ self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+ self.bn2 = nn.BatchNorm2d(planes)
+ self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)
+ self.bn3 = nn.BatchNorm2d(self.expansion*planes)
+
+ self.shortcut = nn.Sequential()
+ if stride != 1 or in_planes != self.expansion*planes:
+ self.shortcut = nn.Sequential(
+ nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
+ nn.BatchNorm2d(self.expansion*planes)
+ )
+
+ def forward(self, x):
+ out = F.relu(self.bn1(self.conv1(x)))
+ out = F.relu(self.bn2(self.conv2(out)))
+ out = self.bn3(self.conv3(out))
+ out += self.shortcut(x)
+ out = F.relu(out)
+ return out
+
+
+class ResNet(nn.Module):
+ def __init__(self, block, num_blocks, num_classes=10):
+ super(ResNet, self).__init__()
+ self.in_planes = 64
+
+ self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
+ self.bn1 = nn.BatchNorm2d(64)
+ self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
+ self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
+ self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
+ self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
+ self.linear = nn.Linear(512*block.expansion, num_classes)
+
+
+ def _make_layer(self, block, planes, num_blocks, stride):
+ strides = [stride] + [1]*(num_blocks-1)
+ layers = []
+ for stride in strides:
+ layers.append(block(self.in_planes, planes, stride))
+ self.in_planes = planes * block.expansion
+ return nn.Sequential(*layers)
+
+ def forward(self, x):
+ out = F.relu(self.bn1(self.conv1(x)))
+ out = self.layer1(out)
+ out = self.layer2(out)
+ out = self.layer3(out)
+ out = self.layer4(out)
+ out = F.avg_pool2d(out, 4)
+ y = out.view(out.size(0), -1)
+ out = self.linear(y)
+ return out
+
+
+def ResNet18():
+ return ResNet(BasicBlock, [2,2,2,2])
+
+def ResNet34():
+ return ResNet(BasicBlock, [3,4,6,3])
+
+def ResNet50():
+ return ResNet(Bottleneck, [3,4,6,3])
+
+def ResNet101():
+ return ResNet(Bottleneck, [3,4,23,3])
+
+def ResNet152():
+ return ResNet(Bottleneck, [3,8,36,3])
+
+
+def test():
+ net = ResNet18()
+ y = net(torch.randn(1,3,32,32))
+ print(y.size())
+
+# test()
\ No newline at end of file
diff --git a/ELR_plus/model/loss.py b/ELR_plus/model/loss.py
new file mode 100644
index 0000000000000000000000000000000000000000..07bd9b92d904f79ba574492bc9f210685456a42f
--- /dev/null
+++ b/ELR_plus/model/loss.py
@@ -0,0 +1,41 @@
+import torch.nn.functional as F
+import torch
+import numpy as np
+from parse_config import ConfigParser
+import torch.nn as nn
+from torch.autograd import Variable
+import math
+from utils import sigmoid_rampup, sigmoid_rampdown, cosine_rampup, cosine_rampdown, linear_rampup
+
+
+def cross_entropy(output, target, M=3):
+ return F.cross_entropy(output, target)
+
+class elr_plus_loss(nn.Module):
+ def __init__(self, num_examp, config, device, num_classes=10, beta=0.3):
+ super(elr_plus_loss, self).__init__()
+ self.config = config
+ self.pred_hist = (torch.zeros(num_examp, num_classes)).to(device)
+ self.q = 0
+ self.beta = beta
+ self.num_classes = num_classes
+
+ def forward(self, iteration, output, y_labeled):
+ y_pred = F.softmax(output,dim=1)
+
+ y_pred = torch.clamp(y_pred, 1e-4, 1.0-1e-4)
+
+ if self.num_classes == 100:
+ y_labeled = y_labeled*self.q
+ y_labeled = y_labeled/(y_labeled).sum(dim=1,keepdim=True)
+
+ ce_loss = torch.mean(-torch.sum(y_labeled * F.log_softmax(output, dim=1), dim = -1))
+ reg = ((1-(self.q * y_pred).sum(dim=1)).log()).mean()
+ final_loss = ce_loss + sigmoid_rampup(iteration, self.config['coef_step'])*(self.config['train_loss']['args']['lambda']*reg)
+
+ return final_loss, y_pred.cpu().detach()
+
+ def update_hist(self, epoch, out, index= None, mix_index = ..., mixup_l = 1):
+ y_pred_ = F.softmax(out,dim=1)
+ self.pred_hist[index] = self.beta * self.pred_hist[index] + (1-self.beta) * y_pred_/(y_pred_).sum(dim=1,keepdim=True)
+ self.q = mixup_l * self.pred_hist[index] + (1-mixup_l) * self.pred_hist[index][mix_index]
diff --git a/ELR_plus/model/metric.py b/ELR_plus/model/metric.py
new file mode 100644
index 0000000000000000000000000000000000000000..51ed18f5e9e33b67d297692e8a987f5237e35f7e
--- /dev/null
+++ b/ELR_plus/model/metric.py
@@ -0,0 +1,20 @@
+import torch
+
+
+def my_metric(output, target):
+ with torch.no_grad():
+ pred = torch.argmax(output, dim=1)
+ assert pred.shape[0] == len(target)
+ correct = 0
+ correct += torch.sum(pred == target).item()
+ return correct / len(target)
+
+
+def my_metric2(output, target, k=3):
+ with torch.no_grad():
+ pred = torch.topk(output, k, dim=1)[1]
+ assert pred.shape[0] == len(target)
+ correct = 0
+ for i in range(k):
+ correct += torch.sum(pred[:, i] == target).item()
+ return correct / len(target)
diff --git a/ELR_plus/model/model.py b/ELR_plus/model/model.py
new file mode 100644
index 0000000000000000000000000000000000000000..410d193d796fa603e45dfaf91d845732c80b8d82
--- /dev/null
+++ b/ELR_plus/model/model.py
@@ -0,0 +1,26 @@
+import torch.nn as nn
+import torch.nn.functional as F
+from base import BaseModel
+from .ResNet_Zoo import ResNet, BasicBlock
+from .PreResNet import PreActResNet, PreActBlock
+import torchvision.models as models
+from .InceptionResNetV2 import InceptionResNetV2
+
+
+def resnet34(num_classes=10):
+ return ResNet(BasicBlock, [3,4,6,3], num_classes=num_classes)
+ #return models.resnet34(num_classes=10)
+
+
+def resnet50(num_classes=14):
+ import torchvision.models as models
+ model_ft = models.resnet50(pretrained=True)
+ num_ftrs = model_ft.fc.in_features
+ model_ft.fc = nn.Linear(num_ftrs, num_classes)
+ return model_ft
+
+
+def PreActResNet34(num_classes=10) -> PreActResNet:
+ return PreActResNet(PreActBlock, [3, 4, 6, 3], num_classes=num_classes)
+def PreActResNet18(num_classes=10) -> PreActResNet:
+ return PreActResNet(PreActBlock, [2, 2, 2, 2], num_classes=num_classes)
diff --git a/ELR_plus/parse_config.py b/ELR_plus/parse_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..8434f5cd529118d5764052ad395d230d47fe793c
--- /dev/null
+++ b/ELR_plus/parse_config.py
@@ -0,0 +1,145 @@
+import os
+import logging
+from pathlib import Path
+from functools import reduce
+from operator import getitem
+from datetime import datetime
+from logger import setup_logging
+from utils import read_json, write_json
+
+
+class ConfigParser:
+
+ __instance = None
+
+ def __new__(cls, args, options='', timestamp=True):
+ raise NotImplementedError('Cannot initialize via Constructor')
+
+ @classmethod
+ def __internal_new__(cls):
+ return super().__new__(cls)
+
+ @classmethod
+ def get_instance(cls, args=None, options='', timestamp=True):
+ if not cls.__instance:
+ if args is None:
+ NotImplementedError('Cannot initialize without args')
+ cls.__instance = cls.__internal_new__()
+ cls.__instance.__init__(args, options)
+
+ return cls.__instance
+
+ def __init__(self, args, options='', timestamp=True):
+ # parse default and custom cli options
+ for opt in options:
+ args.add_argument(*opt.flags, default=None, type=opt.type)
+ args = args.parse_args()
+
+ if args.device:
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.device
+ if args.resume is None:
+ msg_no_cfg = "Configuration file need to be specified. Add '-c config.json', for example."
+ assert args.config is not None, msg_no_cfg
+ self.cfg_fname = Path(args.config)
+ config = read_json(self.cfg_fname)
+ self.resume = None
+ else:
+ self.resume = Path(args.resume)
+ resume_cfg_fname = self.resume.parent / 'config.json'
+ config = read_json(resume_cfg_fname)
+ if args.config is not None:
+ config.update(read_json(Path(args.config)))
+
+ # load config file and apply custom cli options
+ self._config = _update_config(config, options, args)
+
+ # set save_dir where trained model and log will be saved.
+ save_dir = Path(self.config['trainer']['save_dir'])
+ timestamp = datetime.now().strftime(r'%m%d_%H%M%S') if timestamp else ''
+
+ if self.config['trainer']['asym']:
+ exper_name = self.config['name'] + '_asym_' + str(int(self.config['trainer']['percent']*100))
+ else:
+ exper_name = self.config['name'] + '_sym_' + str(int(self.config['trainer']['percent']*100))
+
+ self._save_dir = save_dir / 'models' / exper_name / timestamp
+ self._log_dir = save_dir / 'log' / exper_name / timestamp
+
+ self.save_dir.mkdir(parents=True, exist_ok=True)
+ self.log_dir.mkdir(parents=True, exist_ok=True)
+
+ # save updated config file to the checkpoint dir
+ write_json(self.config, self.save_dir / 'config.json')
+
+ # configure logging module
+ setup_logging(self.log_dir)
+ self.log_levels = {
+ 0: logging.WARNING,
+ 1: logging.INFO,
+ 2: logging.DEBUG
+ }
+
+ def initialize(self, name, module, *args, **kwargs):
+ """
+ finds a function handle with the name given as 'type' in config, and returns the
+ instance initialized with corresponding keyword args given as 'args'.
+ """
+ module_name = self[name]['type']
+ module_args = dict(self[name]['args'])
+ assert all([k not in module_args for k in kwargs]), 'Overwriting kwargs given in config file is not allowed'
+ module_args.update(kwargs)
+ return getattr(module, module_name)(*args, **module_args)
+
+ def __getitem__(self, name):
+ return self.config[name]
+
+ def get_logger(self, name, verbosity=2):
+ msg_verbosity = 'verbosity option {} is invalid. Valid options are {}.'.format(verbosity,
+ self.log_levels.keys())
+ assert verbosity in self.log_levels, msg_verbosity
+ logger = logging.getLogger(name)
+ logger.setLevel(self.log_levels[verbosity])
+ return logger
+
+ # setting read-only attributes
+ @property
+ def config(self):
+ return self._config
+
+ @property
+ def save_dir(self):
+ return self._save_dir
+
+ @property
+ def log_dir(self):
+ return self._log_dir
+
+
+# helper functions used to update config dict with custom cli options
+def _update_config(config, options, args):
+ for opt in options:
+ value = getattr(args, _get_opt_name(opt.flags))
+ if value is not None:
+ _set_by_path(config, opt.target, value)
+ if 'target2' in opt._fields:
+ _set_by_path(config, opt.target2, value)
+ if 'target3' in opt._fields:
+ _set_by_path(config, opt.target3, value)
+ return config
+
+
+def _get_opt_name(flags):
+ for flg in flags:
+ if flg.startswith('--'):
+ return flg.replace('--', '')
+ return flags[0].replace('--', '')
+
+
+def _set_by_path(tree, keys, value):
+ """Set a value in a nested object in tree by sequence of keys."""
+ _get_by_path(tree, keys[:-1])[keys[-1]] = value
+
+
+def _get_by_path(tree, keys):
+ """Access a nested object in tree by sequence of keys."""
+ return reduce(getitem, keys, tree)
diff --git a/ELR_plus/test.py b/ELR_plus/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..3084bdb3a59d756db1c8914962108a44550293de
--- /dev/null
+++ b/ELR_plus/test.py
@@ -0,0 +1,82 @@
+import argparse
+import torch
+from tqdm import tqdm
+import data_loader.data_loaders as module_data
+import model.loss as module_loss
+import model.metric as module_metric
+import model.model as module_arch
+from parse_config import ConfigParser
+
+
+def main(config):
+ logger = config.get_logger('test')
+
+ # setup data_loader instances
+ data_loader = getattr(module_data, config['data_loader']['type'])(
+ config['data_loader']['args']['data_dir'],
+ batch_size=512,
+ shuffle=False,
+ validation_split=0.0,
+ training=False,
+ num_workers=2
+ ).split_validation()
+
+ # build model architecture
+ model = config.initialize('arch', module_arch)
+ logger.info(model)
+
+ # get function handles of loss and metrics
+ loss_fn = getattr(module_loss, config['val_loss'])
+ metric_fns = [getattr(module_metric, met) for met in config['metrics']]
+
+ logger.info('Loading checkpoint: {} ...'.format(config.resume))
+ checkpoint = torch.load(config.resume,map_location='cpu')
+ state_dict = checkpoint['state_dict']
+ if config['n_gpu'] > 1:
+ model = torch.nn.DataParallel(model)
+ model.load_state_dict(state_dict)
+
+ # prepare model for testing
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+ model = model.to(device)
+ model.eval()
+
+ total_loss = 0.0
+ total_metrics = torch.zeros(len(metric_fns))
+
+ with torch.no_grad():
+ for i, (data, target,_,_) in enumerate(tqdm(data_loader)):
+ data, target = data.to(device), target.to(device)
+ output = model(data)
+
+ #
+ # save sample images, or do something with output here
+ #
+
+ # computing loss, metrics on test set
+ loss = loss_fn(output, target)
+ batch_size = data.shape[0]
+ total_loss += loss.item() * batch_size
+ for i, metric in enumerate(metric_fns):
+ total_metrics[i] += metric(output, target) * batch_size
+
+ n_samples = len(data_loader.sampler)
+ log = {'loss': total_loss / n_samples}
+ log.update({
+ met.__name__: total_metrics[i].item() / n_samples for i, met in enumerate(metric_fns)
+ })
+ logger.info(log)
+
+
+if __name__ == '__main__':
+ args = argparse.ArgumentParser(description='PyTorch Template')
+
+ args.add_argument('-c', '--config', default=None, type=str,
+ help='config file path (default: None)')
+ args.add_argument('-r', '--resume', default=None, type=str,
+ help='path to latest checkpoint (default: None)')
+ args.add_argument('-d', '--device', default=None, type=str,
+ help='indices of GPUs to enable (default: all)')
+ config = ConfigParser.get_instance(args, '')
+ #config = ConfigParser(args)
+ main(config)
diff --git a/ELR_plus/train.py b/ELR_plus/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..855c7dea7809447452f81f65054a0b10216e6b4a
--- /dev/null
+++ b/ELR_plus/train.py
@@ -0,0 +1,163 @@
+import argparse
+import collections
+import sys
+import requests
+import socket
+import torch
+import mlflow
+import mlflow.pytorch
+import data_loader.data_loaders as module_data
+import model.loss as module_loss
+import model.metric as module_metric
+import model.model as module_arch
+from parse_config import ConfigParser
+from trainer import Trainer
+from collections import OrderedDict
+import random
+
+
+def log_params(conf: OrderedDict, parent_key: str = None):
+ for key, value in conf.items():
+ if parent_key is not None:
+ combined_key = f'{parent_key}-{key}'
+ else:
+ combined_key = key
+
+ if not isinstance(value, OrderedDict):
+ mlflow.log_param(combined_key, value)
+ else:
+ log_params(value, combined_key)
+
+
+def main(config: ConfigParser):
+
+
+ logger = config.get_logger('train')
+ logger.info(config.config)
+
+ # setup data_loader instances
+ data_loader1 = getattr(module_data, config['data_loader']['type'])(
+ config['data_loader']['args']['data_dir'],
+ batch_size= config['data_loader']['args']['batch_size'],
+ shuffle=config['data_loader']['args']['shuffle'],
+ validation_split=config['data_loader']['args']['validation_split'],
+ num_batches=config['data_loader']['args']['num_batches'],
+ training=True,
+ num_workers=config['data_loader']['args']['num_workers'],
+ pin_memory=config['data_loader']['args']['pin_memory']
+ )
+
+ data_loader2 = getattr(module_data, config['data_loader']['type'])(
+ config['data_loader']['args']['data_dir'],
+ batch_size= config['data_loader']['args']['batch_size2'],
+ shuffle=config['data_loader']['args']['shuffle'],
+ validation_split=config['data_loader']['args']['validation_split'],
+ num_batches=config['data_loader']['args']['num_batches'],
+ training=True,
+ num_workers=config['data_loader']['args']['num_workers'],
+ pin_memory=config['data_loader']['args']['pin_memory']
+ )
+
+
+ valid_data_loader = data_loader1.split_validation()
+
+ test_data_loader = getattr(module_data, config['data_loader']['type'])(
+ config['data_loader']['args']['data_dir'],
+ batch_size=128,
+ shuffle=False,
+ validation_split=0.0,
+ training=False,
+ num_workers=2
+ ).split_validation()
+
+ # build model architecture
+ model1 = config.initialize('arch1', module_arch)
+ model_ema1 = config.initialize('arch1', module_arch)
+ model_ema1_copy = config.initialize('arch1', module_arch)
+ model2 = config.initialize('arch2', module_arch)
+ model_ema2 = config.initialize('arch2', module_arch)
+ model_ema2_copy = config.initialize('arch2', module_arch)
+
+
+ # get function handles of loss and metrics
+ device_id = list(range(min(torch.cuda.device_count(), config['n_gpu'])))
+
+ if hasattr(data_loader1.dataset, 'num_raw_example') and hasattr(data_loader2.dataset, 'num_raw_example'):
+ num_examp1 = data_loader1.dataset.num_raw_example
+ num_examp2 = data_loader2.dataset.num_raw_example
+ else:
+ num_examp1 = len(data_loader1.dataset)
+ num_examp2 = len(data_loader2.dataset)
+
+ train_loss1 = getattr(module_loss, config['train_loss']['type'])(num_examp=num_examp1, num_classes=config['num_classes'],
+ device = 'cuda:'+ str(device_id[0]), config = config.config, beta=config['train_loss']['args']['beta'])
+ train_loss2 = getattr(module_loss, config['train_loss']['type'])(num_examp=num_examp2, num_classes=config['num_classes'],
+ device = 'cuda:'+str(device_id[-1]), config = config.config, beta=config['train_loss']['args']['beta'])
+
+ val_loss = getattr(module_loss, config['val_loss'])
+ metrics = [getattr(module_metric, met) for met in config['metrics']]
+
+ # build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler
+ trainable_params1 = filter(lambda p: p.requires_grad, model1.parameters())
+ trainable_params2 = filter(lambda p: p.requires_grad, model2.parameters())
+
+ optimizer1 = config.initialize('optimizer1', torch.optim, [{'params': trainable_params1}])
+ optimizer2 = config.initialize('optimizer2', torch.optim, [{'params': trainable_params2}])
+
+ lr_scheduler1 = config.initialize('lr_scheduler', torch.optim.lr_scheduler, optimizer1)
+ lr_scheduler2 = config.initialize('lr_scheduler', torch.optim.lr_scheduler, optimizer2)
+
+ trainer = Trainer(model1, model2, model_ema1, model_ema2, train_loss1, train_loss2,
+ metrics,
+ optimizer1, optimizer2,
+ config=config,
+ data_loader1=data_loader1,
+ data_loader2=data_loader2,
+ valid_data_loader=valid_data_loader,
+ test_data_loader=test_data_loader,
+ lr_scheduler1=lr_scheduler1,
+ lr_scheduler2=lr_scheduler2,
+ val_criterion=val_loss,
+ model_ema1_copy = model_ema1_copy,
+ model_ema2_copy = model_ema2_copy)
+
+ trainer.train()
+ logger = config.get_logger('trainer', config['trainer']['verbosity'])
+ cfg_trainer = config['trainer']
+
+
+
+if __name__ == '__main__':
+ args = argparse.ArgumentParser(description='PyTorch Template')
+ args.add_argument('-c', '--config', default=None, type=str,
+ help='config file path (default: None)')
+ args.add_argument('-r', '--resume', default=None, type=str,
+ help='path to latest checkpoint (default: None)')
+ args.add_argument('-d', '--device', default=None, type=str,
+ help='indices of GPUs to enable (default: all)')
+
+ # custom cli options to modify configuration from default values given in json file.
+ CustomArgs = collections.namedtuple('CustomArgs', 'flags type target')
+ options = [
+ CustomArgs(['--lr', '--learning_rate'], type=float, target=('optimizer', 'args', 'lr')),
+ CustomArgs(['--bs', '--batch_size'], type=int, target=('data_loader', 'args', 'batch_size')),
+ CustomArgs(['--beta', '--beta'], type=float, target=('train_loss', 'args', 'beta')),
+ CustomArgs(['--lambda', '--lambda'], type=float, target=('train_loss', 'args', 'lambda')),
+ CustomArgs(['--percent', '--percent'], type=float, target=('trainer', 'percent')),
+ CustomArgs(['--asym', '--asym'], type=bool, target=('trainer', 'asym')),
+ CustomArgs(['--name', '--exp_name'], type=str, target=('name',)),
+ CustomArgs(['--malpha', '--mixup_alpha'], type=float, target=('mixup_alpha',)),
+ CustomArgs(['--ealpha', '--ema_alpha'], type=float, target=('ema_alpha',)),
+ CustomArgs(['--nb', '--num_batches'], type=float, target=('data_loader', 'args', 'num_batches')),
+ CustomArgs(['--warm', '--warmup'], type=int, target=('trainer', 'warmup')),
+ CustomArgs(['--seed', '--seed'], type=int, target=('seed',)),
+ CustomArgs(['--wc1', '--weight_decay1'], type=float, target=('optimizer1','weight_decay')),
+ CustomArgs(['--wc2', '--weight_decay2'], type=float, target=('optimizer2','weight_decay')),
+ CustomArgs(['--estep', '--ema_step'], type=float, target=('ema_step',)),
+
+ ]
+ config = ConfigParser.get_instance(args, options)
+ random.seed(config['seed'])
+ torch.manual_seed(config['seed'])
+ torch.cuda.manual_seed_all(config['seed'])
+ main(config)
diff --git a/ELR_plus/trainer/__init__.py b/ELR_plus/trainer/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fe21f802159f55665fcdebcd9b2c32bb4acf5b9
--- /dev/null
+++ b/ELR_plus/trainer/__init__.py
@@ -0,0 +1 @@
+from .trainer import *
diff --git a/ELR_plus/trainer/trainer.py b/ELR_plus/trainer/trainer.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff2fa43ccdea4ffdb67d4443e2b506bee3b3f05e
--- /dev/null
+++ b/ELR_plus/trainer/trainer.py
@@ -0,0 +1,365 @@
+import numpy as np
+import torch
+from tqdm import tqdm
+from typing import List
+from torchvision.utils import make_grid
+from base import BaseTrainer
+from utils import inf_loop, linear_rampup, sigmoid_rampup, linear_rampdown
+import sys
+from sklearn.mixture import GaussianMixture
+import torch.nn.functional as F
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+class Trainer(BaseTrainer):
+ """
+ Trainer class
+
+ Note:
+ Inherited from BaseTrainer.
+ """
+ def __init__(self, model1, model2, model_ema1, model_ema2, train_criterion1, train_criterion2, metrics, optimizer1, optimizer2, config,
+ data_loader1, data_loader2,
+ valid_data_loader=None,
+ test_data_loader=None,
+ lr_scheduler1=None, lr_scheduler2=None,
+ len_epoch=None, val_criterion=None,
+ model_ema1_copy=None, model_ema2_copy=None):
+ super().__init__(model1, model2, model_ema1, model_ema2, train_criterion1, train_criterion2,
+ metrics, optimizer1, optimizer2, config, val_criterion, model_ema1_copy, model_ema2_copy)
+ self.config = config.config
+ self.data_loader1 = data_loader1
+ self.data_loader2 = data_loader2
+ if len_epoch is None:
+ # epoch-based training
+ self.len_epoch = len(self.data_loader1)
+ else:
+ # iteration-based training
+ self.data_loader1 = inf_loop(data_loader1)
+ self.data_loader2 = inf_loop(data_loader2)
+ self.len_epoch = len_epoch
+ self.valid_data_loader = valid_data_loader
+
+ self.test_data_loader = test_data_loader
+ self.do_validation = self.valid_data_loader is not None
+ self.do_test = self.test_data_loader is not None
+ self.lr_scheduler1 = lr_scheduler1
+ self.lr_scheduler2 = lr_scheduler2
+ self.log_step = int(np.sqrt(self.data_loader1.batch_size))
+ self.train_loss_list: List[float] = []
+ self.val_loss_list: List[float] = []
+ self.test_loss_list: List[float] = []
+
+
+ def _eval_metrics(self, output, target):
+ acc_metrics = np.zeros(len(self.metrics))
+ for i, metric in enumerate(self.metrics):
+ acc_metrics[i] += metric(output, target)
+ self.writer.add_scalar('{}'.format(metric.__name__), acc_metrics[i])
+ return acc_metrics
+
+ def _train_epoch(self, epoch, model, model_ema, model_ema2, data_loader, train_criterion, optimizer, lr_scheduler, device = 'cpu', queue = None):
+ """
+ Training logic for an epoch
+
+ :param epoch: Current training epoch.
+ :return: A log that contains all information you want to save.
+
+ Note:
+ If you have additional information to record, for example:
+ > additional_log = {"x": x, "y": y}
+ merge it with log before return. i.e.
+ > log = {**log, **additional_log}
+ > return log
+
+ The metrics in log must have the key 'metrics'.
+ """
+ model.train()
+ model_ema.train()
+
+ total_loss = 0
+ total_metrics = np.zeros(len(self.metrics))
+ total_metrics_ema = np.zeros(len(self.metrics))
+
+ if hasattr(data_loader.dataset, 'num_raw_example'):
+ num_examp = data_loader.dataset.num_raw_example
+ else:
+ num_examp = len(data_loader.dataset)
+
+ local_step = 0
+
+
+
+ with tqdm(data_loader) as progress:
+ for batch_idx, (data, target, indexs, _) in enumerate(progress):
+ progress.set_description_str(f'Train epoch {epoch}')
+
+ data_original = data
+ target_original = target
+
+ target = torch.zeros(len(target), self.config['num_classes']).scatter_(1, target.view(-1,1), 1)
+ data, target, target_original = data.to(device), target.float().to(device), target_original.to(device)
+
+ data, target, mixup_l, mix_index = self._mixup_data(data, target, alpha = self.config['mixup_alpha'], device = device)
+
+ output = model(data)
+
+ data_original = data_original.to(device)
+ output_original = model_ema2(data_original)
+ output_original = output_original.data.detach()
+ train_criterion.update_hist(epoch, output_original, indexs.numpy().tolist(), mix_index = mix_index, mixup_l = mixup_l)
+
+ local_step += 1
+ loss, probs = train_criterion(self.global_step + local_step, output, target)
+
+ optimizer.zero_grad()
+ loss.backward()
+
+
+ optimizer.step()
+
+ self.update_ema_variables(model, model_ema, self.global_step + local_step, self.config['ema_alpha'])
+
+ self.writer.set_step((epoch - 1) * self.len_epoch + batch_idx)
+ self.writer.add_scalar('loss', loss.item())
+ self.train_loss_list.append(loss.item())
+ total_loss += loss.item()
+ total_metrics += self._eval_metrics(output, target.argmax(dim=1))
+ if output_original is not None:
+ total_metrics_ema += self._eval_metrics(output_original, target.argmax(dim=1))
+
+
+ if batch_idx % self.log_step == 0:
+ progress.set_postfix_str(' {} Loss: {:.6f}'.format(
+ self._progress(batch_idx),
+ loss.item()))
+
+ if batch_idx == self.len_epoch:
+ break
+
+ if hasattr(data_loader, 'run'):
+ data_loader.run()
+
+
+ log = {
+ 'global step': self.global_step,
+ 'local_step': local_step,
+ 'loss': total_loss / self.len_epoch,
+ 'metrics': (total_metrics / self.len_epoch).tolist(),
+ 'metrics_ema': (total_metrics_ema / self.len_epoch).tolist(),
+ 'learning rate': lr_scheduler.get_lr()
+ }
+
+
+ if lr_scheduler is not None:
+ lr_scheduler.step()
+
+ if queue is None:
+ return log
+ else:
+ queue.put(log)
+
+
+ def _valid_epoch(self, epoch, model1, model2, device = 'cpu', queue = None):
+ """
+ Validate after training an epoch
+
+ :return: A log that contains information about validation
+
+ Note:
+ The validation metrics in log must have the key 'val_metrics'.
+ """
+ model1.eval()
+ model2.eval()
+
+ total_val_loss = 0
+ total_val_metrics = np.zeros(len(self.metrics))
+ with torch.no_grad():
+ with tqdm(self.valid_data_loader) as progress:
+ for batch_idx, (data, target, _, _) in enumerate(progress):
+ progress.set_description_str(f'Valid epoch {epoch}')
+ data, target = data.to(device), target.to(device)
+
+ output1 = model1(data)
+ output2 = model2(data)
+
+ output = 0.5*(output1 + output2)
+
+ loss = self.val_criterion(output, target)
+
+ self.writer.set_step((epoch - 1) * len(self.valid_data_loader) + batch_idx, 'valid')
+ self.writer.add_scalar('loss', loss.item())
+ self.val_loss_list.append(loss.item())
+ total_val_loss += loss.item()
+ total_val_metrics += self._eval_metrics(output, target)
+ self.writer.add_image('input', make_grid(data.cpu(), nrow=8, normalize=True))
+
+ # #add histogram of model parameters to the tensorboard
+ # for name, p in model.named_parameters():
+ # self.writer.add_histogram(name, p, bins='auto')
+
+ if queue is None:
+ return {
+ 'val_loss': total_val_loss / len(self.valid_data_loader),
+ 'val_metrics': (total_val_metrics / len(self.valid_data_loader)).tolist()
+ }
+ else:
+ queue.put({
+ 'val_loss': total_val_loss / len(self.valid_data_loader),
+ 'val_metrics': (total_val_metrics / len(self.valid_data_loader)).tolist()
+ })
+
+ def _test_epoch(self, epoch, model1, model2, device = 'cpu', queue = None):
+ """
+ Test after training an epoch
+
+ :return: A log that contains information about test
+
+ Note:
+ The Test metrics in log must have the key 'val_metrics'.
+ """
+ model1.eval()
+ model2.eval()
+
+ total_test_loss = 0
+ total_test_metrics = np.zeros(len(self.metrics))
+ with torch.no_grad():
+ with tqdm(self.test_data_loader) as progress:
+ for batch_idx, (data, target,indexs,_) in enumerate(progress):
+ progress.set_description_str(f'Test epoch {epoch}')
+ data, target = data.to(device), target.to(device)
+
+ output1 = model1(data)
+ output2 = model2(data)
+
+ output = 0.5*(output1 + output2)
+ loss = self.val_criterion(output, target)
+ self.writer.set_step((epoch - 1) * len(self.test_data_loader) + batch_idx, 'test')
+ self.writer.add_scalar('loss', loss.item())
+ self.test_loss_list.append(loss.item())
+ total_test_loss += loss.item()
+ total_test_metrics += self._eval_metrics(output, target)
+ self.writer.add_image('input', make_grid(data.cpu(), nrow=8, normalize=True))
+
+
+
+ #add histogram of model parameters to the tensorboard
+ for name, p in model1.named_parameters():
+ self.writer.add_histogram(name, p, bins='auto')
+ if queue is None:
+ return {
+ 'test_loss': total_test_loss / len(self.test_data_loader),
+ 'test_metrics': (total_test_metrics / len(self.test_data_loader)).tolist()
+ }
+ else:
+ queue.put({
+ 'test_loss': total_test_loss / len(self.test_data_loader),
+ 'test_metrics': (total_test_metrics / len(self.test_data_loader)).tolist()
+ })
+
+
+ def _warmup_epoch(self, epoch, model, data_loader, optimizer, train_criterion, lr_scheduler, device = 'cpu', queue = None):
+ total_loss = 0
+ total_metrics = np.zeros(len(self.metrics))
+ model.train()
+
+ with tqdm(data_loader) as progress:
+ for batch_idx, (data, target, indexs , _) in enumerate(progress):
+ progress.set_description_str(f'Train epoch {epoch}')
+
+ data, target = data.to(device), target.long().to(device)
+ optimizer.zero_grad()
+ output = model(data)
+ out_prob = output.data.detach()
+
+ train_criterion.update_hist(epoch, out_prob ,indexs.cpu().detach().numpy().tolist())
+
+ loss = torch.nn.functional.cross_entropy(output, target)
+
+ loss.backward()
+ optimizer.step()
+
+ # self.writer.set_step((epoch - 1) * self.len_epoch + batch_idx)
+ # self.writer.add_scalar('loss', loss.item())
+ self.train_loss_list.append(loss.item())
+ total_loss += loss.item()
+ total_metrics += self._eval_metrics(output, target)
+
+
+ if batch_idx % self.log_step == 0:
+ progress.set_postfix_str(' {} Loss: {:.6f}'.format(
+ self._progress(batch_idx),
+ loss.item()))
+
+
+ if batch_idx == self.len_epoch:
+ break
+
+ log = {
+ 'loss': total_loss / self.len_epoch,
+ 'noise detection rate' : 0.0,
+ 'metrics': (total_metrics / self.len_epoch).tolist(),
+ 'learning rate': lr_scheduler.get_lr()
+ }
+ if queue is None:
+ return log
+ else:
+ queue.put(log)
+
+ def eval_train(self, epoch, model_ema2, train_criterion):
+ #model.eval()
+ num_samples = args.num_batches*args.batch_size
+ losses = torch.zeros(num_samples)
+ with torch.no_grad():
+ for batch_idx, (inputs, targets, path) in enumerate(eval_loader):
+ inputs, targets = inputs.cuda(), targets.cuda()
+ output0 = model_ema2(inputs)
+ output0 = output0.data.detach()
+ output1, output2, output3 = None, None, None
+ train_criterion.update_hist(epoch, output0, output1, output2, output3, indexs.numpy().tolist(),mix_index = mix_index, mixup_l = mixup_l)
+
+
+ def update_ema_variables(self, model, model_ema, global_step, alpha_=0.997):
+ # Use the true average until the exponential average is more correct
+ if alpha_ == 0:
+ ema_param.data = param.data
+ else:
+ if self.config['ema_update']:
+ alpha = sigmoid_rampup(global_step + 1, self.config['ema_step'])*alpha_
+ else:
+ alpha = min(1 - 1 / (global_step + 1), alpha_)
+ for ema_param, param in zip(model_ema.parameters(), model.parameters()):
+ ema_param.data.mul_(alpha).add_(1 - alpha, param.data)
+
+
+ def _progress(self, batch_idx):
+ base = '[{}/{} ({:.0f}%)]'
+ if hasattr(self.data_loader1, 'n_samples'):
+ current = batch_idx * self.data_loader1.batch_size
+ total = self.data_loader1.n_samples
+ else:
+ current = batch_idx
+ total = self.len_epoch
+ return base.format(current, total, 100.0 * current / total)
+
+ def _mixup_data(self, x, y, alpha=1.0, device = 'cpu'):
+ '''Returns mixed inputs, pairs of targets, and lambda'''
+ if alpha > 0:
+ lam = np.random.beta(alpha, alpha)
+ lam = max(lam, 1-lam)
+ batch_size = x.size()[0]
+ mix_index = torch.randperm(batch_size).to(device)
+
+ mixed_x = lam * x + (1 - lam) * x[mix_index, :]#
+ mixed_target = lam * y + (1 - lam) * y[mix_index, :]
+
+
+ return mixed_x, mixed_target, lam, mix_index
+ else:
+ lam = 1
+ return x, y, lam, ...
+
+
+ def _mixup_criterion(self, pred, y_a, y_b, lam, *args):
+ loss_a, prob_a, entropy_a= self.train_criterion(pred, y_a, *args)
+ loss_b, porb_b, entropy_b = self.train_criterion(pred, y_b, *args)
+ return lam * loss_a + (1 - lam) * loss_b, lam * prob_a + (1-lam) * porb_b, lam * entropy_a + (1-lam) * entropy_b
diff --git a/ELR_plus/utils/__init__.py b/ELR_plus/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..74ace79a2790b01f71138e46a6b4243d416113ab
--- /dev/null
+++ b/ELR_plus/utils/__init__.py
@@ -0,0 +1 @@
+from .util import *
diff --git a/ELR_plus/utils/util.py b/ELR_plus/utils/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c01e3eaee5123ba9cd757e0713b871bd0db9d5e
--- /dev/null
+++ b/ELR_plus/utils/util.py
@@ -0,0 +1,91 @@
+import json
+from pathlib import Path
+from datetime import datetime
+from itertools import repeat
+from collections import OrderedDict
+import numpy as np
+
+def ensure_dir(dirname):
+ dirname = Path(dirname)
+ if not dirname.is_dir():
+ dirname.mkdir(parents=True, exist_ok=False)
+
+
+def read_json(fname):
+ with fname.open('rt') as handle:
+ return json.load(handle, object_hook=OrderedDict)
+
+
+def write_json(content, fname):
+ with fname.open('wt') as handle:
+ json.dump(content, handle, indent=4, sort_keys=False)
+
+
+def inf_loop(data_loader):
+ ''' wrapper function for endless data loader. '''
+ for loader in repeat(data_loader):
+ yield from loader
+
+
+class Timer:
+ def __init__(self):
+ self.cache = datetime.now()
+
+ def check(self):
+ now = datetime.now()
+ duration = now - self.cache
+ self.cache = now
+ return duration.total_seconds()
+
+ def reset(self):
+ self.cache = datetime.now()
+
+
+
+def sigmoid_rampup(current, rampup_length):
+ """Exponential rampup from 2"""
+ if rampup_length == 0:
+ return 1.0
+ else:
+ current = np.clip(current, 0.0, rampup_length)
+ phase = 1.0 - current / rampup_length
+ return float(np.exp(-5.0 * phase * phase))
+
+def sigmoid_rampdown(current, rampdown_length):
+ """Exponential rampdown"""
+ if rampdown_length == 0:
+ return 1.0
+ else:
+ current = np.clip(current, 0.0, rampdown_length)
+ phase = 1.0 - (rampdown_length - current) / rampdown_length
+ return float(np.exp(-12.5 * phase * phase))
+
+def linear_rampup(current, rampup_length):
+ """Linear rampup"""
+ assert current >= 0 and rampup_length >= 0
+ if current >= rampup_length:
+ return 1.0
+ else:
+ return current / rampup_length
+
+def linear_rampdown(current, rampdown_length):
+ """Linear rampup"""
+ assert current >= 0 and rampdown_length >= 0
+ if current >= rampdown_length:
+ return 1.0
+ else:
+ return 1.0 - current / rampdown_length
+
+
+def cosine_rampdown(current, rampdown_length):
+ """Cosine rampdown from https://arxiv.org/abs/1608.03983"""
+ current = np.clip(current, 0.0, rampdown_length)
+ return float(.5 * (np.cos(np.pi * current / rampdown_length) + 1))
+
+
+def cosine_rampup(current, rampup_length):
+ """Cosine rampup"""
+ current = np.clip(current, 0.0, rampup_length)
+ return float(-.5 * (np.cos(np.pi * current / rampup_length) - 1))
+
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..a727e13f64bdcd8be6f62accd85a6c23fdd567aa
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Sheng Liu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index 22aa4bf935316e49699a08ba4c5e6d9394578555..000506a60dd2c58470a504b2b70afbfaab9a229c 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,141 @@
----
-title: Assi1
-emoji: 📉
-colorFrom: blue
-colorTo: purple
-sdk: gradio
-sdk_version: 5.20.1
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+# Early-Learning Regularization Prevents Memorization of Noisy Labels
+[](https://arxiv.org/abs/2007.00151)
+
+
+
+This repository is the official implementation of [Early-Learning Regularization Prevents Memorization of Noisy Labels](https://arxiv.org/abs/2007.00151) (NeurIPS 2020).
+
+We propose a novel framework to perform classification via deep learning in the presence of **noisy annotations**. When trained on noisy labels, deep neural networks have been observed to first fit the training data with clean labels during an **early learning** phase, before eventually **memorizing** the examples with false labels. Our technique exploits the progress of the early learning phase via **regularization** to perform classification from noisy labels. There are two key elements to our approach. First, we leverage semi-supervised learning techniques to produce target probabilities based on the model outputs. Second, we design a regularization term that steers the model towards these targets, implicitly preventing memorization of the false labels. The resulting framework is shown to provide robustness to noisy annotations on several standard benchmarks and real-world datasets, where it achieves results comparable to the state of the art.
+
+
+
+
+These graphs show the results of training a ResNet-34 with a traditional cross entropy loss (top row) and our proposed method (bottom row) to perform classification on the CIFAR-10 dataset where 40% of the labels are flipped at random. The left column shows the fraction of examples with clean labels that are predicted correctly (green) and incorrectly (blue). The right column shows the fraction of examples with wrong labels that are predicted correctly (green), memorized (the prediction equals the wrong label, shown in red), and incorrectly predicted as neither the true nor the labeled class (blue). The model trained with cross entropy begins by learning to predict the true labels, even for many of the examples with wrong labels, but eventually memorizes the wrong labels. Our proposed method based on early-learning regularization prevents memorization, allowing the model to continue learning on the examples with clean labels to attain high accuracy on examples with both clean and wrong labels.
+
+
+
+
+
+
+
+Learning path of sample with correct label (left) and sample with wrong label (right). Corners correspond to one-hot
+vectors. Bright green represents model's prediction: when the example is wrongly labeled, the clean label is predicted during early-learning, and then wrong label is predicted at the end of training. The model is trained with first 3 classes in CIFAR10.
+
+
+
+## Early-learning modeled for each example
+Early-learning could not happen simultaneously for all examples, e.g. when noisy labels are dependent to each instance. Therefore, In [SOP](https://github.com/shengliu66/SOP), We model the early-learning phenomenon for each example using a overparameterization term learned for each instance. We further impose sparsity on it via implicit bias of stochastic gradient descent. This method achieved SoTA for instance dependent noisy label. If you are interested, take a look at our [Paper](https://proceedings.mlr.press/v162/liu22w/liu22w.pdf) published in ICML 2022!
+
+
+## Requirements
+- This codebase is written for `python3`.
+- To install necessary python packages, run `pip install -r requirements.txt`.
+
+
+## Training
+### Basics
+- ELR loss is implemented in the file [`loss.py`](./ELR/model/loss.py)
+- All functions used for training the basic version of our technique (**ELR**) can be found in the `ELR` folder.
+- All functions used for training the more advanced version (**ELR+**) can be found in the `ELR_plus` folder.
+- Experiments settings and configurations used for different datasets are in the corresponding config json files.
+### Data
+- Please download the data before running the code, add path to the downloaded data to `data_loader.args.data_dir` in the corresponding config file.
+### Training
+- Code for training ELR is in the following file: [`train.py`](./ELR/train.py), code for training ELR+ is in the following file: [`train.py`](./ELR_plus/train.py)
+```
+usage: train.py [-c] [-r] [-d] [--lr learning_rate] [--bs batch_size] [--beta beta] [--lambda lambda] [--malpha mixup_alpha]
+ [--percent percent] [--asym asym] [--ealpha ema_alpha] [--name exp_name]
+
+ arguments:
+ -c, --config config file path (default: None)
+ -r, --resume path to latest checkpoint (default: None)
+ -d, --device indices of GPUs to enable (default: all)
+
+ options:
+ --lr learning_rate learning rate (default value is the value in the config file)
+ --bs batch_size batch size (default value is the value in the config file)
+ --beta beta temporal ensembling momentum beta for target estimation
+ --lambda lambda regularization coefficient
+ --malpha mixup_alpha mixup parameter alpha
+ --percent percent noise level (e.g. 0.4 for 40%)
+ --asym asym asymmetric noise is used when set to True
+ --ealpha ema_alpha weight averaging momentum for target estimation
+ --name exp_name experiment name
+```
+Configuration file is **required** to be specified. Default option values, if not reset, will be the values in the configuration file.
+Examples for ELR and ELR+ are shown in the *readme.md* of `ELR` and `ELR_plus` subfolders respectively.
+
+### Example
+In order to use our proposed early learning regularization (ELR), you can simply replace your loss function by the following loss function. Usually, **lambda** which is used to control the strength of the regularization term needs to be tuned more carefully, and the value of **beta** is quite robust (can be 0.7, 0.9 or 0.99, etc.)
+```
+class elr_loss(nn.Module):
+ def __init__(self, num_examp, num_classes=10, lambda = 3, beta=0.7):
+ r"""Early Learning Regularization.
+ Parameters
+ * `num_examp` Total number of training examples.
+ * `num_classes` Number of classes in the classification problem.
+ * `lambda` Regularization strength; must be a positive float, controling the strength of the ELR.
+ * `beta` Temporal ensembling momentum for target estimation.
+ """
+
+ super(elr_loss, self).__init__()
+ self.num_classes = num_classes
+ self.USE_CUDA = torch.cuda.is_available()
+ self.target = torch.zeros(num_examp, self.num_classes).cuda() if self.USE_CUDA else torch.zeros(num_examp, self.num_classes)
+ self.beta = beta
+ self.lambda = lambda
+
+
+ def forward(self, index, output, label):
+ r"""Early Learning Regularization.
+ Args
+ * `index` Training sample index, due to training set shuffling, index is used to track training examples in different iterations.
+ * `output` Model's logits, same as PyTorch provided loss functions.
+ * `label` Labels, same as PyTorch provided loss functions.
+ """
+
+ y_pred = F.softmax(output,dim=1)
+ y_pred = torch.clamp(y_pred, 1e-4, 1.0-1e-4)
+ y_pred_ = y_pred.data.detach()
+ self.target[index] = self.beta * self.target[index] + (1-self.beta) * ((y_pred_)/(y_pred_).sum(dim=1,keepdim=True))
+ ce_loss = F.cross_entropy(output, label)
+ elr_reg = ((1-(self.target[index] * y_pred).sum(dim=1)).log()).mean()
+ final_loss = ce_loss + self.lambda *elr_reg
+ return final_loss
+
+```
+## Identify Wrong Labels
+- After finish training, obtain and compare the self.target of the ELR loss to original labels y
+- the mislabeled examples are identified as those who has argmax(self.target) != y
+
+
+## License and Contributing
+- This README is formatted based on [paperswithcode](https://github.com/paperswithcode/releasing-research-code).
+- Feel free to post issues via Github.
+
+## Reference
+For technical details and full experimental results, please check [our paper](https://arxiv.org/abs/2007.00151).
+```
+@InProceedings{pmlr-v162-liu22w,
+ title = {Robust Training under Label Noise by Over-parameterization},
+ author = {Liu, Sheng and Zhu, Zhihui and Qu, Qing and You, Chong},
+ journal = {Proceedings of the 39th International Conference on Machine Learning},
+ volume = {162},
+ year = {2022}
+}
+```
+```
+@article{liu2020early,
+ title={Early-Learning Regularization Prevents Memorization of Noisy Labels},
+ author={Liu, Sheng and Niles-Weed, Jonathan and Razavian, Narges and Fernandez-Granda, Carlos},
+ journal={Advances in Neural Information Processing Systems},
+ volume={33},
+ year={2020}
+}
+```
+Similar early learning and memorization phenomenon is observed in semantic segmentations, a related paper addressing WSSS by adaptive correction:
+[Adaptive Early-Learning Correction for Segmentation from Noisy Annotations](https://arxiv.org/abs/2110.03740) (CVPR2022 **Oral**).
+## Contact
+Please contact shengliu@nyu.edu if you have any question on the codes.
diff --git a/images/ELR.png b/images/ELR.png
new file mode 100644
index 0000000000000000000000000000000000000000..c2c85844e7a5e9f46c39c77909bfcd8c7bf9ff14
Binary files /dev/null and b/images/ELR.png differ
diff --git a/images/clean_label_simplexheatmap2.gif b/images/clean_label_simplexheatmap2.gif
new file mode 100644
index 0000000000000000000000000000000000000000..bbbe036528f932c4b870c6b4c5d4eb635598dd79
--- /dev/null
+++ b/images/clean_label_simplexheatmap2.gif
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6915d13dc449834481ef5246e02f09160fbe989f769d619ef6b8b15a191e500
+size 833882
diff --git a/images/false_label_simplexheatmap.gif b/images/false_label_simplexheatmap.gif
new file mode 100644
index 0000000000000000000000000000000000000000..566dfbdf750c39b7b1c92e43bc54b6f5ed07dc9a
--- /dev/null
+++ b/images/false_label_simplexheatmap.gif
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3e5cc6928ebc26966b161a9308021195df98b418139b724cd1f6c20748649db
+size 812027
diff --git a/images/illustration_of_ELR.png b/images/illustration_of_ELR.png
new file mode 100644
index 0000000000000000000000000000000000000000..8c72436047a51c4ce447403ca379b3f9241249c5
--- /dev/null
+++ b/images/illustration_of_ELR.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94bc2ee99bbc79324750cf458d48051cf4cd41d4f3c8174c2eebb8605b83425a
+size 1120626
diff --git a/images/simplexheatmap.gif b/images/simplexheatmap.gif
new file mode 100644
index 0000000000000000000000000000000000000000..00d1bd2ccf541d4be1c3b6072aa16d2913ca44a9
--- /dev/null
+++ b/images/simplexheatmap.gif
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18e68ee9843819bd4101c47e0695c2333e6df372cfd5cf847b2db4441f53e0d9
+size 3917605
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..64de891b7a7b4720930323da97a77fb5f4078f77
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,8 @@
+torchvision==0.4.0a0
+requests==2.22.0
+tqdm==4.32.1
+torch==1.2.0
+numpy==1.16.4
+mlflow==1.9.1
+Pillow==8.1.1
+scikit_learn==0.23.1