#9 : code changes for imagenet(resnet) reproducibility

Author: ildoonet

README.md

@@ -61,6 +61,10 @@ $ python RandAugment/train.py -c confs/wresnet28x10_cifar10_b256.yaml --save cif
 
 ### ImageNet Classification
 
+I have experienced some difficulties while reproducing paper's result.
+
+**Issue : https://github.com/ildoonet/pytorch-randaugment/issues/9**
+
 | Model             | Paper's Result | Ours         |
 |-------------------|---------------:|-------------:|
 | ResNet-50         | 77.6 / 92.8    | TODO 

RandAugment/augmentations.py

@@ -160,13 +160,13 @@ def augment_list():  # 16 oeprations and their ranges
         (Identity, 0., 1.0),
         (ShearX, 0., 0.3),  # 0
         (ShearY, 0., 0.3),  # 1
-        (TranslateX, 0., 0.45),  # 2
-        (TranslateY, 0., 0.45),  # 3
+        (TranslateX, 0., 0.33),  # 2
+        (TranslateY, 0., 0.33),  # 3
         (Rotate, 0, 30),  # 4
         (AutoContrast, 0, 1),  # 5
         (Invert, 0, 1),  # 6
         (Equalize, 0, 1),  # 7
-        (Solarize, 0, 256),  # 8
+        (Solarize, 0, 110),  # 8
         (Posterize, 4, 8),  # 9
         # (Contrast, 0.1, 1.9),  # 10
         (Color, 0.1, 1.9),  # 11

RandAugment/metrics.py

@@ -22,19 +22,6 @@ def accuracy(output, target, topk=(1,)):
     return res
 
 
-def cross_entropy_smooth(input, target, size_average=True, label_smoothing=0.1):
-    y = torch.eye(10).cuda()
-    lb_oh = y[target]
-
-    target = lb_oh * (1 - label_smoothing) + 0.5 * label_smoothing
-
-    logsoftmax = nn.LogSoftmax()
-    if size_average:
-        return torch.mean(torch.sum(-target * logsoftmax(input), dim=1))
-    else:
-        return torch.sum(torch.sum(-target * logsoftmax(input), dim=1))
-
-
 class Accumulator:
     def __init__(self):
         self.metrics = defaultdict(lambda: 0.)

RandAugment/networks/resnet.py

@@ -106,8 +106,8 @@ def __init__(self, dataset, depth, num_classes, bottleneck=False):
             self.fc = nn.Linear(64 * block.expansion, num_classes)
 
         elif dataset == 'imagenet':
-            blocks ={18: BasicBlock, 34: BasicBlock, 50: Bottleneck, 101: Bottleneck, 152: Bottleneck, 200: Bottleneck}
-            layers ={18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3], 200: [3, 24, 36, 3]}
+            blocks = {18: BasicBlock, 34: BasicBlock, 50: Bottleneck, 101: Bottleneck, 152: Bottleneck, 200: Bottleneck}
+            layers = {18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3], 200: [3, 24, 36, 3]}
             assert layers[depth], 'invalid detph for ResNet (depth should be one of 18, 34, 50, 101, 152, and 200)'
 
             self.inplanes = 64

RandAugment/smooth_ce.py

@@ -0,0 +1,43 @@
+import numpy as np
+import torch
+from torch.nn.modules.module import Module
+
+
+class SmoothCrossEntropyLoss(Module):
+    def __init__(self, label_smoothing=0.0, size_average=True):
+        super().__init__()
+        self.label_smoothing = label_smoothing
+        self.size_average = size_average
+
+    def forward(self, input, target):
+        if len(target.size()) == 1:
+            target = torch.nn.functional.one_hot(target, num_classes=input.size(-1))
+            target = target.float().cuda()
+        if self.label_smoothing > 0.0:
+            s_by_c = self.label_smoothing / len(input[0])
+            smooth = torch.zeros_like(target)
+            smooth = smooth + s_by_c
+            target = target * (1. - s_by_c) + smooth
+
+        return cross_entropy(input, target, self.size_average)
+
+
+def cross_entropy(input, target, size_average=True):
+    """ Cross entropy that accepts soft targets
+    Args:
+         pred: predictions for neural network
+         targets: targets, can be soft
+         size_average: if false, sum is returned instead of mean
+    Examples::
+        input = torch.FloatTensor([[1.1, 2.8, 1.3], [1.1, 2.1, 4.8]])
+        input = torch.autograd.Variable(out, requires_grad=True)
+        target = torch.FloatTensor([[0.05, 0.9, 0.05], [0.05, 0.05, 0.9]])
+        target = torch.autograd.Variable(y1)
+        loss = cross_entropy(input, target)
+        loss.backward()
+    """
+    logsoftmax = torch.nn.LogSoftmax(dim=1)
+    if size_average:
+        return torch.mean(torch.sum(-target * logsoftmax(input), dim=1))
+    else:
+        return torch.sum(torch.sum(-target * logsoftmax(input), dim=1))

RandAugment/train.py

@@ -20,6 +20,7 @@
 from warmup_scheduler import GradualWarmupScheduler
 
 from RandAugment.common import add_filehandler
+from RandAugment.smooth_ce import SmoothCrossEntropyLoss
 
 logger = get_logger('RandAugment')
 logger.setLevel(logging.INFO)
@@ -94,7 +95,11 @@ def train_and_eval(tag, dataroot, test_ratio=0.0, cv_fold=0, reporter=None, metr
     # create a model & an optimizer
     model = get_model(C.get()['model'], num_class(C.get()['dataset']))
 
-    criterion = nn.CrossEntropyLoss()
+    lb_smooth = C.get()['optimizer'].get('label_smoothing', 0.0)
+    if lb_smooth > 0.0:
+        criterion = SmoothCrossEntropyLoss(lb_smooth)
+    else:
+        criterion = nn.CrossEntropyLoss()
     if C.get()['optimizer']['type'] == 'sgd':
         optimizer = optim.SGD(
             model.parameters(),
@@ -106,6 +111,11 @@ def train_and_eval(tag, dataroot, test_ratio=0.0, cv_fold=0, reporter=None, metr
     else:
         raise ValueError('invalid optimizer type=%s' % C.get()['optimizer']['type'])
 
+    if C.get()['optimizer'].get('lars', False):
+        from torchlars import LARS
+        optimizer = LARS(optimizer)
+        logger.info('*** LARS Enabled.')
+
     lr_scheduler_type = C.get()['lr_schedule'].get('type', 'cosine')
     if lr_scheduler_type == 'cosine':
         scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=C.get()['epoch'], eta_min=0.)

confs/resnet50_b1024.yaml

@@ -20,3 +20,5 @@ optimizer:
   nesterov: True
   decay: 0.0001
   clip: 0
+  lars: False
+  label_smoothing: 0.0

confs/resnet50_b512.yaml

@@ -0,0 +1,24 @@
+model:
+  type: resnet50
+dataset: imagenet
+aug: randaugment
+randaug:
+  N: 2
+  M: 9
+
+cutout: 0
+batch: 512
+epoch: 180  # 270
+lr: 0.1
+lr_schedule:
+  type: 'resnet'
+  warmup:
+    multiplier: 2
+    epoch: 3
+optimizer:
+  type: sgd
+  nesterov: True
+  decay: 0.0001
+  clip: 0
+  lars: False
+  label_smoothing: 0.0

requirements.txt

@@ -11,3 +11,4 @@ ray
 matplotlib
 psutil
 requests
+torchlars