mnist: use mirror only when original server is not available

lib/datasets/dataset.rb

@@ -33,9 +33,21 @@ def cache_path
       @cache_path ||= CachePath.new(@metadata.id)
     end
 
-    def download(output_path, url, &block)
-      downloader = Downloader.new(url)
-      downloader.download(output_path, &block)
+    def download(output_path, *urls, &block)
+      urls.each do |url|
+        downloader = Downloader.new(url)
+        downloader.download(output_path, &block)
+        return
+      rescue Net::HTTPClientException => error
+        if urls.last != url
+          message = "site is not available: " +
+                    "#{error.class}: #{error.message}: " +
+                    "Attempting to download from an alternative site"
+          $stderr.puts(message)
+        else
+          raise error
+        end
+      end
     end
 
     def extract_bz2(bz2)

lib/datasets/fashion-mnist.rb

@@ -2,9 +2,13 @@
 
 module Datasets
   class FashionMNIST < MNIST
-    BASE_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/"
-
     private
+    def base_urls
+      [
+        "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/",
+      ]
+    end
+
     def dataset_name
       "Fashion-MNIST"
     end

lib/datasets/kuzushiji-mnist.rb

@@ -2,9 +2,13 @@
 
 module Datasets
   class KuzushijiMNIST < MNIST
-    BASE_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/"
-
     private
+    def base_urls
+      [
+        "http://codh.rois.ac.jp/kmnist/dataset/kmnist/",
+      ]
+    end
+
     def dataset_name
       "Kuzushiji-MNIST"
     end

lib/datasets/mnist.rb

@@ -4,8 +4,6 @@
 
 module Datasets
   class MNIST < Dataset
-    BASE_URL = "http://yann.lecun.com/exdb/mnist/"
-
     class Record < Struct.new(:data, :label)
       def pixels
         data.unpack("C*")
@@ -27,7 +25,7 @@ def initialize(type: :train)
 
       @metadata.id = "#{dataset_name.downcase}-#{type}"
       @metadata.name = "#{dataset_name}: #{type}"
-      @metadata.url = self.class::BASE_URL
+      @metadata.url = base_urls.first
       @metadata.licenses = licenses
       @type = type
 
@@ -44,15 +42,23 @@ def each(&block)
 
       image_path = cache_dir_path + target_file(:image)
       label_path = cache_dir_path + target_file(:label)
-      base_url = self.class::BASE_URL
 
-      download(image_path, base_url + target_file(:image))
-      download(label_path, base_url + target_file(:label))
+      download(image_path,
+               *base_urls.collect { |base_url| base_url + target_file(:image) })
+      download(label_path,
+               *base_urls.collect { |base_url| base_url + target_file(:label) })
 
       open_data(image_path, label_path, &block)
     end
 
     private
+    def base_urls
+      [
+        "http://yann.lecun.com/exdb/mnist/",
+        "https://ossci-datasets.s3.amazonaws.com/mnist/",
+      ]
+    end
+
     def licenses
       []
     end