Merge branch 'dev'

Author: hkalexling

README.md

@@ -12,6 +12,7 @@ Mango is a self-hosted manga server and reader. Its features include
 - Supported formats: `.cbz`, `.zip`, `.cbr` and `.rar`
 - Supports nested folders in library
 - Automatically stores reading progress
+- Thumbnail generation
 - Built-in [MangaDex](https://mangadex.org/) downloader
 - Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites
 - The web reader is responsive and works well on mobile, so there is no need for a mobile app
@@ -51,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
 ### CLI
 
 ```
-  Mango - Manga Server and Web Reader. Version 0.14.0
+  Mango - Manga Server and Web Reader. Version 0.15.0
 
   Usage:
 
@@ -80,6 +81,8 @@ session_secret: mango-session-secret
 library_path: ~/mango/library
 db_path: ~/mango/mango.db
 scan_interval_minutes: 5
+thumbnail_generation_interval_hours: 24
+db_optimization_interval_hours: 24
 log_level: info
 upload_path: ~/mango/uploads
 plugin_path: ~/mango/plugins
@@ -89,12 +92,12 @@ mangadex:
   api_url: https://mangadex.org/api
   download_wait_seconds: 5
   download_retries: 4
-  download_queue_db_path: ~/mango/queue.db
+  download_queue_db_path: /home/alex_ling/mango/queue.db
   chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}'
   manga_rename_rule: '{title}'
 ```
 
-- `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan
+- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
 - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging
 
 ### Library Structure

public/js/admin.js

@@ -1,40 +1,90 @@
-let scanning = false;
-
-const scan = () => {
-	scanning = true;
-	$('#scan-status > div').removeAttr('hidden');
-	$('#scan-status > span').attr('hidden', '');
-	const color = $('#scan').css('color');
-	$('#scan').css('color', 'gray');
-	$.post(base_url + 'api/admin/scan', (data) => {
-		const ms = data.milliseconds;
-		const titles = data.titles;
-		$('#scan-status > span').text('Scanned ' + titles + ' titles in ' + ms + 'ms');
-		$('#scan-status > span').removeAttr('hidden');
-		$('#scan').css('color', color);
-		$('#scan-status > div').attr('hidden', '');
-		scanning = false;
-	});
-}
-
-String.prototype.capitalize = function() {
-	return this.charAt(0).toUpperCase() + this.slice(1);
-}
-
 $(() => {
-	$('li').click((e) => {
-		const url = $(e.currentTarget).attr('data-url');
-		if (url) {
-			$(location).attr('href', url);
-		}
-	});
-
 	const setting = loadThemeSetting();
-	$('#theme-select').val(setting.capitalize());
-
+	$('#theme-select').val(capitalize(setting));
 	$('#theme-select').change((e) => {
 		const newSetting = $(e.currentTarget).val().toLowerCase();
 		saveThemeSetting(newSetting);
 		setTheme();
 	});
+
+	getProgress();
+	setInterval(getProgress, 5000);
 });
+
+/**
+ * Capitalize String
+ *
+ * @function capitalize
+ * @param {string} str - The string to be capitalized
+ * @return {string} The capitalized string
+ */
+const capitalize = (str) => {
+	return str.charAt(0).toUpperCase() + str.slice(1);
+};
+
+/**
+ * Set an alpine.js property
+ *
+ * @function setProp
+ * @param {string} key - Key of the data property
+ * @param {*} prop - The data property
+ */
+const setProp = (key, prop) => {
+	$('#root').get(0).__x.$data[key] = prop;
+};
+
+/**
+ * Get an alpine.js property
+ *
+ * @function getProp
+ * @param {string} key - Key of the data property
+ * @return {*} The data property
+ */
+const getProp = (key) => {
+	return $('#root').get(0).__x.$data[key];
+};
+
+/**
+ * Get the thumbnail generation progress from the API
+ *
+ * @function getProgress
+ */
+const getProgress = () => {
+	$.get(`${base_url}api/admin/thumbnail_progress`)
+		.then(data => {
+			setProp('progress', data.progress);
+			const generating = data.progress > 0
+			setProp('generating', generating);
+		});
+};
+
+/**
+ * Trigger the thumbnail generation
+ *
+ * @function generateThumbnails
+ */
+const generateThumbnails = () => {
+	setProp('generating', true);
+	setProp('progress', 0.0);
+	$.post(`${base_url}api/admin/generate_thumbnails`)
+		.then(getProgress);
+};
+
+/**
+ * Trigger the scan
+ *
+ * @function scan
+ */
+const scan = () => {
+	setProp('scanning', true);
+	setProp('scanMs', -1);
+	setProp('scanTitles', 0);
+	$.post(`${base_url}api/admin/scan`)
+		.then(data => {
+			setProp('scanMs', data.milliseconds);
+			setProp('scanTitles', data.titles);
+		})
+		.always(() => {
+			setProp('scanning', false);
+		});
+}

public/js/dots.js

@@ -1,17 +1,26 @@
-const truncate = () => {
-	$('.uk-card-title').each((i, e) => {
-		$(e).dotdotdot({
-			truncate: 'letter',
-			watch: true,
-			callback: (truncated) => {
-				if (truncated) {
-					$(e).attr('uk-tooltip', $(e).attr('data-title'));
-				} else {
-					$(e).removeAttr('uk-tooltip');
-				}
+/**
+ * Truncate a .uk-card-title element
+ *
+ * @function truncate
+ * @param {object} e - The title element to truncate
+ */
+const truncate = (e) => {
+	$(e).dotdotdot({
+		truncate: 'letter',
+		watch: true,
+		callback: (truncated) => {
+			if (truncated) {
+				$(e).attr('uk-tooltip', $(e).attr('data-title'));
+			} else {
+				$(e).removeAttr('uk-tooltip');
 			}
-		});
+		}
 	});
 };
 
-truncate();
+$('.uk-card-title').each((i, e) => {
+	// Truncate the title when it first enters the view
+	$(e).one('inview', () => {
+		truncate(e);
+	});
+});

shard.lock

@@ -34,7 +34,7 @@ shards:
 
   image_size:
     github: hkalexling/image_size.cr
-    version: 0.2.0
+    version: 0.4.0
 
   kemal:
     github: kemalcr/kemal

shard.yml

@@ -1,5 +1,5 @@
 name: mango
-version: 0.14.0
+version: 0.15.0
 
 authors:
   - Alex Ling <hkalexling@gmail.com>

src/config.cr

@@ -11,8 +11,9 @@ class Config
   property library_path : String = File.expand_path "~/mango/library",
     home: true
   property db_path : String = File.expand_path "~/mango/mango.db", home: true
-  @[YAML::Field(key: "scan_interval_minutes")]
-  property scan_interval : Int32 = 5
+  property scan_interval_minutes : Int32 = 5
+  property thumbnail_generation_interval_hours : Int32 = 24
+  property db_optimization_interval_hours : Int32 = 24
   property log_level : String = "info"
   property upload_path : String = File.expand_path "~/mango/uploads",
     home: true

src/library/entry.cr

@@ -69,7 +69,7 @@ class Entry
 
   def cover_url
     return "#{Config.current.base_url}img/icon.png" if @err_msg
-    url = "#{Config.current.base_url}api/page/#{@book.id}/#{@id}/1"
+    url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}"
     TitleInfo.new @book.dir do |info|
       info_url = info.entry_cover_url[@title]?
       unless info_url.nil? || info_url.empty?
@@ -207,4 +207,29 @@ class Entry
   def started?(username)
     load_progress(username) > 0
   end
+
+  def generate_thumbnail : Image?
+    return if @err_msg
+
+    img = read_page(1).not_nil!
+    begin
+      size = ImageSize.get img.data
+      if size.height > size.width
+        thumbnail = ImageSize.resize img.data, width: 200
+      else
+        thumbnail = ImageSize.resize img.data, height: 300
+      end
+      img.data = thumbnail
+      Storage.default.save_thumbnail @id, img
+    rescue e
+      Logger.warn "Failed to generate thumbnail for entry " \
+                  "#{@book.title}/#{@title}. #{e}"
+    end
+
+    img
+  end
+
+  def get_thumbnail : Image?
+    Storage.default.get_thumbnail @id
+  end
 end

src/library/library.cr

@@ -1,5 +1,5 @@
 class Library
-  property dir : String, title_ids : Array(String), scan_interval : Int32,
+  property dir : String, title_ids : Array(String),
     title_hash : Hash(String, Title)
 
   use_default
@@ -8,20 +8,48 @@ class Library
     register_mime_types
 
     @dir = Config.current.library_path
-    @scan_interval = Config.current.scan_interval
     # explicitly initialize @titles to bypass the compiler check. it will
     #   be filled with actual Titles in the `scan` call below
     @title_ids = [] of String
     @title_hash = {} of String => Title
 
-    return scan if @scan_interval < 1
-    spawn do
-      loop do
-        start = Time.local
-        scan
-        ms = (Time.local - start).total_milliseconds
-        Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
-        sleep @scan_interval * 60
+    @entries_count = 0
+    @thumbnails_count = 0
+
+    scan_interval = Config.current.scan_interval_minutes
+    if scan_interval < 1
+      scan
+    else
+      spawn do
+        loop do
+          start = Time.local
+          scan
+          ms = (Time.local - start).total_milliseconds
+          Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
+          sleep scan_interval.minutes
+        end
+      end
+    end
+
+    thumbnail_interval = Config.current.thumbnail_generation_interval_hours
+    unless thumbnail_interval < 1
+      spawn do
+        loop do
+          # Wait for scan to complete (in most cases)
+          sleep 1.minutes
+          generate_thumbnails
+          sleep thumbnail_interval.hours
+        end
+      end
+    end
+
+    db_interval = Config.current.db_optimization_interval_hours
+    unless db_interval < 1
+      spawn do
+        loop do
+          Storage.default.optimize
+          sleep db_interval.hours
+        end
       end
     end
   end
@@ -194,4 +222,50 @@ class Library
       .sample(ENTRIES_IN_HOME_SECTIONS)
       .shuffle
   end
+
+  def thumbnail_generation_progress
+    return 0 if @entries_count == 0
+    @thumbnails_count / @entries_count
+  end
+
+  def generate_thumbnails
+    if @thumbnails_count > 0
+      Logger.debug "Thumbnail generation in progress"
+      return
+    end
+
+    Logger.info "Starting thumbnail generation"
+    entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg
+    @entries_count = entries.size
+    @thumbnails_count = 0
+
+    # Report generation progress regularly
+    spawn do
+      loop do
+        unless @thumbnails_count == 0
+          Logger.debug "Thumbnail generation progress: " \
+                       "#{(thumbnail_generation_progress * 100).round 1}%"
+        end
+        # Generation is completed. We reset the count to 0 to allow subsequent
+        #   calls to the function, and break from the loop to stop the progress
+        #   report fiber
+        if thumbnail_generation_progress.to_i == 1
+          @thumbnails_count = 0
+          break
+        end
+        sleep 10.seconds
+      end
+    end
+
+    entries.each do |e|
+      unless e.get_thumbnail
+        e.generate_thumbnail
+        # Sleep after each generation to minimize the impact on disk IO
+        #   and CPU
+        sleep 0.5.seconds
+      end
+      @thumbnails_count += 1
+    end
+    Logger.info "Thumbnail generation finished"
+  end
 end