From 308ad4e063921fb1c6f01f2d3299f18b50757203 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 20 Oct 2020 14:36:56 +0000 Subject: [PATCH 01/13] Only truncate visible titles to improve load time --- public/js/dots.js | 35 ++++-- src/views/components/dots-scripts.html.ecr | 3 + src/views/home.html.ecr | 133 ++++++++++----------- src/views/library.html.ecr | 3 +- src/views/title.html.ecr | 3 +- 5 files changed, 93 insertions(+), 84 deletions(-) create mode 100644 src/views/components/dots-scripts.html.ecr diff --git a/public/js/dots.js b/public/js/dots.js index 75a6aa6..266c846 100644 --- a/public/js/dots.js +++ b/public/js/dots.js @@ -1,17 +1,26 @@ -const truncate = () => { - $('.uk-card-title').each((i, e) => { - $(e).dotdotdot({ - truncate: 'letter', - watch: true, - callback: (truncated) => { - if (truncated) { - $(e).attr('uk-tooltip', $(e).attr('data-title')); - } else { - $(e).removeAttr('uk-tooltip'); - } +/** + * Truncate a .uk-card-title element + * + * @function truncate + * @param {object} e - The title element to truncate + */ +const truncate = (e) => { + $(e).dotdotdot({ + truncate: 'letter', + watch: true, + callback: (truncated) => { + if (truncated) { + $(e).attr('uk-tooltip', $(e).attr('data-title')); + } else { + $(e).removeAttr('uk-tooltip'); } - }); + } }); }; -truncate(); +$('.uk-card-title').each((i, e) => { + // Truncate the title when it first enters the view + $(e).one('inview', () => { + truncate(e); + }); +}); diff --git a/src/views/components/dots-scripts.html.ecr b/src/views/components/dots-scripts.html.ecr new file mode 100644 index 0000000..41bb7b0 --- /dev/null +++ b/src/views/components/dots-scripts.html.ecr @@ -0,0 +1,3 @@ + + + diff --git a/src/views/home.html.ecr b/src/views/home.html.ecr index f48a2ef..0700f4f 100644 --- a/src/views/home.html.ecr +++ b/src/views/home.html.ecr @@ -1,84 +1,83 @@ <%- if new_user && empty_library -%> -
- -

Add your first manga

-

We can't find any files yet. Add some to your library and they'll appear here.

-
-
Current library path
-
<%= Config.current.library_path %>
-
Want to change your library path?
-
Update config.yml located at: <%= Config.current.path %>
-
Can't see your files yet?
-
- You must wait <%= Config.current.scan_interval %> minutes for the library scan to complete - <% if is_admin %> - , or manually re-scan from Admin - <% end %>. -
-
-
+
+ +

Add your first manga

+

We can't find any files yet. Add some to your library and they'll appear here.

+
+
Current library path
+
<%= Config.current.library_path %>
+
Want to change your library path?
+
Update config.yml located at: <%= Config.current.path %>
+
Can't see your files yet?
+
+ You must wait <%= Config.current.scan_interval %> minutes for the library scan to complete + <% if is_admin %> + , or manually re-scan from Admin + <% end %>. +
+
+
<%- elsif new_user && empty_library == false -%> -
- -

Read your first manga

-

Once you start reading, Mango will remember where you left off - and show your entries here.

- View library -
+
+ +

Read your first manga

+

Once you start reading, Mango will remember where you left off + and show your entries here.

+ View library +
<%- elsif new_user == false && empty_library == false -%> - <%- if continue_reading.empty? && recently_added.empty? -%> -
- -

A self-hosted manga server and reader

- View library -
- <%- end -%> + <%- if continue_reading.empty? && recently_added.empty? -%> +
+ +

A self-hosted manga server and reader

+ View library +
+ <%- end -%> - <%- unless continue_reading.empty? -%> -

Continue Reading

-
- <%- continue_reading.each do |cr| -%> - <% item = cr[:entry] %> - <% progress = cr[:percentage] %> - <%= render_component "card" %> - <%- end -%> -
- <%- end -%> + <%- unless continue_reading.empty? -%> +

Continue Reading

+
+ <%- continue_reading.each do |cr| -%> + <% item = cr[:entry] %> + <% progress = cr[:percentage] %> + <%= render_component "card" %> + <%- end -%> +
+ <%- end -%> - <%- unless start_reading.empty? -%> -

Start Reading

-
- <%- start_reading.each do |t| -%> - <% item = t %> - <% progress = 0.0 %> - <%= render_component "card" %> - <%- end -%> -
- <%- end -%> + <%- unless start_reading.empty? -%> +

Start Reading

+
+ <%- start_reading.each do |t| -%> + <% item = t %> + <% progress = 0.0 %> + <%= render_component "card" %> + <%- end -%> +
+ <%- end -%> - <%- unless recently_added.empty? -%> -

Recently Added

-
- <%- recently_added.each do |ra| -%> - <% item = ra %> - <% progress = ra[:percentage] %> - <%= render_component "card" %> - <%- end -%> -
- <%- end -%> + <%- unless recently_added.empty? -%> +

Recently Added

+
+ <%- recently_added.each do |ra| -%> + <% item = ra %> + <% progress = ra[:percentage] %> + <%= render_component "card" %> + <%- end -%> +
+ <%- end -%> - <%= render_component "entry-modal" %> + <%= render_component "entry-modal" %> <%- end -%> <% content_for "script" do %> - - - - + <%= render_component "dots-scripts" %> + + <% end %> diff --git a/src/views/library.html.ecr b/src/views/library.html.ecr index 255ccd3..21ec280 100644 --- a/src/views/library.html.ecr +++ b/src/views/library.html.ecr @@ -24,8 +24,7 @@ <% content_for "script" do %> - - + <%= render_component "dots-scripts" %> <% end %> diff --git a/src/views/title.html.ecr b/src/views/title.html.ecr index a66d887..8eb7881 100644 --- a/src/views/title.html.ecr +++ b/src/views/title.html.ecr @@ -117,8 +117,7 @@ <% content_for "script" do %> - - + <%= render_component "dots-scripts" %> From ad940f30d54e4184051bfb499bb377a9febe3e74 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 23 Oct 2020 12:20:23 +0000 Subject: [PATCH 02/13] Update `image_size.cr` to 0.4.0 for better err msg --- shard.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shard.lock b/shard.lock index 529edf5..f2604c9 100644 --- a/shard.lock +++ b/shard.lock @@ -34,7 +34,7 @@ shards: image_size: github: hkalexling/image_size.cr - version: 0.2.0 + version: 0.4.0 kemal: github: kemalcr/kemal From 968c2f4ad5b53866c62deec7a7f37099c514d4da Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 23 Oct 2020 12:29:20 +0000 Subject: [PATCH 03/13] Update DB to save thumbnails --- src/library/types.cr | 10 ++++++++++ src/storage.cr | 29 ++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/library/types.cr b/src/library/types.cr index b51671b..4e83135 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -57,6 +57,16 @@ struct Image def initialize(@data, @mime, @filename, @size) end + + def self.from_db(res : DB::ResultSet) + img = Image.allocate + res.read String + img.data = res.read Bytes + img.filename = res.read String + img.mime = res.read String + img.size = res.read Int32 + img + end end class TitleInfo diff --git a/src/storage.cr b/src/storage.cr index afbba8d..90646be 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -35,9 +35,11 @@ class Storage MainFiber.run do DB.open "sqlite3://#{@path}" do |db| begin - # We create the `ids` table first. even if the uses has an - # early version installed and has the `user` table only, - # we will still be able to create `ids` + db.exec "create table thumbnails " \ + "(id text, data blob, filename text, " \ + "mime text, size integer)" + db.exec "create unique index tn_index on thumbnails (id)" + db.exec "create table ids" \ "(path text, id text, is_title integer)" db.exec "create unique index path_idx on ids (path)" @@ -243,6 +245,27 @@ class Storage end end + def save_thumbnail(id : String, img : Image) + MainFiber.run do + get_db do |db| + db.exec "insert into thumbnails values (?, ?, ?, ?, ?)", id, img.data, + img.filename, img.mime, img.size + end + end + end + + def get_thumbnail(id : String) : Image? + img = nil + MainFiber.run do + get_db do |db| + db.query_one? "select * from thumbnails where id = (?)", id do |res| + img = Image.from_db res + end + end + end + img + end + def close MainFiber.run do unless @db.nil? From 8ac89c420c06fea17067f398129b7b1ded2d6908 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 23 Oct 2020 12:30:29 +0000 Subject: [PATCH 04/13] Add helper methods for thumbnail generation --- src/library/entry.cr | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 589794a..2c6d1bf 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -69,7 +69,7 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg - url = "#{Config.current.base_url}api/page/#{@book.id}/#{@id}/1" + url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}" TitleInfo.new @book.dir do |info| info_url = info.entry_cover_url[@title]? unless info_url.nil? || info_url.empty? @@ -207,4 +207,23 @@ class Entry def started?(username) load_progress(username) > 0 end + + def generate_thumbnail : Image? + return if @err_msg + + img = read_page(1).not_nil! + begin + thumbnail = ImageSize.resize img.data, height: 300 + img.data = thumbnail + Storage.default.save_thumbnail @id, img + rescue e + Logger.warn "Failed to generate thumbnail for entry #{@id}. #{e}" + end + + img + end + + def get_thumbnail : Image? + Storage.default.get_thumbnail @id + end end From 83d96fd2a125380cc2b6978e07528e458213adc9 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 23 Oct 2020 12:30:47 +0000 Subject: [PATCH 05/13] Add the route to serve thumbnails --- src/routes/api.cr | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/routes/api.cr b/src/routes/api.cr index a131a97..5a5a506 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -26,6 +26,28 @@ class APIRouter < Router end end + get "/api/cover/:tid/:eid" do |env| + begin + tid = env.params.url["tid"] + eid = env.params.url["eid"] + + title = @context.library.get_title tid + raise "Title ID `#{tid}` not found" if title.nil? + entry = title.get_entry eid + raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? + + img = entry.get_thumbnail || entry.read_page 1 + raise "Failed to get cover of `#{title.title}/#{entry.title}`" \ + if img.nil? + + send_img env, img + rescue e + @context.error e + env.response.status_code = 500 + e.message + end + end + get "/api/book/:tid" do |env| begin tid = env.params.url["tid"] From 0582b57d60ee1674fb73af4783cf72cb88109917 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 24 Oct 2020 03:50:26 +0000 Subject: [PATCH 06/13] Add config options for optimization tasks --- src/config.cr | 5 +- src/library/library.cr | 3 +- src/views/home.html.ecr | 132 ++++++++++++++++++++-------------------- 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/src/config.cr b/src/config.cr index 60c2e40..1c80542 100644 --- a/src/config.cr +++ b/src/config.cr @@ -11,8 +11,9 @@ class Config property library_path : String = File.expand_path "~/mango/library", home: true property db_path : String = File.expand_path "~/mango/mango.db", home: true - @[YAML::Field(key: "scan_interval_minutes")] - property scan_interval : Int32 = 5 + property scan_interval_minutes : Int32 = 5 + property thumbnail_generation_interval_hours : Int32 = 24 + property db_optimization_interval_hours : Int32 = 24 property log_level : String = "info" property upload_path : String = File.expand_path "~/mango/uploads", home: true diff --git a/src/library/library.cr b/src/library/library.cr index f145342..2ecb495 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,6 +1,7 @@ class Library property dir : String, title_ids : Array(String), scan_interval : Int32, - title_hash : Hash(String, Title) + title_hash : Hash(String, Title), entries_count = 0, + thumbnails_count = 0 use_default diff --git a/src/views/home.html.ecr b/src/views/home.html.ecr index 0700f4f..598709d 100644 --- a/src/views/home.html.ecr +++ b/src/views/home.html.ecr @@ -1,83 +1,83 @@ <%- if new_user && empty_library -%> -
- -

Add your first manga

-

We can't find any files yet. Add some to your library and they'll appear here.

-
-
Current library path
-
<%= Config.current.library_path %>
-
Want to change your library path?
-
Update config.yml located at: <%= Config.current.path %>
-
Can't see your files yet?
-
- You must wait <%= Config.current.scan_interval %> minutes for the library scan to complete - <% if is_admin %> - , or manually re-scan from Admin - <% end %>. -
-
-
+
+ +

Add your first manga

+

We can't find any files yet. Add some to your library and they'll appear here.

+
+
Current library path
+
<%= Config.current.library_path %>
+
Want to change your library path?
+
Update config.yml located at: <%= Config.current.path %>
+
Can't see your files yet?
+
+ You must wait <%= Config.current.scan_interval_minutes %> minutes for the library scan to complete + <% if is_admin %> + , or manually re-scan from Admin + <% end %>. +
+
+
<%- elsif new_user && empty_library == false -%> -
- -

Read your first manga

-

Once you start reading, Mango will remember where you left off - and show your entries here.

- View library -
+
+ +

Read your first manga

+

Once you start reading, Mango will remember where you left off + and show your entries here.

+ View library +
<%- elsif new_user == false && empty_library == false -%> - <%- if continue_reading.empty? && recently_added.empty? -%> -
- -

A self-hosted manga server and reader

- View library -
- <%- end -%> + <%- if continue_reading.empty? && recently_added.empty? -%> +
+ +

A self-hosted manga server and reader

+ View library +
+ <%- end -%> - <%- unless continue_reading.empty? -%> -

Continue Reading

-
- <%- continue_reading.each do |cr| -%> - <% item = cr[:entry] %> - <% progress = cr[:percentage] %> - <%= render_component "card" %> - <%- end -%> -
- <%- end -%> + <%- unless continue_reading.empty? -%> +

Continue Reading

+
+ <%- continue_reading.each do |cr| -%> + <% item = cr[:entry] %> + <% progress = cr[:percentage] %> + <%= render_component "card" %> + <%- end -%> +
+ <%- end -%> - <%- unless start_reading.empty? -%> -

Start Reading

-
- <%- start_reading.each do |t| -%> - <% item = t %> - <% progress = 0.0 %> - <%= render_component "card" %> - <%- end -%> -
- <%- end -%> + <%- unless start_reading.empty? -%> +

Start Reading

+
+ <%- start_reading.each do |t| -%> + <% item = t %> + <% progress = 0.0 %> + <%= render_component "card" %> + <%- end -%> +
+ <%- end -%> - <%- unless recently_added.empty? -%> -

Recently Added

-
- <%- recently_added.each do |ra| -%> - <% item = ra %> - <% progress = ra[:percentage] %> - <%= render_component "card" %> - <%- end -%> -
- <%- end -%> + <%- unless recently_added.empty? -%> +

Recently Added

+
+ <%- recently_added.each do |ra| -%> + <% item = ra %> + <% progress = ra[:percentage] %> + <%= render_component "card" %> + <%- end -%> +
+ <%- end -%> - <%= render_component "entry-modal" %> + <%= render_component "entry-modal" %> <%- end -%> <% content_for "script" do %> - <%= render_component "dots-scripts" %> - - + <%= render_component "dots-scripts" %> + + <% end %> From 44f4959477005c786fe75ed9a8ffa9a81f36b2c4 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 24 Oct 2020 04:12:36 +0000 Subject: [PATCH 07/13] Finish thumbnail generation and DB optimization (#93) --- src/library/entry.cr | 10 ++++-- src/library/library.cr | 82 +++++++++++++++++++++++++++++++++++------- src/storage.cr | 23 ++++++++++++ 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 2c6d1bf..b96681e 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -213,11 +213,17 @@ class Entry img = read_page(1).not_nil! begin - thumbnail = ImageSize.resize img.data, height: 300 + size = ImageSize.get img.data + if size.height > size.width + thumbnail = ImageSize.resize img.data, width: 200 + else + thumbnail = ImageSize.resize img.data, height: 300 + end img.data = thumbnail Storage.default.save_thumbnail @id, img rescue e - Logger.warn "Failed to generate thumbnail for entry #{@id}. #{e}" + Logger.warn "Failed to generate thumbnail for entry " \ + "#{@book.title}/#{@title}. #{e}" end img diff --git a/src/library/library.cr b/src/library/library.cr index 2ecb495..8d9218e 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,7 +1,6 @@ class Library - property dir : String, title_ids : Array(String), scan_interval : Int32, - title_hash : Hash(String, Title), entries_count = 0, - thumbnails_count = 0 + property dir : String, title_ids : Array(String), + title_hash : Hash(String, Title), entries_count = 0, thumbnails_count = 0 use_default @@ -9,20 +8,45 @@ class Library register_mime_types @dir = Config.current.library_path - @scan_interval = Config.current.scan_interval # explicitly initialize @titles to bypass the compiler check. it will # be filled with actual Titles in the `scan` call below @title_ids = [] of String @title_hash = {} of String => Title - return scan if @scan_interval < 1 - spawn do - loop do - start = Time.local - scan - ms = (Time.local - start).total_milliseconds - Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" - sleep @scan_interval * 60 + scan_interval = Config.current.scan_interval_minutes + if scan_interval < 1 + scan + else + spawn do + loop do + start = Time.local + scan + ms = (Time.local - start).total_milliseconds + Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" + sleep scan_interval.minutes + end + end + end + + thumbnail_interval = Config.current.thumbnail_generation_interval_hours + unless thumbnail_interval < 1 + spawn do + loop do + # Wait for scan to complete (in most cases) + sleep 1.minutes + generate_thumbnails + sleep thumbnail_interval.hours + end + end + end + + db_interval = Config.current.db_optimization_interval_hours + unless db_interval < 1 + spawn do + loop do + Storage.default.optimize + sleep db_interval.hours + end end end end @@ -195,4 +219,38 @@ class Library .sample(ENTRIES_IN_HOME_SECTIONS) .shuffle end + + def thumbnail_generation_progress + @thumbnails_count / @entries_count + end + + def generate_thumbnails + Logger.info "Starting thumbnail generation" + entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg + @entries_count = entries.size + @thumbnails_count = 0 + + # Report generation progress regularly + spawn do + loop do + break if thumbnail_generation_progress.to_i == 1 + Logger.debug "Thumbnail generation progress: " \ + "#{(thumbnail_generation_progress * 100).round 1}%" + sleep 30.seconds + end + end + + entries.each do |e| + unless e.get_thumbnail + e.generate_thumbnail + # Sleep after each generation to minimize the impact on disk IO + # and CPU + sleep 0.5.seconds + end + @thumbnails_count += 1 + end + Logger.info "Thumbnail generation finished. " \ + "#{@thumbnails_count}/#{@entries_count} " \ + "thumbnails generated" + end end diff --git a/src/storage.cr b/src/storage.cr index 90646be..05557ef 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -266,6 +266,29 @@ class Storage img end + def optimize + MainFiber.run do + Logger.info "Starting DB optimization" + get_db do |db| + trash_ids = [] of String + db.query "select path, id from ids" do |rs| + rs.each do + path = rs.read String + trash_ids << rs.read String unless File.exists? path + end + end + + # Delete dangling IDs + db.exec "delete from ids where id in " \ + "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" + + # Delete dangling thumbnails + db.exec "delete from thumbnails where id not in (select id from ids)" + end + Logger.info "DB optimization finished" + end + end + def close MainFiber.run do unless @db.nil? From c4e1ffe023e8b564e840a4c37ebea7c63fa6b209 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 25 Oct 2020 05:41:27 +0000 Subject: [PATCH 08/13] Trigger thumbnail generation from the admin page --- public/js/admin.js | 112 ++++++++++++++++++++++++++++----------- src/library/library.cr | 26 ++++++--- src/routes/api.cr | 12 +++++ src/views/admin.html.ecr | 22 +++++--- 4 files changed, 125 insertions(+), 47 deletions(-) diff --git a/public/js/admin.js b/public/js/admin.js index 5c9051d..283d728 100644 --- a/public/js/admin.js +++ b/public/js/admin.js @@ -1,40 +1,88 @@ -let scanning = false; - -const scan = () => { - scanning = true; - $('#scan-status > div').removeAttr('hidden'); - $('#scan-status > span').attr('hidden', ''); - const color = $('#scan').css('color'); - $('#scan').css('color', 'gray'); - $.post(base_url + 'api/admin/scan', (data) => { - const ms = data.milliseconds; - const titles = data.titles; - $('#scan-status > span').text('Scanned ' + titles + ' titles in ' + ms + 'ms'); - $('#scan-status > span').removeAttr('hidden'); - $('#scan').css('color', color); - $('#scan-status > div').attr('hidden', ''); - scanning = false; - }); -} - -String.prototype.capitalize = function() { - return this.charAt(0).toUpperCase() + this.slice(1); -} - $(() => { - $('li').click((e) => { - const url = $(e.currentTarget).attr('data-url'); - if (url) { - $(location).attr('href', url); - } - }); - const setting = loadThemeSetting(); - $('#theme-select').val(setting.capitalize()); - + $('#theme-select').val(capitalize(setting)); $('#theme-select').change((e) => { const newSetting = $(e.currentTarget).val().toLowerCase(); saveThemeSetting(newSetting); setTheme(); }); + + setInterval(getProgress, 5000); }); + +/** + * Capitalize String + * + * @function capitalize + * @param {string} str - The string to be capitalized + * @return {string} The capitalized string + */ +const capitalize = (str) => { + return str.charAt(0).toUpperCase() + str.slice(1); +}; + +/** + * Set an alpine.js property + * + * @function setProp + * @param {string} key - Key of the data property + * @param {*} prop - The data property + */ +const setProp = (key, prop) => { + $('#root').get(0).__x.$data[key] = prop; +}; + +/** + * Get an alpine.js property + * + * @function getProp + * @param {string} key - Key of the data property + * @return {*} The data property + */ +const getProp = (key) => { + return $('#root').get(0).__x.$data[key]; +}; + +/** + * Get the thumbnail generation progress from the API + * + * @function getProgress + */ +const getProgress = () => { + $.get(`${base_url}api/admin/thumbnail_progress`) + .then(data => { + setProp('progress', data.progress); + const generating = data.progress > 0 + setProp('generating', generating); + }); +}; + +/** + * Trigger the thumbnail generation + * + * @function generateThumbnails + */ +const generateThumbnails = () => { + setProp('generating', true); + setProp('progress', 0.0); + $.post(`${base_url}api/admin/generate_thumbnails`); +}; + +/** + * Trigger the scan + * + * @function scan + */ +const scan = () => { + setProp('scanning', true); + setProp('scanMs', -1); + setProp('scanTitles', 0); + $.post(`${base_url}api/admin/scan`) + .then(data => { + setProp('scanMs', data.milliseconds); + setProp('scanTitles', data.titles); + }) + .always(() => { + setProp('scanning', false); + }); +} diff --git a/src/library/library.cr b/src/library/library.cr index 8d9218e..ce2b1be 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -221,10 +221,16 @@ class Library end def thumbnail_generation_progress + return 0 if @entries_count == 0 @thumbnails_count / @entries_count end def generate_thumbnails + if @thumbnails_count > 0 + Logger.debug "Thumbnail generation in progress" + return + end + Logger.info "Starting thumbnail generation" entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg @entries_count = entries.size @@ -233,10 +239,18 @@ class Library # Report generation progress regularly spawn do loop do - break if thumbnail_generation_progress.to_i == 1 - Logger.debug "Thumbnail generation progress: " \ - "#{(thumbnail_generation_progress * 100).round 1}%" - sleep 30.seconds + unless @thumbnails_count == 0 + Logger.debug "Thumbnail generation progress: " \ + "#{(thumbnail_generation_progress * 100).round 1}%" + end + # Generation is completed. We reset the count to 0 to allow subsequent + # calls to the function, and break from the loop to stop the progress + # report fiber + if thumbnail_generation_progress.to_i == 1 + @thumbnails_count = 0 + break + end + sleep 10.seconds end end @@ -249,8 +263,6 @@ class Library end @thumbnails_count += 1 end - Logger.info "Thumbnail generation finished. " \ - "#{@thumbnails_count}/#{@entries_count} " \ - "thumbnails generated" + Logger.info "Thumbnail generation finished" end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 5a5a506..ce2caf5 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -76,6 +76,18 @@ class APIRouter < Router }.to_json end + get "/api/admin/thumbnail_progress" do |env| + send_json env, { + "progress" => Library.default.thumbnail_generation_progress, + }.to_json + end + + post "/api/admin/generate_thumbnails" do |env| + spawn do + Library.default.generate_thumbnails + end + end + post "/api/admin/user/delete/:username" do |env| begin username = env.params.url["username"] diff --git a/src/views/admin.html.ecr b/src/views/admin.html.ecr index 456dff4..a0959bf 100644 --- a/src/views/admin.html.ecr +++ b/src/views/admin.html.ecr @@ -1,11 +1,17 @@ -
    -
  • User Managerment
    • -
  • - Scan Library Files - - - - +
      +
    • User Managerment
      • +
    • + Scan Library Files +
      +
      + +
      +
      • +
    • + Generate Thumbnails +
      + +
    • Theme From 1b35392f9cb68909cdbe5aa813c55b96f2734ca2 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 25 Oct 2020 07:09:21 +0000 Subject: [PATCH 09/13] Remove unnecessary properties --- src/library/library.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index ce2b1be..35d9e89 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,6 +1,6 @@ class Library property dir : String, title_ids : Array(String), - title_hash : Hash(String, Title), entries_count = 0, thumbnails_count = 0 + title_hash : Hash(String, Title) use_default @@ -13,6 +13,9 @@ class Library @title_ids = [] of String @title_hash = {} of String => Title + @entries_count = 0 + @thumbnails_count = 0 + scan_interval = Config.current.scan_interval_minutes if scan_interval < 1 scan From 670e5cdf6ac66576a5b45f24eab4dc017265e998 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 25 Oct 2020 07:09:37 +0000 Subject: [PATCH 10/13] Better logging when optimizing DB --- src/storage.cr | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/storage.cr b/src/storage.cr index 05557ef..592da0e 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -281,11 +281,19 @@ class Storage # Delete dangling IDs db.exec "delete from ids where id in " \ "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" + Logger.debug "#{trash_ids.size} dangling IDs deleted" \ + if trash_ids.size > 0 # Delete dangling thumbnails - db.exec "delete from thumbnails where id not in (select id from ids)" + trash_thumbnails_count = db.query_one "select count(*) from " \ + "thumbnails where id not in " \ + "(select id from ids)", as: Int32 + if trash_thumbnails_count > 0 + db.exec "delete from thumbnails where id not in (select id from ids)" + Logger.info "#{trash_thumbnails_count} dangling thumbnails deleted" + end end - Logger.info "DB optimization finished" + Logger.debug "DB optimization finished" end end From 56d973b99de1defd664e07c92bae5cea2ea6ffa9 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 25 Oct 2020 07:20:33 +0000 Subject: [PATCH 11/13] Get progress when page loads and when post --- public/js/admin.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/public/js/admin.js b/public/js/admin.js index 283d728..4a5246a 100644 --- a/public/js/admin.js +++ b/public/js/admin.js @@ -7,6 +7,7 @@ $(() => { setTheme(); }); + getProgress(); setInterval(getProgress, 5000); }); @@ -65,7 +66,8 @@ const getProgress = () => { const generateThumbnails = () => { setProp('generating', true); setProp('progress', 0.0); - $.post(`${base_url}api/admin/generate_thumbnails`); + $.post(`${base_url}api/admin/generate_thumbnails`) + .then(getProgress); }; /** From 57d8c100f97033e231293233e13fdd476889c1fc Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 25 Oct 2020 07:22:38 +0000 Subject: [PATCH 12/13] Bump version to v0.15.0 --- README.md | 2 +- shard.yml | 2 +- src/mango.cr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3f8e857..ebccde8 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.14.0 + Mango - Manga Server and Web Reader. Version 0.15.0 Usage: diff --git a/shard.yml b/shard.yml index 02438e5..d187ca4 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.14.0 +version: 0.15.0 authors: - Alex Ling diff --git a/src/mango.cr b/src/mango.cr index 2f9ddd8..16a334e 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -7,7 +7,7 @@ require "option_parser" require "clim" require "./plugin/*" -MANGO_VERSION = "0.14.0" +MANGO_VERSION = "0.15.0" # From http://www.network-science.de/ascii/ BANNER = %{ From 54eb041fe45167b97e6b3749cc8e1fbd126a1281 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 25 Oct 2020 07:29:19 +0000 Subject: [PATCH 13/13] Update README --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ebccde8..108e286 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Mango is a self-hosted manga server and reader. Its features include - Supported formats: `.cbz`, `.zip`, `.cbr` and `.rar` - Supports nested folders in library - Automatically stores reading progress +- Thumbnail generation - Built-in [MangaDex](https://mangadex.org/) downloader - Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites - The web reader is responsive and works well on mobile, so there is no need for a mobile app @@ -76,22 +77,27 @@ The default config file location is `~/.config/mango/config.yml`. It might be di --- port: 9000 base_url: / -library_path: ~/mango/library -db_path: ~/mango/mango.db +session_secret: mango-session-secret +library_path: /home/alex_ling/mango/library +db_path: /home/alex_ling/mango/mango.db scan_interval_minutes: 5 +thumbnail_generation_interval_hours: 24 +db_optimization_interval_hours: 24 log_level: info -upload_path: ~/mango/uploads +upload_path: /home/alex_ling/mango/uploads +plugin_path: /home/alex_ling/mango/plugins +download_timeout_seconds: 30 mangadex: base_url: https://mangadex.org api_url: https://mangadex.org/api download_wait_seconds: 5 download_retries: 4 - download_queue_db_path: ~/mango/queue.db + download_queue_db_path: /home/alex_ling/mango/queue.db chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' manga_rename_rule: '{title}' ``` -- `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan +- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging ### Library Structure