V : 0.12.6

Changelog.md

@@ -1,5 +1,6 @@
 ### Changelog :
 
+0.12.6 : optimisation and display : a chapter that is a todo will not be downloaded twice during an update if it is not in the traces satabase + a fex errors are now esier to read and understand<br />
 0.12.5 : stability fix : fixed a wrong path for the covers ( they were not displayed correctly on the local website ) + corrected a few errors with the todo elements and the related display<br />
 0.12.4 : stability fix : corrected an error with the instruction parsing<br />
 0.12.3 : stability fix : fixed an issue with the instrction parser + an issue where MangaScrap would download 0 pages for a chapter<br />

UnitTests.rb

@@ -24,113 +24,33 @@
 ##################################################################################
 ##################################################################################
 
-puts 'needs to be updated'
-exit 42
 
-verbose = false
-if ARGV.size != 0
-  if ARGV[0] == 'verbose'
-    verbose = true
-  else
-    puts 'unknown argument : "' + ARGV[0] + '"'
-    exit 4
+# get arguments from ARGV
+verbose = true
+high_verbose = false
+
+# used to display an error if the file could not be loaded
+module Unit_tests_API
+  def self.require_file(file)
+    begin
+      require_relative file
+    rescue LoadError => e
+      puts 'Error while loading ' + file + '.rb'
+      puts 'Message is : ' + e.message
+      puts 'Error class is : ' + e.message
+      return false
+    end
+    true
   end
 end
 
-def UnitTest_require_file(file)
-  begin
-    require_relative file
-  rescue LoadError => e
-    puts 'Error while loading ' + file + '. Message is : ' + e.message
-    return false
-  end
-  true
-end
-
-def UnitTest_load_gem(gem)
-  begin
-    require gem
-  rescue LoadError => e
-    puts 'Error while loading the ' + gem + ' gem. Message is : ' + e.message
-    return false
-  end
-  true
-end
-
-
-puts '====> Unit tests'
-puts ''
-
-puts '====> 1 - requiring init file'
-begin
-  if verbose
-    puts 'requiring sources/init.rb'
-  end
-  require_relative 'sources/init'
-rescue LoadError => e
-  puts 'Error while loading the init file (it should have been at : ' + Dir.pwd + '/sources/init.rb)'
-  puts 'aborting'
+puts 'starting unit tests' if verbose
+puts '' if verbose
+if !Unit_tests_API::require_file('sources/unitTests/Init') ||
+!Unit_tests_API::require_file('sources/unitTests/Database') ||
+!Unit_tests_API::require_file('sources/unitTests/Web')
   exit 1
 end
-puts 'ok'
-puts ''
-
-puts '====> 2 requiring all gems'
-failure = false
-Init::get_gem_list.each do |gem|
-  if verbose
-    puts 'requiring ' + gem
-  end
-  failure = !(UnitTest_load_gem gem || failure)
-end
-if failure
-  puts 'aborting'
-  exit 2
-end
-puts 'ok'
-puts ''
-
-puts '====> 3 requiring all MangaScrap files'
-failure = false
-Init::get_file_list.each do |file|
-  if verbose
-    puts 'requiring sources/' + file
-  end
-  failure = !(UnitTest_require_file('sources/' + file) || failure)
-end
-puts 'ok'
-puts ''
-
-Struct.new('Arg', :name, :sub_args, :nb_args, :does_not_need_args?)
-Struct.new('Sub_arg', :name, :nb_args)
-Struct.new('Updated', :name, :downloaded)
-Struct.new('Query_arg', :name, :arg_type, :sql_column, :sub_string)
-Struct.new('HTML_data', :volume, :chapter, :date, :href, :nb_pages, :file_name)
-
-puts '====> 4 getting the database'
-begin
-  params = Manga_database.instance.get_manga_list
-  pp params
-rescue Exception => e
-  puts 'Error'
-  pp e.class.to_s
-  pp e.message
-  exit 5
-end
-puts 'ok'
-puts ''
-
-puts '====> 5 getting the params'
-begin
-  params = Params.instance.get_params
-  pp params
-rescue Exception => e
-  puts 'Error'
-  pp e.class.to_s
-  pp e.message
-  exit 5
-end
-puts 'ok'
-puts ''
 
-puts '====> all good'
+init = UT_Init.new(verbose, high_verbose)
+init.run

sources/DB/Manga_data.rb

@@ -15,6 +15,16 @@ class Manga_data
   attr_reader :to_complete, :link, :name, :site_dir, :id, :site, :data, :status, :in_db
 
   private
+  # places all of the website related information in the @sites array
+  def init_sites
+    @sites << Struct::Website.new('http://mangafox.me/', %w(http://mangafox.me mangafox.me mangafox),
+                                  'mangafox', 'manga/', Download_Mangafox)
+    @sites << Struct::Website.new('http://www.mangareader.net/', %w(http://www.mangareader.net www.mangareader.net mangareader.net mangareader),
+                                  'mangareader', '', Download_Mangareader_Pandamanga)
+    @sites << Struct::Website.new('http://www.mangapanda.com/', %w(http://www.mangapanda.com www.mangapanda.com mangapanda.com mangapanda),
+                                  'pandamanga', '', Download_Mangareader_Pandamanga)
+  end
+
   # ensures that the link starts with http or https
   def link_correction
     unless @link.start_with?('http://', 'https://')
@@ -236,6 +246,8 @@ def initialize(id, name, site, link, data)
     @to_complete = ''
     @in_db = @status
     @download_class = nil
+    @sites = []
+    init_sites
     if @in_db
       is_site_compatible?(false) # function used here to get @site_dir and @to_complete values
     end

sources/Download/base_downloader.rb

@@ -87,6 +87,29 @@ def get_chapter_from_link(link, prep_display, nb_pages_xpath, add_to_link)
     true
   end
 
+  # checks witch are the chapters that have not been downloaded ( they are not in the traces database )
+  # the method first gets all of the links from the database and then compares with the links of the website
+  def missing_chapters
+    traces = @db.get_trace(@manga_data)
+    if @todo.size != 0
+      @todo = @todo.reject{|e| e[:page] != -1}
+    end
+    i = 0
+    all_data = []
+    @links.each do |link|
+      all_data << Struct::Data.new(*@manga_data.extract_values_from_link(link), link)
+    end
+    Utils_misc::sort_chapter_list(all_data).each do |data|
+      if traces.count{|_id, _manga_name, vol_value, chap_value| vol_value == data[:volume] && chap_value == data[:chapter]} == 0 &&
+      @todo.count{|e| e[:chapter] == data[:chapter]} == 0
+        prep_display = " ( link #{i + 1} / #{@links.size} )"
+        chapter_link(data[:link], prep_display)
+      end
+      i += 1
+    end
+    @aff.dump
+  end
+
   public
   # takes the data array [volume, chapter, page] and casts it into a string
   def values_to_string(volume, chapter, page)
@@ -95,19 +118,19 @@ def values_to_string(volume, chapter, page)
     page_string = ''
     case volume
       when -2
-        volume_string = 'volume TBD'
+        volume_string = 'volume TBD '
       when -3
-        volume_string = 'volume NA'
+        volume_string = 'volume NA '
       when -4
-        volume_string = 'volume ANT'
+        volume_string = 'volume ANT '
       else
         if volume >= 0
-          volume_string = "volume #{volume}"
+          volume_string = "volume #{volume} "
         end
     end
-    chapter_string = "chapter #{chapter} " if chapter != nil || chapter == -1
+    chapter_string = "chapter #{chapter}" if chapter != nil || chapter == -1
     page_string = "page #{page}" if page != nil || page == -1
-    volume_string + ' ' + chapter_string + ' ' + page_string
+    volume_string + chapter_string + ' ' + page_string
   end
 
   def data_to_string(data)
@@ -132,10 +155,10 @@ def chapter_link(link, prep_display = '')
   # elem[0] = id of line in database ; elem[1] = id of manga in database
   # elem[2] = volume value ; elem[3] = chapter value ; elem[4] = page value
   def todo
-    todo = @db.get_todo(@manga_data)
-    if todo.size != 0
+    @todo = @db.get_todo(@manga_data)
+    if @todo.size != 0
       @aff.prepare_todo
-      Utils_misc::sort_chapter_list(todo, true).each do |elem|
+      Utils_misc::sort_chapter_list(@todo, true).each do |elem|
         if elem[:page] != -1 # if page
           data = [] << elem[:volume] << elem[:chapter] << elem[:page]
           if page_link(link_generator(elem[:volume], elem[:chapter], elem[:page]), data)
@@ -152,38 +175,6 @@ def todo
     @downloaded_a_page
   end
 
-  # checks witch are the chapters that have not been downloaded ( they are not in the traces database )
-  # the method first gets all of the links from the database and then compares with the links of the website
-  def missing_chapters
-=begin
-    traces = @db.get_trace(@manga_data)
-    i = 0
-    @links.each do |link|
-      data = @manga_data.extract_values_from_link(link)
-      if traces.count{|_id, _manga_name, vol_value, chap_value| vol_value == data[0] && chap_value == data[1]} == 0
-        prep_display = " ( link #{i + 1} / #{@links.size} )"
-        chapter_link(link, prep_display)
-      end
-      i += 1
-    end
-    @aff.dump
-=end
-    traces = @db.get_trace(@manga_data)
-    i = 0
-    all_data = []
-    @links.each do |link|
-      all_data << Struct::Data.new(*@manga_data.extract_values_from_link(link), link)
-    end
-    Utils_misc::sort_chapter_list(all_data).each do |data|
-      if traces.count{|_id, _manga_name, vol_value, chap_value| vol_value == data[:volume] && chap_value == data[:chapter]} == 0
-        prep_display = " ( link #{i + 1} / #{@links.size} )"
-        chapter_link(data[:link], prep_display)
-      end
-      i += 1
-    end
-    @aff.dump
-  end
-
   # the update method will first check for _todo elements to download then check for missing chapters
   # it only calls the data method if a page was downloaded
   def update
@@ -211,6 +202,7 @@ def data
   # @db => the database instance todo : delete the variable
   # @aff => the attached DownloadDisplay class
   # @doc => the first page todo : regarder si le Manga_data ne peut pas faire ça au moment de son check_link
+  # @_todo => an arry containing all of the _todo elements. Used to avoid downloading the same chapter twice when updating
   def initialize(manga, download_data = true)
     @manga_data = manga
     @downloaded_a_page = false
@@ -221,6 +213,7 @@ def initialize(manga, download_data = true)
     # doc is the variable that stores the raw data of the manga's page
     @aff = DownloadDisplay.new(manga.site_dir, manga.name)
     @doc = Utils_connection::get_page(manga.link, true)
+    @todo = []
     if @doc == nil
       raise 'failed to get manga ' + manga.name + "'s chapter index"
     end

sources/Download/mangafox.rb

@@ -52,6 +52,7 @@ def page_link(link, data)
   def chapter_link(link, prep_display = '')
 #    get_chapter_from_link(link, prep_display, '//div[@class="l"]', '.html')
     data = @manga_data.extract_values_from_link(link)
+    @aff.prepare_chapter("downloading #{data_to_string(data)} of #{@manga_data.name}" + prep_display)
     if data[0] == -42
       @aff.unmanaged_link(link)
       false
@@ -65,7 +66,6 @@ def chapter_link(link, prep_display = '')
     rescue RuntimeError
       return link_err(data, true, 'R')
     end
-    @aff.prepare_chapter("downloading #{data_to_string(data)} of #{@manga_data.name}" + prep_display)
     number_of_pages = page.xpath('//div[@class="l"]').text.split.last.to_i
     if number_of_pages == 0
       return link_err(data, true, '?')

sources/api/mangas.rb

@@ -135,7 +135,7 @@ def self.clear(mangas)
   # delete_files = bool, if true, the files of the manga will also be deleted
   def self.delete(manga_list, delete_files = true)
     if Utils_user_input::require_confirmation('you are about to ' + 'delete'.red + ' the following element(s) from the database' + ((delete_files) ?
-      ' and '.yellow + 'all of the downloaded pages' :
+      ' AND '.yellow + 'all of the downloaded pages' :
       ' but ' + 'not'.yellow + ' the downloaded files' ), manga_list)
       db = Manga_database.instance
       path = Params.instance.download[:manga_path]

sources/html/html.rb

@@ -15,6 +15,8 @@ def html_get_data(site, updated_recently = false)
       end
       source_buffer = '      <img src="./mangas/' + manga.name + '.jpg' + '">'
       if @html_params[:nsfw_enabled]
+        ret << source_buffer
+      else
         manga_genres = manga.data[9].split(', ')
         if (@html_params[:nsfw_categories].split(', ') & manga_genres).empty?
           ret << source_buffer
@@ -23,8 +25,6 @@ def html_get_data(site, updated_recently = false)
           # warning : sites must be managed
           ret << '      <img src="' + Dir.home + '/.MangaScrap/pictures/logos/mangafox.png">'
         end
-      else
-        ret << source_buffer
       end
       ret << '      <p>' + Utils_file::description_manipulation(manga.name.gsub('_', ' '), 25, 3).gsub('\n', '<br />') + '</p>'
       ret << '    </a>'

sources/html/html_manga.rb

@@ -94,7 +94,7 @@ def html_generate_chapters(manga_data)
   def chapter_list_to_a_list(template)
     ret = ''
     buff = ''
-    @traces = Utils_misc::sort_chapter_list(@traces)
+    @traces = Utils_misc::sort_chapter_list(@traces).reverse
     @traces.each do |chapter|
   #      pp chapter
   #      exit 42

sources/init.rb

@@ -36,10 +36,11 @@ def self.get_file_list
   Download/mangafox
   Download/mangareader_mangapanda
   DownloadDisplay
-  instructions/Parsers/Instruction_parser
-  instructions/Parsers/Data_parser
+  instructions/Parsers/exec/Instruction_parser
+  instructions/Parsers/exec/Data_parser
   instructions/Parsers/Query_Parser
   instructions/Parsers/File_parser
+  instructions/Parsers/Site_parser
   instructions/query
   instructions/Instructions_exec
   instructions/Manga_data_filter
@@ -82,9 +83,7 @@ def self.load_relative_files
     end
   end
 
-  def self.initialize_mangascrap
-    load_all_gems
-    load_relative_files
+  def self.init_structures
     Struct.new('Arg', :name, :sub_args, :nb_args, :does_not_need_args?)
     Struct.new('Sub_arg', :name, :nb_args)
     Struct.new('Updated', :name, :downloaded)
@@ -93,6 +92,13 @@ def self.initialize_mangascrap
     Struct.new('Data', :volume, :chapter, :page, :link)
     Struct.new('Todo_value', :id, :manga_id, :volume, :chapter, :page, :date)
     Struct.new('Param_value', :string, :id, :type, :value, :class, :min_value, :max_value)
+    Struct.new('Website', :link, :aliases, :dir, :to_complete, :class)
+  end
+
+  def self.initialize_mangascrap
+    load_all_gems
+    load_relative_files
+    init_structures
     begin
       Utils_file::dir_create(Dir.home + '/.MangaScrap/db')
     rescue StandardError => error

sources/instructions/Instructions_exec.rb

@@ -193,6 +193,12 @@ def get_data_parser(instruction)
         puts 'ignoring file instruction'.yellow
       end
     end
+    parser.on('site', 1) do |args|
+      e = Site_parser.new(args[0], @data_to_prepare)
+      unless e.good
+        puts 'ignoring site instruction'.yellow
+      end
+    end
     parser.on('query', 1) do |args| # <======================================================== currently a placeholder
       puts 'query is currently a placeholder'
 #      query = Query_Manager.new