diff --git a/Changelog.md b/Changelog.md index f59100a..1ade8eb 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ ### Changelog : +0.12.0 : huge optimisation : download is close to twice as fast + new management of the parameters + more understandable errors
+ +###### Warning : the params database is not compatible
Please look at the file migration/0.11.x_to_0.12.x.txt
+ 0.11.5 : lots of refactoring + bug fixes + optimisation ( the updates are now faster ) + updated the API + updated the help instruction + added management of 2 new sites : mangareader and pandamanga
0.11.4 : added an API and instructions on how to use it, it will be used for the GUI
0.11.3 : small bug fixes + implemented scripts to allow faster migrations / updates
diff --git a/MangaScrap.rb b/MangaScrap.rb index 77521b1..fe4faf1 100755 --- a/MangaScrap.rb +++ b/MangaScrap.rb @@ -27,6 +27,8 @@ =end +$0='MangaScrap' + require_relative 'sources/init' begin diff --git a/UnitTests.rb b/UnitTests.rb index 3a0e25f..3398572 100755 --- a/UnitTests.rb +++ b/UnitTests.rb @@ -10,7 +10,7 @@ 0 : good 1 : file load error 2 : gem load error -3 : bad Ruby version +3 : bad Ruby versionw 4 : bad argument given to UnitTests.rb 5 : error on class require @@ -24,6 +24,9 @@ ################################################################################## ################################################################################## +puts 'needs to be updated' +exit 42 + verbose = false if ARGV.size != 0 if ARGV[0] == 'verbose' diff --git a/migration/0.10.x_to_0.11.x.sh b/migration/0.10.x_to_0.11.x.sh old mode 100644 new mode 100755 index f968179..2d098ea --- a/migration/0.10.x_to_0.11.x.sh +++ b/migration/0.10.x_to_0.11.x.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash echo "ALTER TABLE manga_trace ADD date VARCHAR(32); ALTER TABLE manga_trace ADD nb_pages INTEGER; ALTER TABLE manga_todo ADD date VARCHAR(32); diff --git a/migration/0.11.x_to_0.12.x.sh b/migration/0.11.x_to_0.12.x.sh new file mode 100755 index 0000000..61c50da --- /dev/null +++ b/migration/0.11.x_to_0.12.x.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +gem install typhoeus diff --git a/migration/0.11.x_to_0.12.x.txt b/migration/0.11.x_to_0.12.x.txt new file mode 100644 index 0000000..fccffca --- /dev/null +++ b/migration/0.11.x_to_0.12.x.txt @@ -0,0 +1,6 @@ +1. note all of your params + +2. install tythoeus +gem install typhoeus + +3. input all of your params into MangaScrap using the "param set" instruction diff --git a/migration/0.9.X_to_0.10.x.sh b/migration/0.9.X_to_0.10.x.sh old mode 100644 new mode 100755 index aefddab..c11b0d5 --- a/migration/0.9.X_to_0.10.x.sh +++ b/migration/0.9.X_to_0.10.x.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # 1 - Delete your params DB ( please note all important information such as mangapath before doing so if you have changed any parameter ) rm ~/.MangaScrap/db/params.db diff --git a/sources/DB/Manga_data.rb b/sources/DB/Manga_data.rb index 9c575aa..daf815a 100644 --- a/sources/DB/Manga_data.rb +++ b/sources/DB/Manga_data.rb @@ -30,8 +30,10 @@ def check_link(display) @link += '/' end end - if Utils_connection::redirection_detection(@link) - puts 'Warning :'.yellow + ' could not connect to ' + @link.yellow if display + begin + Utils_connection::get_page(@link, true) + rescue RuntimeError + puts 'Warning :'.yellow + ' could not connect to ' + @link.yellow if display return false end true @@ -94,7 +96,9 @@ def is_site_compatible?(display) public # returns an array of the sites that MangaScrap currently manages def self.get_compatible_sites - %w(http://mangafox.me/ http://www.mangareader.net/ http://www.mangapanda.com/) + %w(http://mangafox.me/ +http://www.mangareader.net/ +http://www.mangapanda.com/) end def self.get_dir_from_site(site) @@ -149,9 +153,13 @@ def get_download_class(download_data = true) end rescue => e puts 'Exception while trying to get '.red + @name.yellow + puts 'exception is : ' + e.class.to_s puts 'reason is : '.yellow + e.message return nil - end + rescue ArgumentError => e + puts 'Exception while trying to get '.red + @name.yellow + Utils_errors::critical_error('Argument error ( something is wrong with the code', e) + end end @download_class end @@ -185,6 +193,7 @@ def resolve(connect, display) if !ret && connect # if it is not in the database and a connection is required, then it is good if check_link(display) @status = true + return true end return false @@ -225,7 +234,7 @@ def initialize(id, name, site, link, data) @in_db = @status @download_class = nil if @in_db - is_site_compatible?(false) # function used here to get @site_dir and @ to_complete + is_site_compatible?(false) # function used here to get @site_dir and @to_complete values end end end diff --git a/sources/DB/Manga_database.rb b/sources/DB/Manga_database.rb index f5ea108..f7baaaf 100644 --- a/sources/DB/Manga_database.rb +++ b/sources/DB/Manga_database.rb @@ -69,13 +69,15 @@ def add_todo(manga_data, volume_value, chapter_value, page_nb) exit 2 end todo = get_todo(manga_data) - insert = [manga_data.id, volume_value, chapter_value, page_nb, now] + insert = [manga_data.id, volume_value, chapter_value, page_nb] todo.each do |elem| + elem.pop elem.shift if elem == insert return false end end + insert << now Utils_database::db_exec('INSERT INTO manga_todo VALUES (NULL, ?, ?, ?, ?, ?)', 'could not add todo for ' + manga_data.name, @db, insert) true end @@ -102,6 +104,7 @@ def add_trace(manga_data, volume_value, chapter_value, nb_pages) trace = get_trace(manga_data) insert = [manga_data.id, volume_value, chapter_value, now, nb_pages] found = false + # todo : pour les comparaisons de traces, il ne faut surtout pas comparer les dates trace.each do |elem| # checking if the trace does not already exist in database to avoid duplicates elem.shift # the first element is shift as it contains the id of the trace if elem == insert diff --git a/sources/DB/Params.rb b/sources/DB/Params.rb index ea9686a..de5163b 100644 --- a/sources/DB/Params.rb +++ b/sources/DB/Params.rb @@ -1,213 +1,57 @@ -$default_manga_path = Dir.home + '/Documents/mangas/' - class Params include Singleton - private - def write_output(message) - if @display - puts message - end - end - - def exec_reset - Utils_database::db_exec('UPDATE params SET manga_path = ?, between_sleep = ?, failure_sleep = ?, nb_tries_on_fail = ?,error_sleep = ?, delete_diff = ?, catch_exception = ?, generate_html = ?, html_nsfw = ?, html_nsfw_data = ?, color_text = ? WHERE Id = 1', - 'could not set / reset parameters', @db, [Params::get_default_params]) - end - - def set_param_value(param, value) - Utils_database::db_exec("UPDATE params SET #{param} = ? WHERE Id = 1", 'could not update ' + param, @db, value) - end - -# this function should never be called, when called there is an error in the code - def param_critical_error(param, func) - write_output 'Error : the ' + param + ' ended in the wrong function (' + func + ')' - write_output 'This is a Mangascrap error, please report it' - write_output '( unless you caused it by altering the code )' - false - end - -# params set for numbers - def param_check_nb(param, display, value, min_value) - if value < min_value - write_output "the '" + display[0] + "' value cannot be < " + min_value.to_s + 'for ' + param - return false - end - set_param_value(display[1], value) - write_output 'updated ' + display[0] + ' param to ' + value.to_s - true - end - -# params set for booleans - def param_check_bool(param, display, value) - if value != 'true' && value != 'false' - write_output "argument must be 'true' or 'false' for " + param - return false - end - set_param_value(display[1], value) - write_output 'updated ' + display[0] + ' to ' + value - true - end + attr_reader :download, :html, :misc, :threads -# params set for strings - def param_check_string(param, display, value) - case param - when 'mp' - if value[0, 1] != '~' && value[0, 1] != '/' - write_output 'cannot create local directory' - return false - end - begin - Utils_file::dir_create(value) - rescue StandardError => error - write_output 'could not create requested path' - write_output "error message is : '" + error.message + "'" - return false + public + # used to check if the user really wants to reset + def param_reset (require_confirmation = true) + margin = 25 + check_user_input = false + if require_confirmation + prep = "You are about to reset the following params :\n" + @param_classes.each do |param_class| + prep += "\n#{param_class.db_name.blue}\n" + param_class.params_list.each do |param| + prep += param[:string] + ((param[:string].size > margin) ? ' ' : ' ' * (margin - param[:string].size)) + prep += param[:id].red + ((param[:id].size == 3) ? '' : ' ')+ ' => ' + param[:value].to_s.green + "\n" end - set_param_value(display[1], value) - when 'nd' - set_param_value(display[1], value) - write_output 'updated nsfw genres to : \n\n' + value.split(', ').join('\n') + '\n\n' - else - return param_critical_error(param, 'param_check_string') + end + prep += "\n\n" + check_user_input = Utils_user_input::require_confirmation(prep) end - write_output('updated ' + display[0] + ' to ' + value) - true - end - - public - # returns the default values of the params - def self.get_default_params - ret = [] << $default_manga_path << 0.1 << 0.1 << 20 << 30 << 'true' << 'true' << 'true' << 'true' << 'Ecchi, Mature, Smut, Adult' << 'true' - ret - end - -# returns all of the parameters in an array of [name, id, value] -# used for gui ( among others ) - def get_params_list - ret = [] - ret << ['manga path', 'mp', @params[1]] - ret << ['between sleep', 'bs', @params[2]] - ret << ['failure sleep', 'fs', @params[3]] - ret << ['nb tries', 'nb', @params[4]] - ret << ['error sleep', 'es', @params[5]] - ret << ['delete diff', 'dd', @params[6]] - ret << ['catch exception', 'ce', @params[7]] - ret << ['generate html', 'gh', @params[8]] - ret << ['html', 'hn', @params[9]] - ret << ['nsfw data', 'nd', @params[10]] - ret << ['color text', 'ct', @params[11]] - ret - end - - # reading the file, adding DB values and setting the colors ( when needed ) - # used to display the parameters and their setting nicely on the terminal - def param_list_file - params = Params.instance.get_params - template = File.open('sources/templates/text/params.txt').read - template = template.gsub('#{params[1]}', params[1].green) - template = template.gsub('#{params[2]}', params[2].to_s.green) - template = template.gsub('#{params[3]}', params[3].to_s.green) - template = template.gsub('#{params[4]}', params[4].to_s.green) - template = template.gsub('#{params[5]}', params[5].to_s.green) - template = template.gsub('#{params[6]}', params[6].green) - template = template.gsub('#{params[7]}', params[7].green) - template = template.gsub('#{params[8]}', params[8].green) - template = template.gsub('#{params[9]}', params[9].green) - template = template.gsub('#{params[10]}', params[10].green) - template = template.gsub('#{params[11]}', params[11].green) - template = template.gsub('[data]', '====> data :'.blue) - template = template.gsub('[internet]', '====> internet :'.blue) - template = template.gsub('[html]', '====> html :'.blue) - template = template.gsub('[internal]', "====> MangaScrap's internal functioning :".blue) - template = template.gsub('[term output]', '====> MangaScrap terminal output :'.blue) - template = template.gsub('(mp)', '(mp)'.red) - template = template.gsub('(dd)', '(dd)'.red) - template = template.gsub('(bs)', '(bs)'.red) - template = template.gsub('(fs)', '(fs)'.red) - template = template.gsub('(nb)', '(nb)'.red) - template = template.gsub('(es)', '(es)'.red) - template = template.gsub('(ce)', '(ce)'.red) - template = template.gsub('(gh)', '(gh)'.red) - template = template.gsub('(hn)', '(hn)'.red) - template = template.gsub('(nd)', '(nd)'.red) - template.gsub('(ct)', '(ct)'.red) - end - - # returns the params as they are in the database ( a big array of values ) - def get_params - ret = Utils_database::db_exec('SELECT * FROM params', 'could not get params', @db) - if ret.size == 0 - Utils_errors::critical_error('could not get the params as the table is empty') + if check_user_input || !require_confirmation + @param_classes.each do |param_class| + param_class.params_reset + end + puts 'params where set to the displayed values' + else + puts 'did not reset params' end - ret[0] end - # allows you to set the parameters - # id = the 2 letter string that identifies a parameter - # value = value on witch you wish to set the parameter. WARNING : booleans are sent as strings 'true' / 'false' - # display = writes or not on the terminal in case of failure - def param_set (id, value, display = true) - @display = display - case id - when 'dd' - param_check_bool(id, ['delete diff', 'delete_diff'], value) - when 'ce' - param_check_bool(id, ['catch exception', 'catch_execption'], value) - when 'mp' - param_check_string(id, ['manga path', 'manga_path'], value) - when 'bs' - param_check_nb(id, ['between sleep', 'between_sleep'], value.to_f, 0.1) - when 'fs' - param_check_nb(id, ['failure sleep', 'failure_sleep'], value.to_f, 0.1) - when 'es' - param_check_nb(id, ['error sleep', 'error_sleep'], value.to_f, 0.5) - when 'nb' - param_check_nb(id, ['number of tries', 'nb_tries_on_fail'], value.to_i, 1) - when 'gh' - param_check_bool(id, ['generate html', 'generate_html'], value) - when 'hn' - param_check_bool(id, ['html nsfw', 'html_nsfw'], value) - when 'nd' - param_check_string(id, ['nsfw data', 'html_nsfw_data'], value) - when 'ct' - param_check_bool(id, ['color text', 'color_text'], value) - else - if @display - puts 'error, unknown parameter id : ' + id - puts '--help for help' - end - return false + def display_params + @param_classes.each do |param_class| + puts param_class.params_display + puts '' end - true end - # used to check if the user really wants to reset - def param_reset (require_confirmation = true) - if require_confirmation - default = Params::get_default_params - puts '' - puts 'WARNING ! You are about to reset your parameters !' - puts 'the parameters will be set to :' - puts 'manga path = ' + default[0].yellow - puts 'between sleep = ' + default[1].to_s.yellow - puts 'failure sleep = ' + default[2].to_s.yellow - puts 'number of tries = ' + default[3].to_s.yellow - puts 'error sleep = ' + default[4].to_s.yellow - puts 'delete diff = ' + default[5].yellow - puts 'catch exception = ' + default[6].yellow - puts 'generate html = ' + default[7].yellow - puts 'html nsfw = ' + default[8].yellow - puts 'html nsfw data = ' + default[9].yellow - puts 'color text = ' + default[10].yellow - puts '' - if require_confirmation - exec_reset + def param_set (id, value, display = true) + ret = @params.select{|e| e[:id] == id}[0] + if ret == nil + if display + puts 'Error'.red + ' : unknown parameter id : ' + id + puts './MangaScrap help'.yellow + ' for help' end else - exec_reset + ret[:class].param_set(id, value, display) end end + def init_all_databases + param_reset(false) + end + def introduction puts '' puts '' @@ -216,9 +60,9 @@ def introduction puts 'Please make sure those values suit you' puts '' puts 'To check the parameter values, execute :' - puts './MangaScrap.rb params list' + puts './MangaScrap.rb params list'.yellow puts 'For any help, execute :' - puts './MangaScrap.rb help' + puts './MangaScrap.rb help'.yellow puts '' puts 'Please note that by default, the directory in witch MangaScrap will place all the downloaded mangas is :' puts $default_manga_path.yellow @@ -236,31 +80,17 @@ def initialize unless Dir.exists?(Dir.home + '/.MangaScrap/db/') Utils_file::dir_create(Dir.home + '/.MangaScrap/db/') end - begin - @db = SQLite3::Database.new(Dir.home + '/.MangaScrap/db/params.db' ) - @db.execute 'CREATE TABLE IF NOT EXISTS params ( - Id INTEGER PRIMARY KEY AUTOINCREMENT, - manga_path TEXT, - between_sleep FLOAT, - failure_sleep FLOAT, - nb_tries_on_fail INT, - error_sleep FLOAT, - delete_diff VARCHAR(5), - catch_exception VARCHAR(5), - generate_html VARCHAR(5), - html_nsfw VARCHAR(5), - html_nsfw_data VARCHAR(5), - color_text VARCHAR(5))' - ret = @db.execute 'SELECT * FROM params' - if ret.size == 0 - introduction - default = Params::get_default_params - @db.execute "INSERT INTO params values (NULL, '#{default[0]}', #{default[1]}, #{default[2]}, #{default[3]}, -#{default[4]}, '#{default[5]}', '#{default[6]}', '#{default[7]}', '#{default[8]}', '#{default[9]}', '#{default[10]}')" - end - rescue SQLite3::Exception => e - Utils_errors::critical_error('exception while retrieving params / initializing params', e) + unless File.exist?(Dir.home + '/.MangaScrap/db/params.db') + introduction + end + @threads = Params_threads.new + @download = Params_download.new + @html = Params_HTML.new + @misc = Params_misc.new + @param_classes = [] << @download << @html << @misc << @threads + @params = [] + @param_classes.each do |param| + @params += param.params_list end - @params = get_params end end diff --git a/sources/DB/sub_data/History.rb b/sources/DB/sub_data/History.rb new file mode 100644 index 0000000..c40b103 --- /dev/null +++ b/sources/DB/sub_data/History.rb @@ -0,0 +1,18 @@ +# only directly used by the Params class, must NOT be directly called elsewhere +# stores all of the instructions given to the scrapper +class Params_History + def initialise + unless Dir.exists?(Dir.home + '/.MangaScrap/db/') + Utils_file::dir_create(Dir.home + '/.MangaScrap/db/') + end + @db = SQLite3::Database.new(Dir.home + '/.MangaScrap/db/data.db') + @db.execute 'CREATE TABLE IF NOT EXISTS History_conf ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + enable VARCHAR(1), + history_size INT)' + @db.execute 'CREATE TABLE IF NOT EXISTS History ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + date VARCHAR(32), + instruction TEXT)' + end +end diff --git a/sources/DB/sub_data/Macro.rb b/sources/DB/sub_data/Macro.rb new file mode 100644 index 0000000..b26031d --- /dev/null +++ b/sources/DB/sub_data/Macro.rb @@ -0,0 +1,14 @@ +# only directly used by the Params class, must NOT be directly called elsewhere +# stores all of the macros the scrapper can execute +class Macro + def initialise + unless Dir.exists?(Dir.home + '/.MangaScrap/db/') + Utils_file::dir_create(Dir.home + '/.MangaScrap/db/') + end + @db = SQLite3::Database.new(Dir.home + '/.MangaScrap/db/data.db') + @db.execute 'CREATE TABLE IF NOT EXISTS Macros ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + date VARCHAR(32), + macro TEXT)' + end +end diff --git a/sources/DB/sub_data/data_module.rb b/sources/DB/sub_data/data_module.rb new file mode 100644 index 0000000..e69de29 diff --git a/sources/DB/sub_params/Download.rb b/sources/DB/sub_params/Download.rb new file mode 100644 index 0000000..c3f2b28 --- /dev/null +++ b/sources/DB/sub_params/Download.rb @@ -0,0 +1,60 @@ +$default_manga_path = Dir.home + '/Documents/mangas/' + +class Params_download + include Params_module + + public + def param_set(id, value, display = true) + @display = display + param = @params_list.select{|e| e[:id] == id}[0] + if param == nil + critical_error("param_set was called with the id #{id} but the Params_download database does not posses it") + end + case id + when 'bs', 'fs', 'nbf', 'es', 'ct', 'dt' + return param_check_nb(param, value) + when 'mp' + begin + Utils_file::dir_create(value) + rescue StandardError => error + write_output 'could not create requested path' + write_output "error message is : '" + error.message + "'" + return false + end + return param_exec_value_change(param, value) + else + critical_error("Did not find id #{id} in Param_download database") + end + end + + # warning : the params MUST be in the same order as the database + def get_params(default = false) + ret = [] + ret << Struct::Param_value.new('manga_path', 'mp', 'string', ((default) ? $default_manga_path : @params[:manga_path]), self) + ret << Struct::Param_value.new('between_sleep', 'bs', 'float', ((default) ? 0.1 : @params[:between_sleep]), self, 0.1, 300) + ret << Struct::Param_value.new('failure_sleep', 'fs', 'float', ((default) ? 0.1 : @params[:failure_sleep]), self, 0.1, 300) + ret << Struct::Param_value.new('nb_tries_on_fail', 'nbf', 'int', ((default) ? 20 : @params[:nb_tries_on_fail]), self, 1, 300) + ret << Struct::Param_value.new('error_sleep', 'es', 'float', ((default) ? 20 : @params[:error_sleep]), self, 0.1, 300) + ret << Struct::Param_value.new('connect_timeout', 'cto', 'int', ((default) ? 20 : @params[:connect_timeout]), self, 1, 300) + ret << Struct::Param_value.new('download_timeout', 'dt', 'int', ((default) ? 300 : @params[:download_timeout]), self, 0, 300) + end + + def initialize + @display = true + @db_name = 'Download' + @template_file = 'sources/templates/text/params/download.txt' + Struct.new('Download_params', :id, :manga_path, :between_sleep, :failure_sleep, :nb_tries_on_fail, :error_sleep, + :connect_timeout, :download_timeout) + init("CREATE TABLE IF NOT EXISTS #{@db_name} ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + manga_path TEXT, + between_sleep FLOAT, + failure_sleep FLOAT, + nb_tries_on_fail INT, + error_sleep FLOAT, + connect_timeout INT, + download_timeout INT)") do |data| + @params = Struct::Download_params.new(*data) + end + end +end diff --git a/sources/DB/sub_params/HTML.rb b/sources/DB/sub_params/HTML.rb new file mode 100644 index 0000000..f015d8a --- /dev/null +++ b/sources/DB/sub_params/HTML.rb @@ -0,0 +1,54 @@ +# only directly used by the Params class, must NOT be directly called elsewhere +# stores all of the macros the scrapper can execute +class Params_HTML + include Params_module + + public + def param_set(id, value, display = true) + @display = display + param = @params_list.select{|e| e[:id] == id}[0] + if param == nil + critical_error("param_set was called with the id #{id} but the Param_HTML database does not posses it") + end + case id + when 'agh', 'ne', 'nm' + return param_check_bool(param, value) + when 'nc' + ret = param_exec_value_change(param, value) + puts 'array inserted into the database :' + pp value.split(', ') + return ret + when 'td' + return param_exec_value_change(param, value) + else + Utils_errors::critical_error("Did not find id #{id} in Param_HTML database") + end + end + + # warning : the params MUST be in the same order as the database + def get_params(default = false) + ret = [] + ret << Struct::Param_value.new('auto_generate_html', 'agh', 'bool', ((default) ? true : @params[:auto_generate_html]), self) + ret << Struct::Param_value.new('nsfw_enabled', 'ne', 'bool', ((default) ? true : @params[:nsfw_enabled]), self) + ret << Struct::Param_value.new('nsfw_categories', 'nc', 'array', ((default) ? 'Ecchi, Mature, Smut, Adult' : @params[:nsfw_categories]), self) + ret << Struct::Param_value.new('template_dir', 'td', 'string', ((default) ? 'default' : @params[:template_dir]), self) + ret << Struct::Param_value.new('night_mode', 'nm', 'bool', ((default) ? false : @params[:night_mode]), self) + ret + end + + def initialize + @display = true + @db_name = 'HTML' + @template_file = 'sources/templates/text/params/html.txt' + Struct.new('HTML_params', :id, :auto_generate_html, :nsfw_enabled, :nsfw_categories, :template_dir, :night_mode) + init("CREATE TABLE IF NOT EXISTS #{@db_name} ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + auto_generate_html VARCHAR(5), + nsfw_enabled VARCHAR(5), + nsfw_categories TEXT, + template_dir TEXT, + night_mode VARCHAR(5))") do |data| + @params = Struct::HTML_params.new(*data) + end + end +end diff --git a/sources/DB/sub_params/Misc.rb b/sources/DB/sub_params/Misc.rb new file mode 100644 index 0000000..ea76a12 --- /dev/null +++ b/sources/DB/sub_params/Misc.rb @@ -0,0 +1,48 @@ +class Params_misc + include Params_module + + public + def param_set(id, value, display = true) + @display = display + param = @params_list.select{|e| e[:id] == id}[0] + if param == nil + critical_error("param_set was called with the id #{id} but the Param_misc database does not posses it") + end + case id + when 'ct', 'dd', 'ce', 'cu' + return param_check_bool(param, value) + when 've' + if value == 'low' || value == 'normal' || value == 'high' + return param_exec_value_change(param, value) + end + puts 'Error '.red + ' the value for ' + 'verbose'.yellow + ' needs to be : ' + 'low'.yellow + ' or ' + 'normal'.yellow + ' or ' + 'high'/yellow + return false + else + critical_error("Did not find id #{id} in Param_misc database") + end + end + + # warning : the params MUST be in the same order as the database + def get_params(default = false) + ret = [] + ret << Struct::Param_value.new('color_text', 'ct', 'bool', ((default) ? true : @params[:color_text]), self) + ret << Struct::Param_value.new('delete_diff', 'dd', 'bool', ((default) ? true : @params[:delete_diff]), self) + ret << Struct::Param_value.new('verbose', 've', 'string', ((default) ? 'high' : @params[:verbose]), self) + ret << Struct::Param_value.new('check_for_updates', 'cu', 'bool', ((default) ? true : @params[:check_for_updates]), self) + end + + def initialize + @display = true + @db_name = 'Misc' + @template_file = 'sources/templates/text/params/misc.txt' + Struct.new('Misc_params', :id, :color_text, :delete_diff, :verbose, :check_for_updates) + init("CREATE TABLE IF NOT EXISTS #{@db_name} ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + color_text VARCHAR(5), + delete_diff VARCHAR(5), + verbose VARCHAR(32), + check_for_updates VARCHAR(5))") do |data| + @params = Struct::Misc_params.new(*data) + end + end +end diff --git a/sources/DB/sub_params/Threads.rb b/sources/DB/sub_params/Threads.rb new file mode 100644 index 0000000..bb42999 --- /dev/null +++ b/sources/DB/sub_params/Threads.rb @@ -0,0 +1,41 @@ +class Params_threads + include Params_module + + public + def param_set(id, value, display = true) + @display = display + param = @params_list.select{|e| e[:id] == id}[0] + if param == nil + critical_error("param_set was called with the id #{id} but the Param_threads database does not posses it") + end + case id + when 'emt' + return param_check_bool(param, value) + when 'nt' + return param_check_nb(param, value) + else + critical_error("Did not find id #{id} in Param_threads database") + end + end + + # warning : the params MUST be in the same order as the database + def get_params(default = false) + ret = [] + ret << Struct::Param_value.new('enable_multi_threading', 'emt', 'bool', ((default) ? true : @params[:enable_multi_threading]), self) + ret << Struct::Param_value.new('nb_threads', 'nt', 'int', ((default) ? 8 : @params[:nb_threads]), self, 2, 100) + ret + end + + def initialize + @display = true + @db_name = 'Threads' + @template_file = 'sources/templates/text/params/threads.txt' + Struct.new('Thread_params', :id, :enable_multi_threading, :nb_threads) + init("CREATE TABLE IF NOT EXISTS #{@db_name} ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + enable_multi_threading VARCHAR(5), + nb_threads INT)") do |data| + @params = Struct::Thread_params.new(*data) + end + end +end diff --git a/sources/DB/sub_params/params_module.rb b/sources/DB/sub_params/params_module.rb new file mode 100644 index 0000000..c1623a0 --- /dev/null +++ b/sources/DB/sub_params/params_module.rb @@ -0,0 +1,209 @@ +module Params_module + attr_reader :params, :params_list, :db_name + + private + # used to output strings depending on the value of @display + def write_output(message) + if @display + puts message + end + end + + # used to write in the database + def set_param_value(param) + value = param[:value] + if param[:type] == 'bool' + value = value.to_s + end + Utils_database::db_exec("UPDATE #{@db_name} SET #{param[:string]} = ?", 'could not update ' + param[:string], @db, [value]) + end + + # this function should only be called if there is an error + def param_critical_error(param, func) + write_output 'Error : the ' + param + ' ended in the wrong function (' + func + ')' + write_output 'This is a Mangascrap error, please report it' + write_output '( unless you caused it by altering the code )' + false + end + + # used to validate a paarm value change + def param_exec_value_change(param, value) + if param[:type] == 'bool' + if value == 'true' + param[:value] = true + else + param[:value] = false + end + else + param[:value] = value + end + write_output "updated #{param[:string]} (#{param[:id].red}) to #{param[:value].to_s.green}" + set_param_value(param) + true + end + + # params check for numbers + def param_check_nb(param, value) + unless value.instance_of? 'Fixnum' + write_output "#{value} is not a Fixnum, got a #{value.class}" + false + end + if value < param[:min_value] || value > param[:max_value] + write_output "the value cannot be < #{param[:min_value]} or > #{param[:max_value]} for #{param[:string]} (#{param[:id]})" + return false + end + param_exec_value_change(param, value) + end + + # params check for booleans + def param_check_bool(param, value) + if value != 'true' && value != 'false' + write_output 'Error'.red + " : argument must be 'true' or 'false' when setting param #{param[:string]}\n" + return false + end + param_exec_value_change(param, value) + end + + # params check for strings + def param_check_string(_param, _display, _value) + critical_error('the param_check_string method was not overridden after being included in a database class') + end + + # gets the required string for database insertion + def get_insert_string(length) + i = 0 + lock = true + ret = '' + while i < length + if lock + lock = false + else + ret += ', ' + end + ret += '?' + i += 1 + end + ret + end + + # used to ensure that the booleans are booleans and not the strings that can be found in the database + # ( sqlite does not manage the boolean type ) + def prepare_data(data) + i = -1 + buff = get_params(true) + good_data = [] + data.each do |value| + if i == -1 # the first value is an id and is not in the params_list + good_data << value + i += 1 + next + end + if buff[i][:type] == 'bool' + if value.is_a?(TrueClass) || value == 'true' # translate string to boolean + good_data << true + elsif value.is_a?(FalseClass) || value == 'false' # translate string to boolean + good_data << false + else + Utils_errors::critical_error("The param #{buff[i][:string].gsub('_', ' ')} had a value of '#{buff[i][:value]}' \ +(#{buff[i][:value].class.to_s}), it should be 'true' or 'false' (boolean)") + end + else + good_data << value + end + i += 1 + end + good_data + end + + # gets all of the data form the database + def get_data_from_db(init_exec) + @db = SQLite3::Database.new(Dir.home + '/.MangaScrap/db/params.db') + @db.execute init_exec + data = Utils_database::db_exec("Select * from #{@db_name}", "could not get params from the #{@db_name} table", @db)[0] + if data == nil || data.size == 0 # if the database is empty, the parameters are initialized with the default values + default = [] << 1 # the id + begin + get_params(true).each do |param| + if param[:type] == 'bool' + if param[:value].is_a?(TrueClass) + default << 'true' + else + default << 'false' + end + else + default << param[:value] + end + end + insert_string = get_insert_string(default.length) + Utils_database::db_exec("INSERT INTO #{@db_name} VALUES (#{insert_string})", "could not init #{@db_name} database please reset it", @db, default) + rescue SQLite3::Exception => e + Utils_errors::critical_error("exception while retrieving params (#{@db_name}) / initializing params", e) + end + data = default + end + data + end + + # method called by the initialize of the Params_class + def init(init_exec) + data = get_data_from_db(init_exec) + data = prepare_data(data) + yield(data) + @params_list = get_params # todo : les params ont déjà été get avec les valeurs par défaut, regarder s'il ne serait pas possible de faire usae du tableau de structures plutot que d'en demander un autre + end + + public # ========================================================================================================= public + # used to access the params directly from the class ( you can see it as some sort of glorified getter ) + def [](id) + begin + @params[id] + rescue NameError => e + Utils_errors::critical_error("Params module #{@db_name}'s getter was called with a bad argument : '#{id}' ( there is no such id )", e) + end + end + + # used to display the parameters and their setting nicely on the terminal by reading a template file in the @template_file + def params_display + margin_string = 22 + template = File.open(@template_file).read + template = template.gsub('(not yet implemented)', '(not yet implemented)'.magenta) + template = template.gsub("[#{@db_name.downcase}]", "======================= #{@db_name.upcase} =======================".blue) + @params_list.each do |param| + template = template.gsub("[#{param[:id]}]", param[:string].gsub('_', ' ').yellow + ((param[:string].size > margin_string) ? ' ' : ' ' * (margin_string - param[:string].size))) + template = template.gsub("|#{param[:id]}|", param[:string].gsub('_', ' ').yellow) + template = template.gsub("(#{param[:id]})", "(#{param[:id]})".red + ((param[:id].size == 3) ? '' : ' ')) + template = template.gsub("{#{param[:id]}}", param[:value].to_s.green) + end + template + end + + # id = the 2 short string that identifies a parameter + # value = value on witch you wish to set the parameter. WARNING : booleans are sent as strings 'true' / 'false' + # display = writes or not on the terminal in case of failure + # allows you to set the parameters + def param_set (_id, _value, _display = true) + critical_error('the param_set method was not overridden after being included in a database class') + end + + # used to reset all params + def params_reset + buff = [] + names = [] + get_params(true).each do |param| + if param[:type] == 'bool' + if param[:value].is_a?(TrueClass) + buff << 'true' + elsif param[:value].is_a?(FalseClass) + buff << 'false' + else + Utils_errors::critical_error("The param #{param[:string].gsub('_', ' ')} has a value of '#{param[:value]}' \ +(#{param[:value].class.to_s}), it should be 'true' or 'false' (boolean)") + end + else + buff << param[:value] + end + names << param[:string] + ' = ?' + end + Utils_database::db_exec("UPDATE #{@db_name} SET #{names.join(', ')} WHERE id=1", "could not update #{@db_name}", @db, buff) + end +end diff --git a/sources/Download/base_downloader.rb b/sources/Download/base_downloader.rb new file mode 100644 index 0000000..06486fb --- /dev/null +++ b/sources/Download/base_downloader.rb @@ -0,0 +1,167 @@ +# this module is used by every download class +module Base_downloader + attr_reader :links + + private + def extract_links(manga) + Utils_errors::critical_error('The "extract_links" method was not overridden after being included in a download class') + end + + # used to display an error on a page or chapter download + def link_err(data, chapter, disp) + if chapter + @db.add_todo(@manga_data, data[0], data[1], -1) + @aff.error_on_page_download(disp) + else + @db.add_todo(@manga_data, data[0], data[1], data[2]) + @aff.error_on_page_download(disp) + end + false + end + + def validate_data(description, author, artist, type, status, genres, release, html_name, alternative_names, rank, rating, rating_max, cover_xpath) + Utils_file::dir_create(@dir) + Utils_connection::write_cover(@doc, cover_xpath, @dir + 'cover.jpg', @params[:manga_path] + @manga_data.site_dir + @manga_data.name + '.jpg') + File.open(@dir + 'description.txt', 'w') do |txt| + txt << Utils_file::data_concatenation(@manga_data, Utils_file::description_manipulation(description), author, artist, type, status, genres, release, html_name, alternative_names) + end + @aff.data_disp(@manga_data.in_db) + if @manga_data.in_db + @db.update_manga(@manga_data.name, description, author, artist, genres, html_name, alternative_names, rank, rating, rating_max) + else + @db.add_manga(@manga_data, description, author, artist, type, status, genres, release, html_name, alternative_names, rank, rating, rating_max) + end + end + + public + # takes the data array [volume, chapter, page] and casts it into a string + def values_to_string(volume, chapter, page) + volume_string = '' + chapter_string = '' + page_string = '' + case volume + when -2 + volume_string = 'volume TBD' + when -3 + volume_string = 'volume NA' + when -4 + volume_string = 'volume ANT' + else + if volume >= 0 + volume_string = "volume #{volume}" + end + end + chapter_string = "chapter #{chapter} " if chapter != nil || chapter == -1 + page_string = "page #{page}" if page != nil || page == -1 + volume_string + ' ' + chapter_string + ' ' + page_string + end + + def data_to_string(data) + values_to_string(data[0], data[1], data[2]) + end + + def link_generator(volume, chapter, page) + Utils_errors::critical_error('The "link_generator" method was not overridden after being included in a download class') + end + + def page_link(link, data) + Utils_errors::critical_error('The "page_link" method was not overridden after being included in a download class') + end + + def chapter_link(link, prep_display = '') + Utils_errors::critical_error('The "chapter_link" method was not overridden after being included in a download class') + end + + # _todo is the method that downloads all the pages / chapters that could not be downloaded previously and where placed + # in the _todo database + # once successfully downloaded, the _todo element is deleted from the _todo database + # elem[0] = id of line in database ; elem[1] = id of manga in database + # elem[2] = volume value ; elem[3] = chapter value ; elem[4] = page value + def todo + todo = @db.get_todo(@manga_data) + if todo.size != 0 + @aff.prepare_todo + biggest = todo.map { |a| [a[2], 0].max }.max + todo = todo.sort_by! { |a| [(a[2] < 0) ? (biggest + a[2] * -1) : a[2], -a[3]] } + todo.each do |elem| + @aff.display_todo('downloading ' + values_to_string(elem[2], elem[3], elem[4])) + if elem[4] != -1 # if chapter + data = [] << -1 << elem[3] << elem[4] + if page_link(link_generator(-1, elem[3], elem[4]), data) + @db.delete_todo(elem[0]) + else + @aff.todo_err('failed to download ' + values_to_string(elem[2], elem[3], elem[4])) + end + else # if not a chapter + if chapter_link(link_generator(-1, elem[3], 1)) + @db.delete_todo(elem[0]) + else + @aff.todo_err('failed to download ' + values_to_string(elem[2], elem[3], nil), true) + end + end + end + @aff.end_todo + end + @downloaded_a_page + end + + # checks witch are the chapters that have not been downloaded ( they are not in the traces database ) + # the method first gets all of the links from the database and then compares with the links of the website + def missing_chapters + traces = @db.get_trace(@manga_data) + i = 0 + @links.each do |link| + data = Utils_website_specific::Mangafox::data_extractor(link) + if traces.count{|_id, _manga_name, vol_value, chap_value| vol_value == data[0] && chap_value == data[1]} == 0 + prep_display = " ( link #{i + 1} / #{@links.size} )" + chapter_link(link, prep_display) + end + i += 1 + end + @aff.dump + end + + # the update method will first check for _todo elements to download then check for missing chapters + # it only calls the data method if a page was downloaded + def update + todo + missing_chapters + if @downloaded_a_page + data + end + @downloaded_a_page + end + + # the data method is used to download the description, author and artists names, ... + # by default it raises an exception and MUST be overridden in the class + def data + Utils_errors::critical_error('The "data" method was not overridden after being included in a download class') + end + + # all download classes use the same initializer with a Manga_data as argument ( it must be resolved ) + # variables are :0 + # @manga_data => the Manga_data class + # @downloaded_a_page => allows the class to know if a page was ( or not ) downloaded + # @extracted_data => used to block multiple calls to the data method, does not download if set to true + # @params => all of the download parameters + # @dir => the directory in witch the mangas are placed + # @db => the database instance todo : delete the variable + # @aff => the attached DownloadDisplay class + # @doc => the first page todo : regarder si le Manga_data ne peut pas faire ça au moment de son check_link + def initialize(manga, download_data = true) + @manga_data = manga + @downloaded_a_page = false + @extracted_data = !download_data + @params = Params.instance.download + @dir = @params[:manga_path] + manga.site_dir + 'mangas/' + manga.name + '/' + @db = Manga_database.instance + # doc is the variable that stores the raw data of the manga's page + @aff = DownloadDisplay.new(manga.site_dir, manga.name) + @doc = Utils_connection::get_page(manga.link, true) + if @doc == nil + raise 'failed to get manga ' + manga.name + "'s chapter index" + end + @links = extract_links(manga).reverse + data if download_data || !manga.in_db + end +end diff --git a/sources/Download/mangafox.rb b/sources/Download/mangafox.rb index 9ce6683..11bb4e8 100644 --- a/sources/Download/mangafox.rb +++ b/sources/Download/mangafox.rb @@ -1,11 +1,13 @@ class Download_Mangafox + include Base_downloader + private def extract_links(manga) tries = @params[4] links = @doc.xpath('//a[@class="tips"]').map{ |link| link['href'] } while (links == nil || links.size == 0) && tries > 0 puts ('error while retrieving chapter index of ' + manga.name).yellow - doc = get_page(manga.link) + doc = Utils_connection::get_page(manga.link) if doc != nil links = doc.xpath('//a[@class="tips"]').map{ |link| link['href'] } end @@ -17,44 +19,7 @@ def extract_links(manga) links end - def link_err(data, chapter = false) - if !chapter - @db.add_todo(@manga_data, data[0], data[1], data[2]) - @aff.error_on_page_download('X') - else - @db.add_todo(@manga_data, data[0], data[1], -1) - @aff.error_on_page_download('X') - end - false - end - - # takes the volume value to return it's string value - def volume_string(value) - volume_string = '' - case value - when (value == -2) - volume_string = ' of volume TBD' - when (value == -3) - volume_string = ' of volume NA' - when (value == -4) - volume_string = ' of volume ANT' - when (value >= 0) - volume_string = " of volume #{value}" - if value % 1 == 0 - volume_string += ' ' - end - else - # volume string remains empty for return - end - volume_string - end - public - def get_links - @links - end - - # todo => vérifier s'il n'est pas possible de supprimer cette fonction def link_generator(volume, chapter, page) chapter = chapter.to_i if chapter % 1 == 0 link = @manga_data.site + 'manga/' + @manga_data.name + '/' @@ -75,14 +40,10 @@ def link_generator(volume, chapter, page) else link += chapter.to_s end - link += '/' + page.to_s + '.html' - if Utils_connection::redirection_detection(link) - puts 'Error : generated a bad link\nlink = ' + link - return nil - end - link + link + '/' + page.to_s + '.html' end + # downloads a page, with link = the link, data = [volume, chapter, page] def page_link(link, data) page = Utils_connection::get_page(link, true) if page == nil @@ -90,156 +51,72 @@ def page_link(link, data) end pic_link = page.xpath('//img[@id="image"]').map{|img| img['src']} if pic_link[0] == nil - return link_err(data) + return link_err(data, false, 'x') end pic_buffer = Utils_connection::get_pic(pic_link[0], true) if pic_buffer == nil || Utils_file::write_pic(pic_buffer, data, @dir) == false - return link_err(data) + return link_err(data, false, '!') end @downloaded_a_page = true true end + # downloads a chapter with link = the link and prep_display = small string displayed when announcing the download of the chapter def chapter_link(link, prep_display = '') - data = Utils_website_specific::Mangafox::data_extractor(link) + data = @manga_data.extract_values_from_link(link) if data[0] == -42 @aff.unmanaged_link(link) false end last_pos = link.rindex(/\//) link = link[0..last_pos].strip + '1.html' - page = Utils_connection::get_page(link, true) - if page == nil - return link_err(data, true) + begin + if (page = Utils_connection::get_page(link, true)) == nil + return link_err(data, true, 'X') + end + rescue RuntimeError + return link_err(data, true, 'R') end - @aff.prepare_chapter("downloading chapter #{data[1]}#{volume_string(data[0])} of #{@manga_data.name}" + prep_display) + @aff.prepare_chapter("downloading #{data_to_string(data)} of #{@manga_data.name}" + prep_display) number_of_pages = page.xpath('//div[@class="l"]').text.split.last.to_i + if number_of_pages == 0 + raise 'could not get number of pages from page. Link = ' + link + end page_nb = 1 while page_nb <= number_of_pages data[2] = page_nb unless page_link(link, data) - link_err(data) + return false end @aff.downloaded_page(page_nb) - page_nb += 1 last_pos = link.rindex(/\//) + page_nb += 1 link = link[0..last_pos].strip + page_nb.to_s + '.html' - page = Utils_connection::get_page(link, true) - if page == nil - return link_err(data, true) - end end @aff.dump_chapter @db.add_trace(@manga_data, data[0], data[1], number_of_pages) - HTML_buffer.instance.add_chapter(@manga_data, data[0], data[1]) true end - def todo - todo = @db.get_todo(@manga_data) - if todo.size != 0 - @aff.prepare_todo - biggest = todo.map { |a| [a[2], 0].max }.max - todo = todo.sort_by! { |a| [(a[2] < 0) ? (biggest + a[2] * -1) : a[2], -a[3]] } - todo.each do |elem| - volume_string = volume_string(elem[2]) - if elem[4] != -1 - @aff.display_todo("downloading page #{elem[4]}, chapter #{elem[3]}" + ((elem[2] == -1) ? '' : ", volume #{elem[2]} ")) - data = [] - data << elem[2] << elem[3] << elem[4] - if (link = link_generator(elem[2], elem[3], elem[4])) != nil && page_link(link, data) == true - @db.delete_todo(elem[0]) - else - @aff.todo_err("failed to download page #{elem[4]} of chapter #{elem[3]}" + volume_string) - end - else - @aff.display_todo("downloading chapter #{elem[3]}" + ((elem[2] == -1) ? '' : ", volume #{elem[2]} "), true) - if (link = link_generator(elem[2], elem[3], 1)) != nil && chapter_link(link) == true - @db.delete_todo(elem[0]) - else - @aff.todo_err("failed to download chapter #{elem[3]}" + volume_string, true) - end - end - end - @aff.end_todo - end - @downloaded_a_page - end - - def missing_chapters(get_data = true) - traces = @db.get_trace(@manga_data) - i = 0 - @links.each do |link| - data = Utils_website_specific::Mangafox::data_extractor(link) - if traces.count{|_id, _manga_name, vol_value, chap_value| vol_value == data[0] && chap_value == data[1]} == 0 - prep_display = " ( link #{i + 1} / #{@links.size} )" - chapter_link(link, prep_display) - end - i += 1 - end - if @downloaded_a_page - if get_data - data - end - end - @aff.dump - end - - def update - todo - missing_chapters - if @downloaded_a_page && !@extracted_data - data - end - @downloaded_a_page - end - def data - @extracted_data = true - alternative_names = @doc.xpath('//div[@id="title"]/h3').text - release_author_artist_genres = @doc.xpath('//td[@valign="top"]') - release = release_author_artist_genres[0].text.to_i - author = release_author_artist_genres[1].text.gsub(/\s+/, '').gsub(',', ', ') - artist = release_author_artist_genres[2].text.gsub(/\s+/, '').gsub(',', ', ') - genres = release_author_artist_genres[3].text.gsub(/\s+/, '').gsub(',', ', ') - description = @doc.xpath('//p[@class="summary"]').text - data = @doc.xpath('//div[@class="data"]/span') - status = data[0].text.gsub(/\s+/, '').split(',')[0] - rank = data[1].text[/\d+/] - rating = data[2].text[/\d+[.,]\d+/] - rating_max = 5 # rating max is a constant in mangafox - tmp_type = @doc.xpath('//div[@id="title"]/h1')[0].text.split(' ') - type = tmp_type[tmp_type.size - 1] - html_name = tmp_type.take(tmp_type.size - 1).join(' ') - Utils_file::dir_create(@dir) - Utils_connection::write_cover(@doc, '//div[@class="cover"]/img', @dir + 'cover.jpg', @params[1] + 'mangafox/mangas/' + @manga_data.name + '.jpg') - File.open(@dir + 'description.txt', 'w') do |txt| - txt << Utils_file::data_concatenation(@manga_data.name, Utils_file::description_manipulation(description), @manga_data.site, @manga_data.link, author, artist, type, status, genres, release, html_name, alternative_names) - end - @aff.data_disp(@manga_data.in_db) - if @manga_data.in_db - @db.update_manga(@manga_data.name, description, author, artist, genres, html_name, alternative_names, rank, rating, rating_max) - else - @db.add_manga(@manga_data, description, author, artist, type, status, genres, release, html_name, alternative_names, rank, rating, rating_max) - end - end - - def initialize(manga, download_data = true) - @manga_data = manga - @downloaded_a_page = false - @extracted_data = false - @params = Params.instance.get_params - @dir = @params[1] + manga.site_dir + 'mangas/' + manga.name + '/' - @db = Manga_database.instance - # doc is the variable that stores the raw data of the manga's page - @aff = DownloadDisplay.new('mangafox', manga.name) - @doc = Utils_connection::get_page(manga.link, true) - if @doc == nil - raise 'failed to get manga ' + manga.name + "'s chapter index" - end - if download_data - data + unless @extracted_data + @extracted_data = true + alternative_names = @doc.xpath('//div[@id="title"]/h3').text + release_author_artist_genres = @doc.xpath('//td[@valign="top"]') + release = release_author_artist_genres[0].text.to_i + author = release_author_artist_genres[1].text.gsub(/\s+/, '').gsub(',', ', ') + artist = release_author_artist_genres[2].text.gsub(/\s+/, '').gsub(',', ', ') + genres = release_author_artist_genres[3].text.gsub(/\s+/, '').gsub(',', ', ') + description = @doc.xpath('//p[@class="summary"]').text + data = @doc.xpath('//div[@class="data"]/span') + status = data[0].text.gsub(/\s+/, '').split(',')[0] + rank = data[1].text[/\d+/] + rating = data[2].text[/\d+[.,]\d+/] + rating_max = 5 # rating max is a constant in mangafox + tmp_type = @doc.xpath('//div[@id="title"]/h1')[0].text.split(' ') + type = tmp_type[tmp_type.size - 1] + html_name = tmp_type.take(tmp_type.size - 1).join(' ') + validate_data(description, author, artist, type, status, genres, release, html_name, alternative_names, rank, rating, rating_max, '//div[@class="cover"]/img') end - @links = extract_links(manga).reverse end end diff --git a/sources/Download/mangareader_mangapanda.rb b/sources/Download/mangareader_mangapanda.rb index 58244e5..2e037b5 100644 --- a/sources/Download/mangareader_mangapanda.rb +++ b/sources/Download/mangareader_mangapanda.rb @@ -1,11 +1,13 @@ class Download_Mangareader_Pandamanga + include Base_downloader + private def extract_links(manga) tries = @params[4] links = @doc.xpath('//table[@id="listing"]/tr/td/a').map{ |link| link['href'] } while (links == nil || links.size == 0) && tries > 0 puts ('error while retrieving chapter index of ' + manga.name).yellow - doc = get_page(manga.link) + doc = Utils_connection::get_page(manga.link) if doc != nil links = doc.xpath('//a[@class="tips"]').map{ |link| link['href'] } end @@ -21,44 +23,27 @@ def extract_links(manga) ret end - def link_err(data, chapter = false) - if !chapter - @db.add_todo(@manga_data, data[0], data[1], data[2]) - @aff.error_on_page_download('X') - else - @db.add_todo(@manga_data, data[0], data[1], -1) - @aff.error_on_page_download('X') - end - false - end - - public - def get_links - @links - end - - # todo => vérifier s'il n'est pas possible de supprimer cette fonction - def link_generator(volume, chapter, page) - link = @manga_data.site + @manga_data.name + '/' + chapter.to_s + '/' + page.to_s - if Utils_connection::redirection_detection(link) - puts 'Error : generated a bad link\nlink = ' + link - return nil - end - link + # warning : the volume value is not used for both mangareader and pandamanga + def link_generator(_volume, chapter, page) + @manga_data.site + @manga_data.name + '/' + chapter.to_s + '/' + page.to_s end def page_link(link, data) - page = Utils_connection::get_page(link, true) + begin + page = Utils_connection::get_page(link, true) + rescue RuntimeError + return link_err(data, false, 'r') + end if page == nil return false end pic_link = page.xpath('//img').map{|img| img['src']} if pic_link[0] == nil - return link_err(data) + return link_err(data, false, 'x') end pic_buffer = Utils_connection::get_pic(pic_link[0], true) if pic_buffer == nil || Utils_file::write_pic(pic_buffer, data, @dir) == false - return link_err(data) + return link_err(data, false, '!') end @downloaded_a_page = true true @@ -66,93 +51,36 @@ def page_link(link, data) def chapter_link(link, prep_display = '') data = @manga_data.extract_values_from_link(link) - page = Utils_connection::get_page(link, true) + begin + page = Utils_connection::get_page(link, true) + rescue RuntimeError + return link_err(data, true, 'R') + end if page == nil - return link_err(data, true) + return link_err(data, true, 'X') end @aff.prepare_chapter("downloading chapter #{data[1]} of #{@manga_data.name}" + prep_display) number_of_pages = page.xpath('//option').last.text.to_i + if number_of_pages == 0 + raise 'could not get number of pages from page. Link = ' + link + end page_nb = 1 link += '/1' while page_nb <= number_of_pages data[2] = page_nb unless page_link(link, data) - link_err(data) + return false end @aff.downloaded_page(page_nb) last_pos = link.rindex(/\//) - link = link[0..last_pos].strip + page_nb.to_s - page = '' - page = Utils_connection::get_page(link, true) - if page == nil - return link_err(data, true) - end page_nb += 1 + link = link[0..last_pos].strip + page_nb.to_s end @aff.dump_chapter @db.add_trace(@manga_data, data[0], data[1], number_of_pages) - HTML_buffer.instance.add_chapter(@manga_data, data[0], data[1]) true end - def todo - todo = @db.get_todo(@manga_data) - if todo.size != 0 - @aff.prepare_todo - biggest = todo.map { |a| [a[2], 0].max }.max - todo = todo.sort_by! { |a| [(a[2] < 0) ? (biggest + a[2] * -1) : a[2], -a[3]] } - todo.each do |elem| - if elem[4] != -1 - @aff.display_todo("downloading page #{elem[4]}, chapter #{elem[3]}") - data = [] - data << -1 << elem[3] << elem[4] - if (link = link_generator(-1, elem[3], elem[4])) != nil && page_link(link, data) == true - @db.delete_todo(elem[0]) - else - @aff.todo_err("failed to download page #{elem[4]} of chapter #{elem[3]}" + volume_string) - end - else - @aff.display_todo("downloading chapter #{elem[3]}") - if (link = link_generator(-1, elem[3], 1)) != nil && chapter_link(link) == true - @db.delete_todo(elem[0]) - else - @aff.todo_err("failed to download chapter #{elem[3]}" + volume_string, true) - end - end - end - @aff.end_todo - end - @downloaded_a_page - end - - def missing_chapters(get_data = true) - traces = @db.get_trace(@manga_data) - i = 0 - @links.each do |link| - data = @manga_data.extract_values_from_link(link) - if traces.count{|_id, _manga_name, _vol_value, chap_value| chap_value == data[1]} == 0 - prep_display = " ( link #{i + 1} / #{@links.size} )" - chapter_link(link, prep_display) - end - i += 1 - end - if @downloaded_a_page - if get_data - data - end - end - @aff.dump - end - - def update - todo - missing_chapters - if @downloaded_a_page && !@extracted_data - data - end - @downloaded_a_page - end - def data @extracted_data = true tmp = @doc.xpath('//div[@id="mangaproperties"]/table/tr') @@ -168,35 +96,6 @@ def data rating_max = -1 type = (tmp[6].text.split(':')[1].strip == 'Right to Left') ? 'Manga' : 'Manhwa' html_name = @doc.xpath('//h2')[0].text - Utils_file::dir_create(@dir) - Utils_connection::write_cover(@doc, '//div[@id="mangaimg"]/img', @dir + 'cover.jpg', @params[1] + 'mangareader/mangas/' + @manga_data.name + '.jpg') - File.open(@dir + 'description.txt', 'w') do |txt| - txt << Utils_file::data_concatenation(@manga_data.name, Utils_file::description_manipulation(description), @manga_data.site, @manga_data.link, author, artist, type, status, genres, release, html_name, alternative_names) - end - @aff.data_disp(@manga_data.in_db) - if @manga_data.in_db - @db.update_manga(@manga_data.name, description, author, artist, genres, html_name, alternative_names, rank, rating, rating_max) - else - @db.add_manga(@manga_data, description, author, artist, type, status, genres, release, html_name, alternative_names, rank, rating, rating_max) - end - end - - def initialize(manga, download_data = true) - @manga_data = manga - @downloaded_a_page = false - @extracted_data = false - @params = Params.instance.get_params - @dir = @params[1] + manga.site_dir + 'mangas/' + manga.name + '/' - @db = Manga_database.instance - # doc is the variable that stores the raw data of the manga's page - @aff = DownloadDisplay.new('mangareader', manga.name) - @doc = Utils_connection::get_page(manga.link, true) - if @doc == nil - raise 'failed to get manga ' + manga.name + "'s chapter index" - end - if download_data - data - end - @links = extract_links(manga) + validate_data(description, author, artist, type, status, genres, release, html_name, alternative_names, rank, rating, rating_max, '//div[@id="mangaimg"]/img') end end diff --git a/sources/DownloadDisplay.rb b/sources/DownloadDisplay.rb index d9cbce8..5032239 100644 --- a/sources/DownloadDisplay.rb +++ b/sources/DownloadDisplay.rb @@ -75,10 +75,8 @@ def prepare_todo # normal display for a todo element def display_todo(string, chapter = false) + puts '' puts string - if chapter - puts '' - end end # called when an error occured while trying to download a todo element diff --git a/sources/api/mangas.rb b/sources/api/mangas.rb index 49f6d6d..78d5487 100644 --- a/sources/api/mangas.rb +++ b/sources/api/mangas.rb @@ -28,7 +28,7 @@ def self.add(mangas, generate_html = false) # fast_update = bool, if true, the update function will ignore all mangas with the 'Completed' status def self.update(mangas, todo_only = false, fast_update = false) html = HTML.new - params = Params.instance.get_params + params = Params.instance.misc if fast_update mangas = mangas.reject{|manga| manga.data[8] == 'Completed'} end @@ -39,8 +39,8 @@ def self.update(mangas, todo_only = false, fast_update = false) next end generate_html = (todo_only ? dw.todo : dw.update) - if params[6] == 'true' - if Delete_diff::delete_diff(dw.get_links, manga) || generate_html + if params[:delete_diff] + if Delete_diff::delete_diff(dw.links, manga) || generate_html html_manga = HTML_manga.new(manga) html_manga.generate_chapter_index end @@ -104,7 +104,7 @@ def self.data(mangas) puts 'downloading data of ' + mangas.size.to_s + ' element(s)' html = HTML.new mangas.each do |manga| - dw = manga.get_download_class + dw = manga.get_download_class(false) if dw == nil next end diff --git a/sources/api/params.rb b/sources/api/params.rb index 931bd98..9e73387 100644 --- a/sources/api/params.rb +++ b/sources/api/params.rb @@ -3,7 +3,11 @@ module MangaScrap_API # returns all of the parameters in an array of [name, id, value] def self.get_params_list - Params.instance.get_params_list + Params.instance.param_classes + end + + def self.display_params + Params.instance.display_params end # allows you to set the parameters @@ -27,7 +31,7 @@ def self.param_reset (require_confirmation = true) def self.params_management(args) case args[0] when 'list' - puts Params.instance.param_list_file + display_params when 'reset' Params.instance.param_reset when 'set' diff --git a/sources/html/html.rb b/sources/html/html.rb index e2f5a8e..232112b 100644 --- a/sources/html/html.rb +++ b/sources/html/html.rb @@ -14,11 +14,12 @@ def html_get_data(site, updated_recently = false) ret << " ' end source_buffer = ' ' - if @params[9] == 'false' + if @html_params[:nsfw_enabled] manga_genres = manga.data[9].split(', ') - if (@params[10].split(', ') & manga_genres).empty? + if (@html_params[:nsfw_categories].split(', ') & manga_genres).empty? ret << source_buffer else + # todo manage sites # warning : sites must be managed ret << ' ' end @@ -57,12 +58,12 @@ def index_js_data(site) public # generates the manga index ( the page containing the links to all mangas ) def generate_index - if @params[8] == 'true' || @force_html == true + if @html_params[:auto_generate_html] puts 'updating html of manga index' sites = Manga_data::get_compatible_sites sites.each do |site| site_dir = Manga_data::get_dir_from_site(site) - dir = @params[1] + site_dir + dir = @manga_path + site_dir HTML_utils::copy_html_related_files(site, dir) template = File.open('sources/templates/web/site_index/site_index_template.html').read template = template.gsub('#####list#####', html_get_data(site)) @@ -84,12 +85,12 @@ def generate_index # WARNING ====================================================================================================================================================== def generate_updated - if @params[8] == 'true' || @force_html == true + if @html_params[:auto_generate_html] puts 'generating updated index' sites = Manga_data.get_compatible_sites sites.each do |site| site_dir = Manga_data::get_dir_from_site(site) - dir = @params[1] + site_dir + dir = @manga_path + site_dir HTML_utils::copy_html_related_files(site, dir) template = File.open('sources/templates/web/manga_updated_index_template.html').read template = template.gsub('#####list#####', html_get_data(site, true)) @@ -102,8 +103,8 @@ def generate_updated # the constructor copies all the css and places them in the html dir def initialize(force_html = false) @force_html = force_html - @params = Params.instance.get_params - @nsfw_genres = @params[10].split(', ') + @manga_path = Params.instance.download[:manga_path] + @html_params = Params.instance.html @db = Manga_database.instance end end diff --git a/sources/html/html_buffer.rb b/sources/html/html_buffer.rb deleted file mode 100644 index 97be6a0..0000000 --- a/sources/html/html_buffer.rb +++ /dev/null @@ -1,25 +0,0 @@ -class HTML_buffer - include Singleton - - def add_chapter(manga_name, chapter, volume) - ret = @dl.select{|struct| struct['name'] == manga_name} - if ret.size != 0 - ret[0]['downloaded'] << [volume, chapter] - else - buff = [volume, chapter] - @dl << Struct::Updated.new(manga_name, buff); - end - end - - def clear_data - @dl = [] - end - - def get_data - @dl - end - - def initialize - @dl = [] - end -end diff --git a/sources/html/html_manga.rb b/sources/html/html_manga.rb index c2afe7f..1848d53 100644 --- a/sources/html/html_manga.rb +++ b/sources/html/html_manga.rb @@ -113,7 +113,7 @@ def chapter_list_to_a_list(template) public # generates the chapter index, includes description, cover, names of artist and author, ... def generate_chapter_index - if @params[8] == 'true' || @force_html == true + if @html_params[:auto_generate_html] puts 'updating chapter index of ' + @manga_data[1] Utils_file::dir_create(@path_html) template = File.open('sources/templates/web/manga_presentation/presentation_template.html').read @@ -139,7 +139,7 @@ def generate_chapter_index end def initialize(manga_data, force_html = false) - @params = Params.instance.get_params + @html_params = Params.instance.html @db = Manga_database.instance @force_html = force_html @manga_data = [] @@ -150,7 +150,7 @@ def initialize(manga_data, force_html = false) end @manga_name = @manga_data[11] @path_pictures = '/mangas/' + @manga_data[1] + '/' - @dir = @params[1] + Manga_data::get_dir_from_site(manga_data.site) + @dir = Params.instance.download[:manga_path] + Manga_data::get_dir_from_site(manga_data.site) @path_html = @dir + 'html/' + @manga_data[1] HTML_utils::copy_html_related_files(manga_data.site, @dir) @traces = get_traces(manga_data) diff --git a/sources/init.rb b/sources/init.rb index f35dd8a..2a6ed4c 100644 --- a/sources/init.rb +++ b/sources/init.rb @@ -2,12 +2,13 @@ class Init def self.get_gem_list - %w(singleton + %w(singleton open-uri pp colorize nokogiri - sqlite3) + sqlite3 + typhoeus) end def self.get_file_list @@ -30,7 +31,7 @@ def self.get_file_list utils/website_specific/mangareader_mangapanda_utils html/html html/html_manga - html/html_buffer + Download/base_downloader Download/mangafox Download/mangareader_mangapanda DownloadDisplay @@ -41,9 +42,17 @@ def self.get_file_list instructions/query instructions/Instructions_exec instructions/Manga_data_filter + DB/sub_data/data_module + DB/sub_data/History + DB/sub_data/Macro + DB/sub_params/params_module + DB/sub_params/HTML + DB/sub_params/Download + DB/sub_params/Misc + DB/sub_params/Threads + DB/Params DB/Manga_data - DB/Manga_database - DB/Params) + DB/Manga_database) end def self.load_gem(gem) @@ -53,9 +62,10 @@ def self.load_gem(gem) puts '' puts "exception while trying to load #{gem}, please follow the installation instructions in the install directory" puts 'message is : ' + e.message - puts 'please note that a ruby update may require a re-download of the gems' puts '' - exit exit_value + puts 'please note that a ruby update may require a re-download of the gems'.yellow + puts '' + exit 1 end end @@ -79,6 +89,7 @@ def self.initialize_mangascrap Struct.new('Updated', :name, :downloaded) Struct.new('Query_arg', :name, :arg_type, :sql_column, :sub_string) Struct.new('HTML_data', :volume, :chapter, :date, :href, :nb_pages, :file_name) + Struct.new('Param_value', :string, :id, :type, :value, :class, :min_value, :max_value) begin Utils_file::dir_create(Dir.home + '/.MangaScrap/db') rescue StandardError => error @@ -87,7 +98,7 @@ def self.initialize_mangascrap exit 5 end Utils_connection::init_utils - if Params.instance.get_params[11] == 'false' + unless Params.instance.misc[:color_text] String.disable_colorization = true end end diff --git a/sources/instructions/Instructions_exec.rb b/sources/instructions/Instructions_exec.rb index 033606c..5376ded 100644 --- a/sources/instructions/Instructions_exec.rb +++ b/sources/instructions/Instructions_exec.rb @@ -1,4 +1,5 @@ =begin + This class parses the ARGV to extract all of the instructions MangaScrap will use To add your own instruction, you need to add your own block respecting this format in the init_parser method : @@ -160,11 +161,13 @@ def init_parser_mangas_related public # returns a pre-configured parser for the data arguments - # id needs 2 arguments : name ( args[0] and site args[1] ) - # link needs one argument : link ( args[0] ) - # File parser uses one argument : file_name ( args[0] ) and directly outputs it's content in @data_to_prepare - # query takes one argument : query witch is the parsed to extract the required mangas - # all does not use any arguments and gets everything from the database + # takes an arguments : instruction = the instruction that calls these data_arguments + # the data argument's arguments : + # id needs 2 arguments : name ( args[0] and site args[1] ) + # link needs one argument : link ( args[0] ) + # File parser uses one argument : file_name ( args[0] ) and directly outputs it's content in @data_to_prepare + # query takes one argument : query witch is the parsed to extract the required mangas + # all does not use any arguments and gets everything from the database def get_data_parser(instruction) parser = Data_parser.new(instruction) parser.on('id', 2) do |args| diff --git a/sources/instructions/delete_diff.rb b/sources/instructions/delete_diff.rb index 5561bf4..f146314 100644 --- a/sources/instructions/delete_diff.rb +++ b/sources/instructions/delete_diff.rb @@ -9,7 +9,7 @@ def self.delete_bad_files(traces, data, dir) #todo => attention à ce que le fichier repéré soit bel et bien celui supprimé def self.delete_diff(chap_list, manga_data) - params = Params.instance.get_params + params = Params.instance.download db = Manga_database.instance dir = params[1] + manga_data.site_dir + 'mangas/' + manga_data.name + '/' unless File.directory?(dir) @@ -32,7 +32,7 @@ def self.delete_diff(chap_list, manga_data) puts 'deleting file : '.yellow + file File.delete(file) end - Dir.glob(params[1] + manga_data.site_dir + 'html/' + manga_data.name + HTML_utils::html_chapter_filename(chap[1], chap[0])).each do |file| + Dir.glob(params[:manga_path] + manga_data.site_dir + 'html/' + manga_data.name + HTML_utils::html_chapter_filename(chap[1], chap[0])).each do |file| puts 'deleting file : '.yellow + file File.delete(file) end diff --git a/sources/instructions/redl.rb b/sources/instructions/redl.rb index c3affe6..75b888e 100644 --- a/sources/instructions/redl.rb +++ b/sources/instructions/redl.rb @@ -6,7 +6,7 @@ def self.redl_volume(element, volume) if dw == nil return false end - links = dw.get_links + links = dw.links links.sort.each do |link| data = element.extract_values_from_link(link) if data[0] == volume @@ -34,7 +34,7 @@ def self.redl_chapter(element, chapter, volume) if dw == nil return false end - links = dw.get_links + links = dw.links links.each do |link| data = element.extract_values_from_link(link) if data[0] == volume && data[1] == chapter @@ -59,14 +59,11 @@ def self.redl_page(element, page, chapter, volume) if dw == nil return false end - links = dw.get_links + links = dw.links links.each do |link| data = element.extract_values_from_link(link) if data[0] == volume && data[1] == chapter new_link = dw.link_generator(volume, chapter, page) - if Utils_connection::redirection_detection(new_link) - break - end failure = false puts "downloading page #{page} of chapter #{chapter}" + ((volume == -1) ? '' : ((volume == -2) ? ' of volume TBD' : " of volume #{volume}")) dw.page_link(new_link, element.extract_values_from_link(new_link)) diff --git a/sources/scan/scan.rb b/sources/scan/scan.rb index 28b87fc..de5ef90 100644 --- a/sources/scan/scan.rb +++ b/sources/scan/scan.rb @@ -34,7 +34,7 @@ def scan(db, mode) # todo => must be able to use files ( -f option ) exit 4 end puts 'getting chapter list from site' - chap_list = Download_mf.new(db, name, false).get_links + chap_list = Download_mf.new(db, name, false).links dir = params.get_params[1] + site + '/' + name + '/' Dir.chdir(dir) if mode == 'add' @@ -45,7 +45,7 @@ def scan(db, mode) # todo => must be able to use files ( -f option ) puts 'correcting scan' scan_correct(db, name, dir, chap_list) else - puts 'critical error : unkown scan mode : ' + mode + puts 'critical error : unknown scan mode : ' + mode end =end end diff --git a/sources/scan/scan_utils.rb b/sources/scan/scan_utils.rb index f4e697e..f2e31ed 100644 --- a/sources/scan/scan_utils.rb +++ b/sources/scan/scan_utils.rb @@ -1,3 +1,4 @@ +=begin # gets the value of the page / chapter / volume def value_extract(elem) if elem[0] != 'v' && elem[0] != 'c' && elem[0] != 'p' @@ -62,3 +63,4 @@ def scan_dir(dir, db, name) end data end +=end diff --git a/sources/templates/text/help.txt b/sources/templates/text/help.txt index 8ef0856..9ea15d2 100644 --- a/sources/templates/text/help.txt +++ b/sources/templates/text/help.txt @@ -158,7 +158,11 @@ ex : "crepuscule" => http://mangafox.me/manga/crepuscule_yamchi/ => "crepuscule_yamchi" information concerning the display of the download : + good ( completed chapters get placed in the trace database ) : . => downloaded a page , => downloaded a page ( multiple of 10 ) ; => downloaded a page ( multiple of 50 ) - X => could not download a page ( error, page placed in todo ) + errors ( get placed in the todo database ), small letter for a page, capital letter for a chapter + x => could not download the html page ( needed to get to the image ) + ! => could not download the image + r => got redirected when trying to connect to the link diff --git a/sources/templates/text/params.txt b/sources/templates/text/params.txt deleted file mode 100644 index 5f5698e..0000000 --- a/sources/templates/text/params.txt +++ /dev/null @@ -1,63 +0,0 @@ - -list of paramaters and values : - - -[data] - -manga path (mp) = #{params[1]} -folder where you would like your mangas downloaded - -delete diff (dd) = #{params[6]} -if ever durring an update a chapter is in the traces database but not in the chapter list, it is deleted ( or not ) -this can happen when a chapter is mistankingly uploaded to the wrong manga or the chapter list / organisation changed - - -[internet] - -between sleep (bs) = #{params[2]} -time between 2 requests on the site of the manga ( seconds ) - -failure sleep (fs) = #{params[3]} -time between 2 request failures ( seconds ) - -nb tries (nb) = #{params[4]} -the number of tries MangaScrap will do before putting the page in the todo database - -error sleep (es) = #{params[5]} -time between 2 errors - such as a conection loss - ( seconds ) -30 seconds or more is advised - - -[internal] - -catch exception (ce) = #{params[7]} -While downloading pages, there is a very very low possibility of Ruby raising a exception. This only happens once every 1500 -pages downloaded and only on old machines. -This option is to be set at true if you had a deadlock exception ( default value ) and false if you think this may cause -stability issues - - -[html] - -generate html (gh) = #{params[8]} -allows you to choose if you whant the html to be generated automaticaly or not - -html nsfw (hn) = #{params[9]} -allows you to choose if you wish to have the covers of NSFW ( Not Safe For Work ) displayed in the manga index ( if -nsfw is enabled, the covers will be replaced by the site's logo ) -use the nsfw data (nd) to choose yourself witch are the genres that are NSFW -true = display cover -false = display logo - -nsfw data (nd) = #{params[10]} -determines witch genres you determine as being NSFW, default is Mature, Ecchi, Smut and Adult -syntax is : write all the genres you wish to add at once separating them by a comma and a space ( ", " ) -be carefull to respect the capital letters -this option is only used when the html nsfw (hn) option is set to true - - -[term output] - -color text (ct) = #{params[11]} -allows you to choose between your default bold text output ( false ) and a very colored one ( true ) - diff --git a/sources/templates/text/params/download.txt b/sources/templates/text/params/download.txt new file mode 100644 index 0000000..50424b2 --- /dev/null +++ b/sources/templates/text/params/download.txt @@ -0,0 +1,27 @@ + +[download] + +[mp] (mp) = {mp} +folder where you would like your mangas downloaded + +[bs] (bs) = {bs} +time between 2 requests on the site of the manga ( seconds ) + +[fs] (fs) = {fs} +time between 2 request failures ( seconds ) + +[nbf] (nbf) = {nbf} +the number of tries MangaScrap will do before putting the page in the todo database + +[es] (es) = {es} +time between 2 errors - such as a connection loss - ( seconds ) +30 seconds or more is advised + +[cto] (cto) = {cto} +Allows you to decide how much time MangaScrap will try to connect to the site before stopping or trying again +Values range from 1 to 300 seconds + +[dt] (dt) = {dt} +Allows you to decide how much time MangaScrap will try to download the web page / comic page / ... +Warning : this time value includes the |cto| time +Values from 0 ( no timeout ) to 300 seconds diff --git a/sources/templates/text/params/html.txt b/sources/templates/text/params/html.txt new file mode 100644 index 0000000..6e9d90d --- /dev/null +++ b/sources/templates/text/params/html.txt @@ -0,0 +1,24 @@ + +[html] + +[agh] (agh) = {agh} +allows you to choose if you want the html to be generated automatically or not + +[ne] (ne) = {ne} +allows you to choose if you wish to have the covers of NSFW ( Not Safe For Work ) displayed in the manga index ( if +nsfw is enabled, the covers will be replaced by the site's logo ) +use the nsfw data (nd) to choose yourself witch are the genres that are NSFW +true = display cover +false = display logo + +[nc] (nc) = {nc} +determines witch genres you determine as being NSFW, default is Mature, Ecchi, Smut and Adult +syntax is : write all the genres you wish to add at once separating them by a comma and a space ( ", " ) +be careful to respect the capital letters +this option is only used when the html nsfw |ne| option is set to true + +[td] (td) = {td} +(not yet implemented) + +[nm] (nm) = {nm} +(not yet implemented) diff --git a/sources/templates/text/params/misc.txt b/sources/templates/text/params/misc.txt new file mode 100644 index 0000000..a4cfbfc --- /dev/null +++ b/sources/templates/text/params/misc.txt @@ -0,0 +1,15 @@ + +[misc] + +[ct] (ct) = {ct} +allows you to choose between a very boring and colorless text output ( false ) and a very colored one ( true ) + +[dd] (dd) = {dd} +if ever during an update a chapter is in the traces database but not in the chapter list, it is deleted ( or not ) +this can happen when a chapter is mistakenly uploaded to the wrong manga or the chapter list / organisation changed + +[ve] (ve) = {ve} +(not yet implemented) + +[cu] (cu) = {cu} +(not yet implemented) diff --git a/sources/templates/text/params/threads.txt b/sources/templates/text/params/threads.txt new file mode 100644 index 0000000..d583672 --- /dev/null +++ b/sources/templates/text/params/threads.txt @@ -0,0 +1,8 @@ + +[threads] + +[emt] (emt) = {emt} +(not yet implemented) + +[nt] (nt) = {nt} +(not yet implemented) diff --git a/sources/templates/web/site_index/site_index_template.css b/sources/templates/web/site_index/site_index_template.css index 21f7c5b..3e24f56 100644 --- a/sources/templates/web/site_index/site_index_template.css +++ b/sources/templates/web/site_index/site_index_template.css @@ -1,5 +1,6 @@ html { - background-color: #F2F2F2; + background-color: #F2F2F2; + font-size: 12px; } div.container { @@ -13,6 +14,11 @@ div.header, div.footer { background-color: black; clear: left; text-align: center; + margin-top: 30px; + margin-bottom: 30px; + width: 60%; + margin-left: 20%; + margin-right: 20%; } a { @@ -20,11 +26,11 @@ a { } div.list { - margin-left: 200px; - margin-right: 600px; + margin-left: 1%; + margin-right: 30%; border-left: 1px solid gray; border-right: 1px solid gray; - padding: 1em; + padding: 5px; overflow: hidden; min-height: 650px; text-align: justify; @@ -33,14 +39,18 @@ div.list { .manga { display: inline-block; text-align: center; - margin: 5px auto; - min-height: 180px; + margin-top: 5px; + margin-left: 5px; + margin-right: 5px; + min-height: 250px; overflow: hidden; + max-width: 160px; + vertical-align: top; } .manga img { - max-height: 300px; - max-width: 200px; + max-height: 200px; + max-width: 140px; width: auto; height: auto; } @@ -56,7 +66,7 @@ a { div#details { right: 0; top : 170px; - width: 580px; + width: 28%; margin-right: 15px; position: absolute; } diff --git a/sources/utils/utils_co.rb b/sources/utils/utils_co.rb index 185c796..1a918f2 100644 --- a/sources/utils/utils_co.rb +++ b/sources/utils/utils_co.rb @@ -1,17 +1,13 @@ -$nb_tries = -1 -$between_sleep = -1 -$failure_sleep = -1 -$error_sleep = -1 -$catch_fatal = 'false' +$download_params = nil module Utils_connection private # used to know hom much sleep time is needed def self.sleep_manager(error) if error.class.to_s == 'SocketError' # connection error - sleep($error_sleep) + sleep($download_params[:error_sleep]) else - sleep($failure_sleep) + sleep($download_params[:failure_sleep]) end end @@ -25,114 +21,59 @@ def self.download_rescue(tries, link, error, message, silent = false) unless silent print "\n" STDOUT.flush - puts message + ' ' + link + ' after ' + $nb_tries.to_s + ' tries' - puts 'message is : ' + error.message + puts message + ' ' + link + ' after ' + $download_params[:nb_tries_on_fail].to_s + ' tries' + puts 'message is : ' + error end nil end end - # exception manager for the utils_co.rb functions - def self.rescue_fatal(error, silent = false) - if $catch_fatal == 'false' - unless silent - print "\n" - STDOUT.flush - puts 'Warning : exception occured, message is : ' + error.message - puts 'Exception class is : ' + error.class.to_s - puts 'raising it again' - puts '' - end - raise error - else - if error.class.to_s != 'fatal' - raise error - end - end - end - public # initialises the global variables def self.init_utils - params = Params.instance.get_params - $between_sleep = params[2] - $failure_sleep = params[3] - $nb_tries = params[4] - $error_sleep = params[5] - $catch_fatal = params[7] + $download_params = Params.instance.download end - # detects if there was a redirection on the required link - def self.redirection_detection(url) - tries ||= $nb_tries - begin - open(url) do |resp| - if resp.base_uri.to_s != url - return true + def self.download(link, silent, error_message) + tries ||= $download_params[:nb_tries_on_fail] + while tries != 0 + sleep($download_params[:between_sleep]) + request = Typhoeus::Request.new(link, accept_encoding: 'gzip', connecttimeout: $download_params[:connect_timeout], + timeout: $download_params[:download_timeout], followlocation: false) + request.on_complete do |response| + if response.success? + return response.body + elsif response.timed_out? # time out + tries = download_rescue(tries, link, 'time out', error_message, silent) + if tries == nil + break + end + elsif response.code == 302 # redirection + raise 'redirection' + else # all other errors + tries = download_rescue(tries, link, "http code error #{response.code}", error_message, silent) + if tries == nil + break + end end end - rescue StandardError => error - if tries > 0 - tries -= 1 - sleep_manager(error) - retry - else - puts 'connection is lost or could not find manga, stopping programm' - puts url - puts 'message is : ' + error.message - exit 3 - end - rescue Exception => error - rescue_fatal(error) + request.run end - false + return nil end # connects to link and download page def self.get_page(link, silent = false) - tries ||= $nb_tries - begin - html = open(link, 'User-Agent' => "Ruby/#{RUBY_VERSION}") - page = Nokogiri::HTML(html) do |nokogiri| - nokogiri.noblanks.noerror - end - rescue StandardError => error - tries = download_rescue(tries, link, error, 'could not download picture', silent) - if tries == nil - return nil - end - retry - rescue Exception => error - rescue_fatal(error) + html = download(link, silent, 'could not download picture') + if html == nil + return nil end - sleep($between_sleep) - page + Nokogiri::HTML(html, 'utf-8') end # connects to link and download picture def self.get_pic(link, silent = false) - safe_link = link.gsub(/[\[\]]/) { '%%%s' % $&.ord.to_s(16) } - tries ||= $nb_tries - begin - page = open(safe_link, 'User-Agent' => "Ruby/#{RUBY_VERSION}") - rescue URI::InvalidURIError => error - unless silent - puts 'Warning : bad url' - puts link - puts 'message is : ' + error.message - end - return nil - rescue StandardError => error - tries = download_rescue(tries, link, error, 'could not download picture', silent) - if tries == nil - return nil - end - retry - rescue Exception => error - rescue_fatal(error, silent) - end - sleep($between_sleep) - page + download(link, silent, 'could not download picture') end # extracts the cover link, downloads it and writes it twice @@ -146,15 +87,12 @@ def self.write_cover(doc, xpath, path1, path2) cover_buffer = File.open('./pictures/other/404.png') end if cover_buffer != nil - cover1 = File.new(path1, 'wb') - cover2 = File.new(path2, 'wb') - until cover_buffer.eof? - chunk = cover_buffer.read(1024) - cover1.write(chunk) - cover2.write(chunk) + File.open(path1, 'wb') do |pic| + pic << cover_buffer + end + File.open(path2, 'wb') do |pic| + pic << cover_buffer end - cover1.close - cover2.close else puts 'WARNING : cover could not download cover' end diff --git a/sources/utils/utils_db.rb b/sources/utils/utils_db.rb index 988b84a..39f0d1f 100644 --- a/sources/utils/utils_db.rb +++ b/sources/utils/utils_db.rb @@ -1,5 +1,9 @@ module Utils_database # tries and caches an sql request + # request = the string that is executed + # error = the error message + # db = the database + # args = the (optional) array of arguments def self.db_exec(request, error, db, args = []) begin ret = db.execute request, args diff --git a/sources/utils/utils_file.rb b/sources/utils/utils_file.rb index 07edaaf..74b4235 100644 --- a/sources/utils/utils_file.rb +++ b/sources/utils/utils_file.rb @@ -95,7 +95,7 @@ def self.write_pic(pic_buffer, data, dir) if pic_buffer != nil begin File.open(name_buffer + '.jpg', 'wb') do |pic| - pic << pic_buffer.read + pic << pic_buffer end if File.exist?(name_buffer + '.txt') File.delete(name_buffer + '.txt') @@ -132,8 +132,8 @@ def self.delete_files(path, extension) end # used for the description.txt file of every manga - def self.data_concatenation(manga_name, description, site, link, author, artist, type, status, genres, release, html_name, alternative_names) - ret = 'name = ' + manga_name + "\n" + def self.data_concatenation(manga_data, description, author, artist, type, status, genres, release, html_name, alternative_names) + ret = 'name = ' + manga_data.name + "\n" ret += 'html name = ' + html_name + "\n" ret += 'other names = ' + alternative_names + "\n" ret += 'author = ' + author + "\n" @@ -143,8 +143,8 @@ def self.data_concatenation(manga_name, description, site, link, author, artist, ret += 'status = ' + status + "\n" ret += 'genres = ' + genres + "\n" ret += "\n" - ret += 'site = ' + site + "\n" - ret += 'link = ' + link + "\n" + ret += 'site = ' + manga_data.site + "\n" + ret += 'link = ' + manga_data.link + "\n" ret += "\n" ret += "description :\n" ret += "\n" diff --git a/tools/file_renamer.rb b/tools/file_renamer.rb index 6550fa6..61b128b 100755 --- a/tools/file_renamer.rb +++ b/tools/file_renamer.rb @@ -6,7 +6,12 @@ require 'singleton' require 'sqlite3' -require_relative '../Classes/DB/params_db' +require_relative '../sources/DB/Params' +require_relative '../sources/DB/sub_params/params_module' +require_relative '../sources/DB/sub_params/Download' +require_relative '../sources/DB/sub_params/HTML' +require_relative '../sources/DB/sub_params/Misc' +require_relative '../sources/DB/sub_params/Threads' if ARGV.size != 2 puts 'this tool needs 2 arguments' @@ -34,8 +39,8 @@ def confirm_delete (pattern_to_replace, replacing_pattern) exit 0 end -params = Params.instance.get_params -dirs = Dir[params[1] + '/mangafox/mangas/*'] +# todo, enable the multi_sites +dirs = Dir[Params.instance.download[:manga_path] + '/mangafox/mangas/*'] dirs.delete_if {|dir| dir.include?('.jpg')} dirs.sort.each do |dir| puts dir