From a043bcf9dd268cec5c9bdfc09dab7fcb0fc33644 Mon Sep 17 00:00:00 2001 From: Matthias H Date: Tue, 18 Feb 2014 19:25:17 +0100 Subject: [PATCH] removes bdb&tc registry; adds dbm registry This removes the existing registries, ruby-bdb (Berkeley DB) and tokyocabinet, instead it creates a new registry based on ruby DBM. Ruby-bdb has long since been abandoned in favor for rubys DBM and tokyocabinet also has been ceased development since 2011 (for kyotocabinet). --- lib/rbot/ircbot.rb | 36 +-- lib/rbot/plugins.rb | 4 - lib/rbot/registry/bdb.rb | 562 -------------------------------------- lib/rbot/registry/dbm.rb | 276 +++++++++++++++++++ lib/rbot/registry/tc.rb | 573 --------------------------------------- 5 files changed, 282 insertions(+), 1169 deletions(-) delete mode 100644 lib/rbot/registry/bdb.rb create mode 100644 lib/rbot/registry/dbm.rb delete mode 100644 lib/rbot/registry/tc.rb diff --git a/lib/rbot/ircbot.rb b/lib/rbot/ircbot.rb index 06a91c4d..ed3c910c 100644 --- a/lib/rbot/ircbot.rb +++ b/lib/rbot/ircbot.rb @@ -186,11 +186,6 @@ class Bot # TODO multiserver attr_reader :socket - # bot's object registry, plugins get an interface to this for persistant - # storage (hash interface tied to a db file, plugins use Accessors to store - # and restore objects in their own namespaces.) - attr_reader :registry - # bot's plugins. This is an instance of class Plugins attr_reader :plugins @@ -431,11 +426,11 @@ class Bot }, :desc => "Percentage of IRC penalty to consider when sending messages to prevent being disconnected for excess flood. Set to 0 to disable penalty control.") Config.register Config::StringValue.new('core.db', - :default => "bdb", - :wizard => true, :default => "bdb", - :validate => Proc.new { |v| ["bdb", "tc"].include? v }, + :default => "dbm", + :wizard => true, :default => "dbm", + :validate => Proc.new { |v| ["dbm"].include? v }, :requires_restart => true, - :desc => "DB adaptor to use for storing settings and plugin data. Options are: bdb (Berkeley DB, stable adaptor, but troublesome to install and unmaintained), tc (Tokyo Cabinet, new adaptor, fast and furious, but may be not available and contain bugs)") + :desc => "DB adaptor to use for storing the plugin data/registries. Options: dbm (included in ruby)") @argv = params[:argv] @run_dir = params[:run_dir] || Dir.pwd @@ -506,10 +501,8 @@ class Bot end case @config["core.db"] - when "bdb" - require 'rbot/registry/bdb' - when "tc" - require 'rbot/registry/tc' + when "dbm" + require 'rbot/registry/dbm' else raise _("Unknown DB adaptor: %s") % @config["core.db"] end @@ -581,8 +574,6 @@ class Bot pf << "#{$$}\n" end - @registry = Registry.new self - @timer = Timer.new @save_mutex = Mutex.new if @config['core.save_every'] > 0 @@ -1026,11 +1017,6 @@ class Bot rescue SystemExit log_session_end exit 0 - rescue DBFatal => e - fatal "fatal db error: #{e.pretty_inspect}" - DBTree.stats - log_session_end - exit 2 rescue Exception => e error e will_wait = true @@ -1109,13 +1095,6 @@ class Bot log "Killed by server, extra delay multiplier #{oldtf} -> #{too_fast}" end retry - rescue DBFatal => e - fatal "fatal db error: #{e.pretty_inspect}" - DBTree.stats - # Why restart? DB problems are serious stuff ... - # restart("Oops, we seem to have registry problems ...") - log_session_end - exit 2 rescue Exception => e error "non-net exception: #{e.pretty_inspect}" quit_msg = e.to_s @@ -1359,8 +1338,6 @@ class Bot # @timer.stop # debug "Closing registries" # @registry.close - debug "\t\tcleaning up the db environment ..." - DBTree.cleanup_env log "rbot quit (#{message})" end end @@ -1402,7 +1379,6 @@ class Bot def save @save_mutex.synchronize do @plugins.save - DBTree.cleanup_logs end end diff --git a/lib/rbot/plugins.rb b/lib/rbot/plugins.rb index 493f177f..a05a5b8f 100644 --- a/lib/rbot/plugins.rb +++ b/lib/rbot/plugins.rb @@ -573,7 +573,6 @@ module Plugins return :loaded rescue Exception => err # rescue TimeoutError, StandardError, NameError, LoadError, SyntaxError => err - raise if err.kind_of? DBFatal error report_error("#{desc}#{fname} load failed", err) bt = err.backtrace.select { |line| line.match(/^(\(eval\)|#{fname}):\d+/) @@ -956,7 +955,6 @@ module Plugins rescue Exception => err raise if err.kind_of?(SystemExit) error report_error("#{p.botmodule_class} #{p.name} #{method}() failed:", err) - raise if err.kind_of?(DBFatal) end } else @@ -972,7 +970,6 @@ module Plugins rescue Exception => err raise if err.kind_of?(SystemExit) error report_error("#{p.botmodule_class} #{p.name} #{method}() failed:", err) - raise if err.kind_of?(DBFatal) end end } @@ -1002,7 +999,6 @@ module Plugins rescue Exception => err raise if err.kind_of?(SystemExit) error report_error("#{p.botmodule_class} #{p.name} privmsg() failed:", err) - raise if err.kind_of?(DBFatal) end debug "Successfully delegated #{m.inspect}" return true diff --git a/lib/rbot/registry/bdb.rb b/lib/rbot/registry/bdb.rb deleted file mode 100644 index 09a2579f..00000000 --- a/lib/rbot/registry/bdb.rb +++ /dev/null @@ -1,562 +0,0 @@ -#-- vim:sw=2:et -#++ -# -# :title: Berkeley DB interface - -begin - require 'bdb' -rescue LoadError - fatal "rbot couldn't load the bdb module, perhaps you need to install it? try http://www.ruby-lang.org/en/raa-list.rhtml?name=bdb or http://github.com/knu/ruby-bdb" -rescue Exception => e - fatal "rbot couldn't load the bdb module: #{e.pretty_inspect}" -end - -if not defined? BDB - fatal "A bdb module was found, but it doesn't seem to be one I know about, sorry" -end - -module Irc - DBFatal = BDB::Fatal -end - -if BDB::VERSION_MAJOR < 4 - fatal "Your bdb (Berkeley DB) version #{BDB::VERSION} is too old!" - fatal "rbot will only run with bdb version 4 or higher, please upgrade." - fatal "For maximum reliability, upgrade to version 4.2 or higher." - raise BDB::Fatal, BDB::VERSION + " is too old" -end - -if BDB::VERSION_MAJOR == 4 and BDB::VERSION_MINOR < 2 - warning "Your bdb (Berkeley DB) version #{BDB::VERSION} may not be reliable." - warning "If possible, try upgrade version 4.2 or later." -end - -# make BTree lookups case insensitive -module BDB - class CIBtree < Btree - def bdb_bt_compare(a, b) - if a == nil || b == nil - warning "CIBTree: comparing key #{a.inspect} with #{b.inspect}" - end - (a||'').downcase <=> (b||'').downcase - end - end -end - -module Irc - - # DBHash is for tying a hash to disk (using bdb). - # Call it with an identifier, for example "mydata". It'll look for - # mydata.db, if it exists, it will load and reference that db. - # Otherwise it'll create and empty db called mydata.db - class DBHash - - # absfilename:: use +key+ as an actual filename, don't prepend the bot's - # config path and don't append ".db" - def initialize(bot, key, absfilename=false) - @bot = bot - @key = key - relfilename = @bot.path key - relfilename << '.db' - if absfilename && File.exist?(key) - # db already exists, use it - @db = DBHash.open_db(key) - elsif absfilename - # create empty db - @db = DBHash.create_db(key) - elsif File.exist? relfilename - # db already exists, use it - @db = DBHash.open_db relfilename - else - # create empty db - @db = DBHash.create_db relfilename - end - end - - def method_missing(method, *args, &block) - return @db.send(method, *args, &block) - end - - def DBHash.create_db(name) - debug "DBHash: creating empty db #{name}" - return BDB::Hash.open(name, nil, - BDB::CREATE | BDB::EXCL, 0600) - end - - def DBHash.open_db(name) - debug "DBHash: opening existing db #{name}" - return BDB::Hash.open(name, nil, "r+", 0600) - end - - end - - - # DBTree is a BTree equivalent of DBHash, with case insensitive lookups. - class DBTree - @@env=nil - # TODO: make this customizable - # Note that it must be at least four times lg_bsize - @@lg_max = 8*1024*1024 - # absfilename:: use +key+ as an actual filename, don't prepend the bot's - # config path and don't append ".db" - def initialize(bot, key, absfilename=false) - @bot = bot - @key = key - if @@env.nil? - begin - @@env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER, "set_lg_max" => @@lg_max) - debug "DBTree: environment opened with max log size #{@@env.conf['lg_max']}" - rescue => e - debug "DBTree: failed to open environment: #{e.pretty_inspect}. Retrying ..." - @@env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER) - end - #@@env = BDB::Env.open(@bot.botclass, BDB::CREATE | BDB::INIT_MPOOL | BDB::RECOVER) - end - - relfilename = @bot.path key - relfilename << '.db' - - if absfilename && File.exist?(key) - # db already exists, use it - @db = DBTree.open_db(key) - elsif absfilename - # create empty db - @db = DBTree.create_db(key) - elsif File.exist? relfilename - # db already exists, use it - @db = DBTree.open_db relfilename - else - # create empty db - @db = DBTree.create_db relfilename - end - end - - def method_missing(method, *args, &block) - return @db.send(method, *args, &block) - end - - def DBTree.create_db(name) - debug "DBTree: creating empty db #{name}" - return @@env.open_db(BDB::CIBtree, name, nil, BDB::CREATE | BDB::EXCL, 0600) - end - - def DBTree.open_db(name) - debug "DBTree: opening existing db #{name}" - return @@env.open_db(BDB::CIBtree, name, nil, "r+", 0600) - end - - def DBTree.cleanup_logs() - if @@env.nil? - debug "No BDB environment, skipping checkpoint" - return - end - begin - debug "DBTree: checkpointing ..." - @@env.checkpoint - rescue Exception => e - debug "Failed: #{e.pretty_inspect}" - end - begin - debug "DBTree: flushing log ..." - @@env.log_flush - logs = @@env.log_archive(BDB::ARCH_ABS) - debug "DBTree: deleting archivable logs: #{logs.join(', ')}." - logs.each { |log| - File.delete(log) - } - rescue Exception => e - debug "Failed: #{e.pretty_inspect}" - end - end - - def DBTree.stats() - if @@env.nil? - debug "No BDB environment, no stats" - return - end - begin - debug "General stats:" - debug @@env.stat - debug "Lock stats:" - debug @@env.lock_stat - debug "Log stats:" - debug @@env.log_stat - debug "Txn stats:" - debug @@env.txn_stat - rescue - debug "Couldn't dump stats" - end - end - - def DBTree.cleanup_env() - if @@env.nil? - debug "No BDB environment, skipping cleanup" - return - end - begin - debug "DBTree: checking transactions ..." - has_active_txn = @@env.txn_stat["st_nactive"] > 0 - if has_active_txn - warning "DBTree: not all transactions completed!" - end - DBTree.cleanup_logs - debug "DBTree: closing environment #{@@env}" - path = @@env.home - @@env.close - @@env = nil - if has_active_txn - debug "DBTree: keeping file because of incomplete transactions" - else - debug "DBTree: cleaning up environment in #{path}" - BDB::Env.remove("#{path}") - end - rescue Exception => e - error "failed to clean up environment: #{e.pretty_inspect}" - end - end - - end - -end - - -module Irc -class Bot - - # This class is now used purely for upgrading from prior versions of rbot - # the new registry is split into multiple DBHash objects, one per plugin - class Registry - def initialize(bot) - @bot = bot - upgrade_data - upgrade_data2 - end - - # check for older versions of rbot with data formats that require updating - # NB this function is called _early_ in init(), pretty much all you have to - # work with is @bot.botclass. - def upgrade_data - oldreg = @bot.path 'registry.db' - newreg = @bot.path 'plugin_registry.db' - if File.exist?(oldreg) - log _("upgrading old-style (rbot 0.9.5 or earlier) plugin registry to new format") - old = BDB::Hash.open(oldreg, nil, "r+", 0600) - new = BDB::CIBtree.open(newreg, nil, BDB::CREATE | BDB::EXCL, 0600) - old.each {|k,v| - new[k] = v - } - old.close - new.close - File.rename(oldreg, oldreg + ".old") - end - end - - def upgrade_data2 - oldreg = @bot.path 'plugin_registry.db' - newdir = @bot.path 'registry' - if File.exist?(oldreg) - Dir.mkdir(newdir) unless File.exist?(newdir) - env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER)# | BDB::TXN_NOSYNC) - dbs = Hash.new - log _("upgrading previous (rbot 0.9.9 or earlier) plugin registry to new split format") - old = BDB::CIBtree.open(oldreg, nil, "r+", 0600, "env" => env) - old.each {|k,v| - prefix,key = k.split("/", 2) - prefix.downcase! - # subregistries were split with a +, now they are in separate folders - if prefix.gsub!(/\+/, "/") - # Ok, this code needs to be put in the db opening routines - dirs = File.dirname("#{@bot.botclass}/registry/#{prefix}.db").split("/") - dirs.length.times { |i| - dir = dirs[0,i+1].join("/")+"/" - unless File.exist?(dir) - log _("creating subregistry directory #{dir}") - Dir.mkdir(dir) - end - } - end - unless dbs.has_key?(prefix) - log _("creating db #{@bot.botclass}/registry/#{prefix}.db") - dbs[prefix] = BDB::CIBtree.open("#{@bot.botclass}/registry/#{prefix}.db", - nil, BDB::CREATE | BDB::EXCL, - 0600, "env" => env) - end - dbs[prefix][key] = v - } - old.close - File.rename(oldreg, oldreg + ".old") - dbs.each {|k,v| - log _("closing db #{k}") - v.close - } - env.close - end - end - - - # This class provides persistent storage for plugins via a hash interface. - # The default mode is an object store, so you can store ruby objects and - # reference them with hash keys. This is because the default store/restore - # methods of the plugins' RegistryAccessor are calls to Marshal.dump and - # Marshal.restore, - # for example: - # blah = Hash.new - # blah[:foo] = "fum" - # @registry[:blah] = blah - # then, even after the bot is shut down and disconnected, on the next run you - # can access the blah object as it was, with: - # blah = @registry[:blah] - # The registry can of course be used to store simple strings, fixnums, etc as - # well, and should be useful to store or cache plugin data or dynamic plugin - # configuration. - # - # WARNING: - # in object store mode, don't make the mistake of treating it like a live - # object, e.g. (using the example above) - # @registry[:blah][:foo] = "flump" - # will NOT modify the object in the registry - remember that Registry#[] - # returns a Marshal.restore'd object, the object you just modified in place - # will disappear. You would need to: - # blah = @registry[:blah] - # blah[:foo] = "flump" - # @registry[:blah] = blah - # - # If you don't need to store objects, and strictly want a persistant hash of - # strings, you can override the store/restore methods to suit your needs, for - # example (in your plugin): - # def initialize - # class << @registry - # def store(val) - # val - # end - # def restore(val) - # val - # end - # end - # end - # Your plugins section of the registry is private, it has its own namespace - # (derived from the plugin's class name, so change it and lose your data). - # Calls to registry.each etc, will only iterate over your namespace. - class Accessor - - attr_accessor :recovery - - # plugins don't call this - a Registry::Accessor is created for them and - # is accessible via @registry. - def initialize(bot, name) - @bot = bot - @name = name.downcase - @filename = @bot.path 'registry', @name - dirs = File.dirname(@filename).split("/") - dirs.length.times { |i| - dir = dirs[0,i+1].join("/")+"/" - unless File.exist?(dir) - debug "creating subregistry directory #{dir}" - Dir.mkdir(dir) - end - } - @filename << ".db" - @registry = nil - @default = nil - @recovery = nil - # debug "initializing registry accessor with name #{@name}" - end - - def registry - @registry ||= DBTree.new @bot, "registry/#{@name}" - end - - def flush - # debug "fushing registry #{registry}" - return if !@registry - registry.flush - registry.sync - end - - def close - # debug "closing registry #{registry}" - return if !@registry - registry.close - @registry = nil - end - - # convert value to string form for storing in the registry - # defaults to Marshal.dump(val) but you can override this in your module's - # registry object to use any method you like. - # For example, if you always just handle strings use: - # def store(val) - # val - # end - def store(val) - Marshal.dump(val) - end - - # restores object from string form, restore(store(val)) must return val. - # If you override store, you should override restore to reverse the - # action. - # For example, if you always just handle strings use: - # def restore(val) - # val - # end - def restore(val) - begin - Marshal.restore(val) - rescue Exception => e - error _("failed to restore marshal data for #{val.inspect}, attempting recovery or fallback to default") - debug e - if defined? @recovery and @recovery - begin - return @recovery.call(val) - rescue Exception => ee - error _("marshal recovery failed, trying default") - error ee - end - end - return default - end - end - - # lookup a key in the registry - def [](key) - if File.exist?(@filename) && registry.has_key?(key) - return restore(registry[key]) - else - return default - end - end - - # set a key in the registry - def []=(key,value) - registry[key] = store(value) - end - - # set the default value for registry lookups, if the key sought is not - # found, the default will be returned. The default default (har) is nil. - def set_default (default) - @default = default - end - - def default - @default && (@default.dup rescue @default) - end - - # just like Hash#each - def each(set=nil, bulk=0, &block) - return nil unless File.exist?(@filename) - registry.each(set, bulk) {|key,value| - block.call(key, restore(value)) - } - end - - # just like Hash#each_key - def each_key(set=nil, bulk=0, &block) - return nil unless File.exist?(@filename) - registry.each_key(set, bulk) {|key| - block.call(key) - } - end - - # just like Hash#each_value - def each_value(set=nil, bulk=0, &block) - return nil unless File.exist?(@filename) - registry.each_value(set, bulk) { |value| - block.call(restore(value)) - } - end - - # just like Hash#has_key? - def has_key?(key) - return false unless File.exist?(@filename) - return registry.has_key?(key) - end - alias include? has_key? - alias member? has_key? - alias key? has_key? - - # just like Hash#has_both? - def has_both?(key, value) - return false unless File.exist?(@filename) - return registry.has_both?(key, store(value)) - end - - # just like Hash#has_value? - def has_value?(value) - return false unless File.exist?(@filename) - return registry.has_value?(store(value)) - end - - # just like Hash#index? - def index(value) - return nil unless File.exist?(@filename) - ind = registry.index(store(value)) - if ind - return ind - else - return nil - end - end - - # delete a key from the registry - def delete(key) - return default unless File.exist?(@filename) - return registry.delete(key) - end - - # returns a list of your keys - def keys - return [] unless File.exist?(@filename) - return registry.keys - end - - # Return an array of all associations [key, value] in your namespace - def to_a - return [] unless File.exist?(@filename) - ret = Array.new - registry.each {|key, value| - ret << [key, restore(value)] - } - return ret - end - - # Return an hash of all associations {key => value} in your namespace - def to_hash - return {} unless File.exist?(@filename) - ret = Hash.new - registry.each {|key, value| - ret[key] = restore(value) - } - return ret - end - - # empties the registry (restricted to your namespace) - def clear - return true unless File.exist?(@filename) - registry.clear - end - alias truncate clear - - # returns an array of the values in your namespace of the registry - def values - return [] unless File.exist?(@filename) - ret = Array.new - self.each {|k,v| - ret << restore(v) - } - return ret - end - - def sub_registry(prefix) - return Accessor.new(@bot, @name + "/" + prefix.to_s) - end - - # returns the number of keys in your registry namespace - def length - self.keys.length - end - alias size length - - end - - end -end -end diff --git a/lib/rbot/registry/dbm.rb b/lib/rbot/registry/dbm.rb new file mode 100644 index 00000000..fb684d5f --- /dev/null +++ b/lib/rbot/registry/dbm.rb @@ -0,0 +1,276 @@ +#-- vim:sw=2:et +#++ +# +# The DBM class of the ruby std-lib provides wrappers for Unix-style +# dbm or Database Manager libraries. The exact library used depends +# on how ruby was compiled. Its any of the following: ndbm, bdb, +# gdbm or qdbm. +# DBM API Documentation: +# http://ruby-doc.org/stdlib-2.1.0/libdoc/dbm/rdoc/DBM.html +# +# :title: DB interface + +require 'dbm' + +module Irc +class Bot +class Registry + + # This class provides persistent storage for plugins via a hash interface. + # The default mode is an object store, so you can store ruby objects and + # reference them with hash keys. This is because the default store/restore + # methods of the plugins' RegistryAccessor are calls to Marshal.dump and + # Marshal.restore, + # for example: + # blah = Hash.new + # blah[:foo] = "fum" + # @registry[:blah] = blah + # then, even after the bot is shut down and disconnected, on the next run you + # can access the blah object as it was, with: + # blah = @registry[:blah] + # The registry can of course be used to store simple strings, fixnums, etc as + # well, and should be useful to store or cache plugin data or dynamic plugin + # configuration. + # + # WARNING: + # in object store mode, don't make the mistake of treating it like a live + # object, e.g. (using the example above) + # @registry[:blah][:foo] = "flump" + # will NOT modify the object in the registry - remember that Registry#[] + # returns a Marshal.restore'd object, the object you just modified in place + # will disappear. You would need to: + # blah = @registry[:blah] + # blah[:foo] = "flump" + # @registry[:blah] = blah + # + # If you don't need to store objects, and strictly want a persistant hash of + # strings, you can override the store/restore methods to suit your needs, for + # example (in your plugin): + # def initialize + # class << @registry + # def store(val) + # val + # end + # def restore(val) + # val + # end + # end + # end + # Your plugins section of the registry is private, it has its own namespace + # (derived from the plugin's class name, so change it and lose your data). + # Calls to registry.each etc, will only iterate over your namespace. + class Accessor + + attr_accessor :recovery + + # plugins don't call this - a Registry::Accessor is created for them and + # is accessible via @registry. + def initialize(bot, name) + @bot = bot + @name = name.downcase + @filename = @bot.path 'registry', @name + dirs = File.dirname(@filename).split("/") + dirs.length.times { |i| + dir = dirs[0,i+1].join("/")+"/" + unless File.exist?(dir) + debug "creating subregistry directory #{dir}" + Dir.mkdir(dir) + end + } + @registry = nil + @default = nil + @recovery = nil + # debug "initializing registry accessor with name #{@name}" + end + + def registry + #debug 'access registry (cached)' if @registry + #@registry ||= DBM.open(@filename, 0666, DBM::WRCREAT) + if not @registry + debug 'open registry, ' + @filename + @registry = DBM.open(@filename, 0666, DBM::WRCREAT) + end + @registry + end + + def flush + return if !@registry + # ruby dbm has no flush, so we close/reopen :( + close + registry + end + + def close + return if !@registry + registry.close + @registry = nil + end + + # convert value to string form for storing in the registry + # defaults to Marshal.dump(val) but you can override this in your module's + # registry object to use any method you like. + # For example, if you always just handle strings use: + # def store(val) + # val + # end + def store(val) + Marshal.dump(val) + end + + # restores object from string form, restore(store(val)) must return val. + # If you override store, you should override restore to reverse the + # action. + # For example, if you always just handle strings use: + # def restore(val) + # val + # end + def restore(val) + begin + Marshal.restore(val) + rescue Exception => e + error _("failed to restore marshal data for #{val.inspect}, attempting recovery or fallback to default") + debug e + if defined? @recovery and @recovery + begin + return @recovery.call(val) + rescue Exception => ee + error _("marshal recovery failed, trying default") + debug ee + end + end + return default + end + end + + # lookup a key in the registry + def [](key) + if registry.has_key?(key.to_s) + return restore(registry[key.to_s]) + else + return default + end + end + + # set a key in the registry + def []=(key,value) + registry[key.to_s] = store(value) + end + + # set the default value for registry lookups, if the key sought is not + # found, the default will be returned. The default default (har) is nil. + def set_default (default) + @default = default + end + + def default + @default && (@default.dup rescue @default) + end + + # like Hash#each + def each(&block) + registry.each_key do |key| + block.call(key, self[key]) + end + end + + alias each_pair each + + # like Hash#each_key + def each_key(&block) + registry.each_key do |key| + block.call(key) + end + end + + # like Hash#each_value + def each_value(&block) + registry.each_key do |key| + block.call(self[key]) + end + end + + # just like Hash#has_key? + def has_key?(key) + return registry.has_key?(key.to_s) + end + + alias include? has_key? + alias member? has_key? + alias key? has_key? + + # just like Hash#has_both? + def has_both?(key, value) + registry.has_key?(key.to_s) and registry.has_value?(store(value)) + end + + # just like Hash#has_value? + def has_value?(value) + return registry.has_value?(store(value)) + end + + # just like Hash#index? + def index(value) + self.each do |k,v| + return k if v == value + end + return nil + end + + # delete a key from the registry + def delete(key) + return registry.delete(key.to_s) + end + + # returns a list of your keys + def keys + return registry.keys + end + + # Return an array of all associations [key, value] in your namespace + def to_a + ret = Array.new + registry.each {|key, value| + ret << [key, restore(value)] + } + return ret + end + + # Return an hash of all associations {key => value} in your namespace + def to_hash + ret = Hash.new + registry.each {|key, value| + ret[key] = restore(value) + } + return ret + end + + # empties the registry (restricted to your namespace) + def clear + registry.clear + end + alias truncate clear + + # returns an array of the values in your namespace of the registry + def values + ret = Array.new + self.each {|k,v| + ret << restore(v) + } + return ret + end + + def sub_registry(prefix) + return Accessor.new(@bot, @name + "/" + prefix.to_s) + end + + # returns the number of keys in your registry namespace + def length + registry.length + end + alias size length + end + +end # Registry +end # Bot +end # Irc + diff --git a/lib/rbot/registry/tc.rb b/lib/rbot/registry/tc.rb deleted file mode 100644 index 06013f8f..00000000 --- a/lib/rbot/registry/tc.rb +++ /dev/null @@ -1,573 +0,0 @@ -#-- vim:sw=2:et -#++ -# -# :title: DB interface - -begin - require 'bdb' - if BDB::VERSION_MAJOR < 4 - fatal "Your bdb (Berkeley DB) version #{BDB::VERSION} is too old!" - fatal "rbot will only run with bdb version 4 or higher, please upgrade." - fatal "For maximum reliability, upgrade to version 4.2 or higher." - raise BDB::Fatal, BDB::VERSION + " is too old" - end - - if BDB::VERSION_MAJOR == 4 and BDB::VERSION_MINOR < 2 - warning "Your bdb (Berkeley DB) version #{BDB::VERSION} may not be reliable." - warning "If possible, try upgrade version 4.2 or later." - end -rescue LoadError - warning "rbot couldn't load the bdb module. Old registries won't be upgraded" -rescue Exception => e - warning "rbot couldn't load the bdb module: #{e.pretty_inspect}" -end - - - - -require 'tokyocabinet' - -module Irc - - class DBFatal < Exception ; end - - if defined? BDB - # DBHash is for tying a hash to disk (using bdb). - # Call it with an identifier, for example "mydata". It'll look for - # mydata.db, if it exists, it will load and reference that db. - # Otherwise it'll create and empty db called mydata.db - class DBHash - - # absfilename:: use +key+ as an actual filename, don't prepend the bot's - # config path and don't append ".db" - def initialize(bot, key, absfilename=false) - @bot = bot - @key = key - relfilename = @bot.path key - relfilename << '.db' - if absfilename && File.exist?(key) - # db already exists, use it - @db = DBHash.open_db(key) - elsif absfilename - # create empty db - @db = DBHash.create_db(key) - elsif File.exist? relfilename - # db already exists, use it - @db = DBHash.open_db relfilename - else - # create empty db - @db = DBHash.create_db relfilename - end - end - - def method_missing(method, *args, &block) - return @db.send(method, *args, &block) - end - - def DBHash.create_db(name) - debug "DBHash: creating empty db #{name}" - return BDB::Hash.open(name, nil, - BDB::CREATE | BDB::EXCL, 0600) - end - - def DBHash.open_db(name) - debug "DBHash: opening existing db #{name}" - return BDB::Hash.open(name, nil, "r+", 0600) - end - - end - # make BTree lookups case insensitive - module ::BDB - class CIBtree < Btree - def bdb_bt_compare(a, b) - if a == nil || b == nil - warning "CIBTree: comparing #{a.inspect} (#{self[a].inspect}) with #{b.inspect} (#{self[b].inspect})" - end - (a||'').downcase <=> (b||'').downcase - end - end - end - end - - module ::TokyoCabinet - class CIBDB < TokyoCabinet::BDB - def open(path, omode) - res = super - if res - self.setcmpfunc(Proc.new do |a, b| - a.downcase <=> b.downcase - end) - end - res - end - end - end - - # DBTree is a BTree equivalent of DBHash, with case insensitive lookups. - class DBTree - # absfilename:: use +key+ as an actual filename, don't prepend the bot's - # config path and don't append ".db" - def initialize(bot, key, absfilename=false) - @bot = bot - @key = key - - relfilename = @bot.path key - relfilename << '.tdb' - - if absfilename && File.exist?(key) - # db already exists, use it - @db = DBTree.open_db(key) - @fname = key.dup - elsif absfilename - # create empty db - @db = DBTree.create_db(key) - @fname = key.dup - elsif File.exist? relfilename - # db already exists, use it - @db = DBTree.open_db relfilename - @fname = relfilename.dup - else - # create empty db - @db = DBTree.create_db relfilename - @fname = relfilename.dup - end - oldbasename = (absfilename ? key : relfilename).gsub(/\.tdb$/, ".db") - if File.exists? oldbasename and defined? BDB - # upgrading - warning "Upgrading old database #{oldbasename}..." - oldb = ::BDB::CIBtree.open(oldbasename, nil, "r", 0600) - oldb.each_key do |k| - @db.outlist k - @db.putlist k, (oldb.duplicates(k, false)) - end - oldb.close - File.rename oldbasename, oldbasename+".bak" - end - @db - end - - def method_missing(method, *args, &block) - return @db.send(method, *args, &block) - end - - # Since TokyoCabinet does not have the concept of an environment, we have to do the - # database management ourselves. In particular, we have to keep a list of open - # registries to be sure we to close all of them on exit - @@bot_registries={ } - def self.close_bot_registries - @@bot_registries.each { |name, reg| reg.close } - @@bot_registries.clear - end - - def close - db = @@bot_registries.delete(@fname) - if db != @db - error "We think we have #{@db} from #{@fname}, TC pseudo-env gives us #{db}" - end - @db.close - end - - def DBTree.create_db(name) - debug "DBTree: creating empty db #{name}" - if @@bot_registries.key? name - error "DBTree: creating assumingly allocated db #{name}?!" - return @@bot_registries[name] - end - db = TokyoCabinet::CIBDB.new - res = db.open(name, TokyoCabinet::CIBDB::OREADER | TokyoCabinet::CIBDB::OCREAT | TokyoCabinet::CIBDB::OWRITER) - if res - @@bot_registries[name] = db - else - error "DBTree: creating empty db #{name}: #{db.errmsg(db.ecode)}" - end - return db - end - - def DBTree.open_db(name) - debug "DBTree: opening existing db #{name}" - if @@bot_registries.key? name - return @@bot_registries[name] - end - db = TokyoCabinet::CIBDB.new - res = db.open(name, TokyoCabinet::CIBDB::OREADER | TokyoCabinet::CIBDB::OWRITER) - if res - @@bot_registries[name] = db - else - error "DBTree: opening db #{name}: #{db.errmsg(db.ecode)}" - end - return db - end - - def DBTree.cleanup_logs() - # no-op - end - - def DBTree.stats() - # no-op - end - - def DBTree.cleanup_env() - DBTree.close_bot_registries - end - - end - -end - -module Irc -class Bot - - # This class is now used purely for upgrading from prior versions of rbot - # the new registry is split into multiple DBHash objects, one per plugin - class Registry - def initialize(bot) - @bot = bot - upgrade_data - upgrade_data2 - end - - # check for older versions of rbot with data formats that require updating - # NB this function is called _early_ in init(), pretty much all you have to - # work with is @bot.botclass. - def upgrade_data - oldreg = @bot.path 'registry.db' - if defined? DBHash - newreg = @bot.path 'plugin_registry.db' - if File.exist?(oldreg) - log _("upgrading old-style (rbot 0.9.5 or earlier) plugin registry to new format") - old = ::BDB::Hash.open(oldreg, nil, "r+", 0600) - new = ::BDB::CIBtree.open(newreg, nil, ::BDB::CREATE | ::BDB::EXCL, 0600) - old.each {|k,v| - new[k] = v - } - old.close - new.close - File.rename(oldreg, oldreg + ".old") - end - else - warning "Won't upgrade data: BDB not installed" if File.exist? oldreg - end - end - - def upgrade_data2 - oldreg = @bot.path 'plugin_registry.db' - if not defined? BDB - warning "Won't upgrade data: BDB not installed" if File.exist? oldreg - return - end - newdir = @bot.path 'registry' - if File.exist?(oldreg) - Dir.mkdir(newdir) unless File.exist?(newdir) - env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER)# | BDB::TXN_NOSYNC) - dbs = Hash.new - log _("upgrading previous (rbot 0.9.9 or earlier) plugin registry to new split format") - old = BDB::CIBtree.open(oldreg, nil, "r+", 0600, "env" => env) - old.each {|k,v| - prefix,key = k.split("/", 2) - prefix.downcase! - # subregistries were split with a +, now they are in separate folders - if prefix.gsub!(/\+/, "/") - # Ok, this code needs to be put in the db opening routines - dirs = File.dirname("#{@bot.botclass}/registry/#{prefix}.db").split("/") - dirs.length.times { |i| - dir = dirs[0,i+1].join("/")+"/" - unless File.exist?(dir) - log _("creating subregistry directory #{dir}") - Dir.mkdir(dir) - end - } - end - unless dbs.has_key?(prefix) - log _("creating db #{@bot.botclass}/registry/#{prefix}.tdb") - dbs[prefix] = TokyoCabinet::CIBDB.open("#{@bot.botclass}/registry/#{prefix}.tdb", - TokyoCabinet::CIBDB::OREADER | TokyoCabinet::CIBDB::OCREAT | TokyoCabinet::CIBDB::OWRITER) - end - dbs[prefix][key] = v - } - old.close - File.rename(oldreg, oldreg + ".old") - dbs.each {|k,v| - log _("closing db #{k}") - v.close - } - env.close - end - end - - # This class provides persistent storage for plugins via a hash interface. - # The default mode is an object store, so you can store ruby objects and - # reference them with hash keys. This is because the default store/restore - # methods of the plugins' RegistryAccessor are calls to Marshal.dump and - # Marshal.restore, - # for example: - # blah = Hash.new - # blah[:foo] = "fum" - # @registry[:blah] = blah - # then, even after the bot is shut down and disconnected, on the next run you - # can access the blah object as it was, with: - # blah = @registry[:blah] - # The registry can of course be used to store simple strings, fixnums, etc as - # well, and should be useful to store or cache plugin data or dynamic plugin - # configuration. - # - # WARNING: - # in object store mode, don't make the mistake of treating it like a live - # object, e.g. (using the example above) - # @registry[:blah][:foo] = "flump" - # will NOT modify the object in the registry - remember that Registry#[] - # returns a Marshal.restore'd object, the object you just modified in place - # will disappear. You would need to: - # blah = @registry[:blah] - # blah[:foo] = "flump" - # @registry[:blah] = blah - # - # If you don't need to store objects, and strictly want a persistant hash of - # strings, you can override the store/restore methods to suit your needs, for - # example (in your plugin): - # def initialize - # class << @registry - # def store(val) - # val - # end - # def restore(val) - # val - # end - # end - # end - # Your plugins section of the registry is private, it has its own namespace - # (derived from the plugin's class name, so change it and lose your data). - # Calls to registry.each etc, will only iterate over your namespace. - class Accessor - - attr_accessor :recovery - - # plugins don't call this - a Registry::Accessor is created for them and - # is accessible via @registry. - def initialize(bot, name) - @bot = bot - @name = name.downcase - @filename = @bot.path 'registry', @name - dirs = File.dirname(@filename).split("/") - dirs.length.times { |i| - dir = dirs[0,i+1].join("/")+"/" - unless File.exist?(dir) - debug "creating subregistry directory #{dir}" - Dir.mkdir(dir) - end - } - @filename << ".tdb" - @registry = nil - @default = nil - @recovery = nil - # debug "initializing registry accessor with name #{@name}" - end - - def registry - @registry ||= DBTree.new @bot, "registry/#{@name}" - end - - def flush - # debug "fushing registry #{registry}" - return if !@registry - registry.sync - end - - def close - # debug "closing registry #{registry}" - return if !@registry - registry.close - @registry = nil - end - - # convert value to string form for storing in the registry - # defaults to Marshal.dump(val) but you can override this in your module's - # registry object to use any method you like. - # For example, if you always just handle strings use: - # def store(val) - # val - # end - def store(val) - Marshal.dump(val) - end - - # restores object from string form, restore(store(val)) must return val. - # If you override store, you should override restore to reverse the - # action. - # For example, if you always just handle strings use: - # def restore(val) - # val - # end - def restore(val) - begin - Marshal.restore(val) - rescue Exception => e - error _("failed to restore marshal data for #{val.inspect}, attempting recovery or fallback to default") - debug e - if defined? @recovery and @recovery - begin - return @recovery.call(val) - rescue Exception => ee - error _("marshal recovery failed, trying default") - debug ee - end - end - return default - end - end - - # lookup a key in the registry - def [](key) - if File.exist?(@filename) and registry.has_key?(key.to_s) - return restore(registry[key.to_s]) - else - return default - end - end - - # set a key in the registry - def []=(key,value) - registry[key.to_s] = store(value) - end - - # set the default value for registry lookups, if the key sought is not - # found, the default will be returned. The default default (har) is nil. - def set_default (default) - @default = default - end - - def default - @default && (@default.dup rescue @default) - end - - # just like Hash#each - def each(set=nil, bulk=0, &block) - return nil unless File.exist?(@filename) - registry.fwmkeys(set.to_s).each {|key| - block.call(key, restore(registry[key])) - } - end - - # just like Hash#each_key - def each_key(set=nil, bulk=0, &block) - return nil unless File.exist?(@filename) - registry.fwmkeys(set.to_s).each do |key| - block.call(key) - end - end - - # just like Hash#each_value - def each_value(set=nil, bulk=0, &block) - return nil unless File.exist?(@filename) - registry.fwmkeys(set.to_s).each do |key| - block.call(restore(registry[key])) - end - end - - # just like Hash#has_key? - def has_key?(key) - return false unless File.exist?(@filename) - return registry.has_key?(key.to_s) - end - - alias include? has_key? - alias member? has_key? - alias key? has_key? - - # just like Hash#has_both? - def has_both?(key, value) - return false unless File.exist?(@filename) - registry.has_key?(key.to_s) and registry.has_value?(store(value)) - end - - # just like Hash#has_value? - def has_value?(value) - return false unless File.exist?(@filename) - return registry.has_value?(store(value)) - end - - # just like Hash#index? - def index(value) - self.each do |k,v| - return k if v == value - end - return nil - end - - # delete a key from the registry - def delete(key) - return default unless File.exist?(@filename) - return registry.delete(key.to_s) - end - - # returns a list of your keys - def keys - return [] unless File.exist?(@filename) - return registry.keys - end - - # Return an array of all associations [key, value] in your namespace - def to_a - return [] unless File.exist?(@filename) - ret = Array.new - registry.each {|key, value| - ret << [key, restore(value)] - } - return ret - end - - # Return an hash of all associations {key => value} in your namespace - def to_hash - return {} unless File.exist?(@filename) - ret = Hash.new - registry.each {|key, value| - ret[key] = restore(value) - } - return ret - end - - # empties the registry (restricted to your namespace) - def clear - return true unless File.exist?(@filename) - registry.vanish - end - alias truncate clear - - # returns an array of the values in your namespace of the registry - def values - return [] unless File.exist?(@filename) - ret = Array.new - self.each {|k,v| - ret << restore(v) - } - return ret - end - - def sub_registry(prefix) - return Accessor.new(@bot, @name + "/" + prefix.to_s) - end - - # returns the number of keys in your registry namespace - def length - return 0 unless File.exist?(@filename) - registry.length - end - alias size length - - # That is btree! - def putdup(key, value) - registry.putdup(key.to_s, store(value)) - end - - def putlist(key, values) - registry.putlist(key.to_s, value.map {|v| store(v)}) - end - - def getlist(key) - return [] unless File.exist?(@filename) - (registry.getlist(key.to_s) || []).map {|v| restore(v)} - end - end - - end -end -end -- 2.39.2