4 # :title: Berkeley DB interface
9 fatal "rbot couldn't load the bdb module, perhaps you need to install it? try http://www.ruby-lang.org/en/raa-list.rhtml?name=bdb or http://github.com/knu/ruby-bdb"
11 fatal "rbot couldn't load the bdb module: #{e.pretty_inspect}"
22 if BDB::VERSION_MAJOR < 4
23 fatal "Your bdb (Berkeley DB) version #{BDB::VERSION} is too old!"
24 fatal "rbot will only run with bdb version 4 or higher, please upgrade."
25 fatal "For maximum reliability, upgrade to version 4.2 or higher."
26 raise BDB::Fatal, BDB::VERSION + " is too old"
29 if BDB::VERSION_MAJOR == 4 and BDB::VERSION_MINOR < 2
30 warning "Your bdb (Berkeley DB) version #{BDB::VERSION} may not be reliable."
31 warning "If possible, try upgrade version 4.2 or later."
34 # make BTree lookups case insensitive
37 def bdb_bt_compare(a, b)
38 if a == nil || b == nil
39 warning "CIBTree: comparing key #{a.inspect} with #{b.inspect}"
41 (a||'').downcase <=> (b||'').downcase
48 # DBHash is for tying a hash to disk (using bdb).
49 # Call it with an identifier, for example "mydata". It'll look for
50 # mydata.db, if it exists, it will load and reference that db.
51 # Otherwise it'll create and empty db called mydata.db
54 # absfilename:: use +key+ as an actual filename, don't prepend the bot's
55 # config path and don't append ".db"
56 def initialize(bot, key, absfilename=false)
59 relfilename = @bot.path key
61 if absfilename && File.exist?(key)
62 # db already exists, use it
63 @db = DBHash.open_db(key)
66 @db = DBHash.create_db(key)
67 elsif File.exist? relfilename
68 # db already exists, use it
69 @db = DBHash.open_db relfilename
72 @db = DBHash.create_db relfilename
76 def method_missing(method, *args, &block)
77 return @db.send(method, *args, &block)
80 def DBHash.create_db(name)
81 debug "DBHash: creating empty db #{name}"
82 return BDB::Hash.open(name, nil,
83 BDB::CREATE | BDB::EXCL, 0600)
86 def DBHash.open_db(name)
87 debug "DBHash: opening existing db #{name}"
88 return BDB::Hash.open(name, nil, "r+", 0600)
94 # DBTree is a BTree equivalent of DBHash, with case insensitive lookups.
97 # TODO: make this customizable
98 # Note that it must be at least four times lg_bsize
99 @@lg_max = 8*1024*1024
100 # absfilename:: use +key+ as an actual filename, don't prepend the bot's
101 # config path and don't append ".db"
102 def initialize(bot, key, absfilename=false)
107 @@env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER, "set_lg_max" => @@lg_max)
108 debug "DBTree: environment opened with max log size #{@@env.conf['lg_max']}"
110 debug "DBTree: failed to open environment: #{e.pretty_inspect}. Retrying ..."
111 @@env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER)
113 #@@env = BDB::Env.open(@bot.botclass, BDB::CREATE | BDB::INIT_MPOOL | BDB::RECOVER)
116 relfilename = @bot.path key
119 if absfilename && File.exist?(key)
120 # db already exists, use it
121 @db = DBTree.open_db(key)
124 @db = DBTree.create_db(key)
125 elsif File.exist? relfilename
126 # db already exists, use it
127 @db = DBTree.open_db relfilename
130 @db = DBTree.create_db relfilename
134 def method_missing(method, *args, &block)
135 return @db.send(method, *args, &block)
138 def DBTree.create_db(name)
139 debug "DBTree: creating empty db #{name}"
140 return @@env.open_db(BDB::CIBtree, name, nil, BDB::CREATE | BDB::EXCL, 0600)
143 def DBTree.open_db(name)
144 debug "DBTree: opening existing db #{name}"
145 return @@env.open_db(BDB::CIBtree, name, nil, "r+", 0600)
148 def DBTree.cleanup_logs()
150 debug "No BDB environment, skipping checkpoint"
154 debug "DBTree: checkpointing ..."
156 rescue Exception => e
157 debug "Failed: #{e.pretty_inspect}"
160 debug "DBTree: flushing log ..."
162 logs = @@env.log_archive(BDB::ARCH_ABS)
163 debug "DBTree: deleting archivable logs: #{logs.join(', ')}."
167 rescue Exception => e
168 debug "Failed: #{e.pretty_inspect}"
174 debug "No BDB environment, no stats"
178 debug "General stats:"
181 debug @@env.lock_stat
187 debug "Couldn't dump stats"
191 def DBTree.cleanup_env()
193 debug "No BDB environment, skipping cleanup"
197 debug "DBTree: checking transactions ..."
198 has_active_txn = @@env.txn_stat["st_nactive"] > 0
200 warning "DBTree: not all transactions completed!"
203 debug "DBTree: closing environment #{@@env}"
208 debug "DBTree: keeping file because of incomplete transactions"
210 debug "DBTree: cleaning up environment in #{path}"
211 BDB::Env.remove("#{path}")
213 rescue Exception => e
214 error "failed to clean up environment: #{e.pretty_inspect}"
226 # This class is now used purely for upgrading from prior versions of rbot
227 # the new registry is split into multiple DBHash objects, one per plugin
235 # check for older versions of rbot with data formats that require updating
236 # NB this function is called _early_ in init(), pretty much all you have to
237 # work with is @bot.botclass.
239 oldreg = @bot.path 'registry.db'
240 newreg = @bot.path 'plugin_registry.db'
241 if File.exist?(oldreg)
242 log _("upgrading old-style (rbot 0.9.5 or earlier) plugin registry to new format")
243 old = BDB::Hash.open(oldreg, nil, "r+", 0600)
244 new = BDB::CIBtree.open(newreg, nil, BDB::CREATE | BDB::EXCL, 0600)
250 File.rename(oldreg, oldreg + ".old")
255 oldreg = @bot.path 'plugin_registry.db'
256 newdir = @bot.path 'registry'
257 if File.exist?(oldreg)
258 Dir.mkdir(newdir) unless File.exist?(newdir)
259 env = BDB::Env.open(@bot.botclass, BDB::INIT_TRANSACTION | BDB::CREATE | BDB::RECOVER)# | BDB::TXN_NOSYNC)
261 log _("upgrading previous (rbot 0.9.9 or earlier) plugin registry to new split format")
262 old = BDB::CIBtree.open(oldreg, nil, "r+", 0600, "env" => env)
264 prefix,key = k.split("/", 2)
266 # subregistries were split with a +, now they are in separate folders
267 if prefix.gsub!(/\+/, "/")
268 # Ok, this code needs to be put in the db opening routines
269 dirs = File.dirname("#{@bot.botclass}/registry/#{prefix}.db").split("/")
270 dirs.length.times { |i|
271 dir = dirs[0,i+1].join("/")+"/"
272 unless File.exist?(dir)
273 log _("creating subregistry directory #{dir}")
278 unless dbs.has_key?(prefix)
279 log _("creating db #{@bot.botclass}/registry/#{prefix}.db")
280 dbs[prefix] = BDB::CIBtree.open("#{@bot.botclass}/registry/#{prefix}.db",
281 nil, BDB::CREATE | BDB::EXCL,
287 File.rename(oldreg, oldreg + ".old")
289 log _("closing db #{k}")
297 # This class provides persistent storage for plugins via a hash interface.
298 # The default mode is an object store, so you can store ruby objects and
299 # reference them with hash keys. This is because the default store/restore
300 # methods of the plugins' RegistryAccessor are calls to Marshal.dump and
305 # @registry[:blah] = blah
306 # then, even after the bot is shut down and disconnected, on the next run you
307 # can access the blah object as it was, with:
308 # blah = @registry[:blah]
309 # The registry can of course be used to store simple strings, fixnums, etc as
310 # well, and should be useful to store or cache plugin data or dynamic plugin
314 # in object store mode, don't make the mistake of treating it like a live
315 # object, e.g. (using the example above)
316 # @registry[:blah][:foo] = "flump"
317 # will NOT modify the object in the registry - remember that Registry#[]
318 # returns a Marshal.restore'd object, the object you just modified in place
319 # will disappear. You would need to:
320 # blah = @registry[:blah]
321 # blah[:foo] = "flump"
322 # @registry[:blah] = blah
324 # If you don't need to store objects, and strictly want a persistant hash of
325 # strings, you can override the store/restore methods to suit your needs, for
326 # example (in your plugin):
337 # Your plugins section of the registry is private, it has its own namespace
338 # (derived from the plugin's class name, so change it and lose your data).
339 # Calls to registry.each etc, will only iterate over your namespace.
342 attr_accessor :recovery
344 # plugins don't call this - a Registry::Accessor is created for them and
345 # is accessible via @registry.
346 def initialize(bot, name)
348 @name = name.downcase
349 @filename = @bot.path 'registry', @name
350 dirs = File.dirname(@filename).split("/")
351 dirs.length.times { |i|
352 dir = dirs[0,i+1].join("/")+"/"
353 unless File.exist?(dir)
354 debug "creating subregistry directory #{dir}"
362 # debug "initializing registry accessor with name #{@name}"
366 @registry ||= DBTree.new @bot, "registry/#{@name}"
370 # debug "fushing registry #{registry}"
377 # debug "closing registry #{registry}"
382 # convert value to string form for storing in the registry
383 # defaults to Marshal.dump(val) but you can override this in your module's
384 # registry object to use any method you like.
385 # For example, if you always just handle strings use:
393 # restores object from string form, restore(store(val)) must return val.
394 # If you override store, you should override restore to reverse the
396 # For example, if you always just handle strings use:
403 rescue Exception => e
404 error _("failed to restore marshal data for #{val.inspect}, attempting recovery or fallback to default")
406 if defined? @recovery and @recovery
408 return @recovery.call(val)
409 rescue Exception => ee
410 error _("marshal recovery failed, trying default")
418 # lookup a key in the registry
420 if File.exist?(@filename) && registry.has_key?(key)
421 return restore(registry[key])
427 # set a key in the registry
429 registry[key] = store(value)
432 # set the default value for registry lookups, if the key sought is not
433 # found, the default will be returned. The default default (har) is nil.
434 def set_default (default)
439 @default && (@default.dup rescue @default)
442 # just like Hash#each
443 def each(set=nil, bulk=0, &block)
444 return nil unless File.exist?(@filename)
445 registry.each(set, bulk) {|key,value|
446 block.call(key, restore(value))
450 # just like Hash#each_key
451 def each_key(set=nil, bulk=0, &block)
452 return nil unless File.exist?(@filename)
453 registry.each_key(set, bulk) {|key|
458 # just like Hash#each_value
459 def each_value(set=nil, bulk=0, &block)
460 return nil unless File.exist?(@filename)
461 registry.each_value(set, bulk) { |value|
462 block.call(restore(value))
466 # just like Hash#has_key?
468 return false unless File.exist?(@filename)
469 return registry.has_key?(key)
471 alias include? has_key?
472 alias member? has_key?
475 # just like Hash#has_both?
476 def has_both?(key, value)
477 return false unless File.exist?(@filename)
478 return registry.has_both?(key, store(value))
481 # just like Hash#has_value?
482 def has_value?(value)
483 return false unless File.exist?(@filename)
484 return registry.has_value?(store(value))
487 # just like Hash#index?
489 return nil unless File.exist?(@filename)
490 ind = registry.index(store(value))
498 # delete a key from the registry
500 return default unless File.exist?(@filename)
501 return registry.delete(key)
504 # returns a list of your keys
506 return [] unless File.exist?(@filename)
510 # Return an array of all associations [key, value] in your namespace
512 return [] unless File.exist?(@filename)
514 registry.each {|key, value|
515 ret << [key, restore(value)]
520 # Return an hash of all associations {key => value} in your namespace
522 return {} unless File.exist?(@filename)
524 registry.each {|key, value|
525 ret[key] = restore(value)
530 # empties the registry (restricted to your namespace)
532 return true unless File.exist?(@filename)
537 # returns an array of the values in your namespace of the registry
539 return [] unless File.exist?(@filename)
547 def sub_registry(prefix)
548 return Accessor.new(@bot, @name + "/" + prefix.to_s)
551 # returns the number of keys in your registry namespace