X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=test%2Ftest_journal.rb;h=73dbcf42e4ecd80b61933e693e3a537bbb82dbc5;hb=7bee70f012b85c03f73fa3e765417a939e02a5cf;hp=3f1ce7667a5020538793277f612e52f68c32126b;hpb=075c7e031b3449ff026e51a2299f56df573ef688;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git

diff --git a/test/test_journal.rb b/test/test_journal.rb
index 3f1ce766..73dbcf42 100644
--- a/test/test_journal.rb
+++ b/test/test_journal.rb
@@ -1,9 +1,19 @@
 $:.unshift File.join(File.dirname(__FILE__), '../lib')
 
+module Irc
+class Bot
+  module Config
+    @@datadir = File.expand_path(File.dirname($0) + '/../data/rbot')
+    @@coredir = File.expand_path(File.dirname($0) + '/../lib/rbot/core') 
+  end
+end
+end
+
 require 'test/unit'
 require 'rbot/ircbot'
 require 'rbot/journal'
-require 'rbot/journal/postgres.rb'
+
+require 'benchmark'
 
 DAY=60*60*24
 
@@ -19,7 +29,9 @@ class JournalMessageTest < Test::Unit::TestCase
     end
     assert_nil(m.get('nope', nil))
     assert_nil(m.get('baz'))
-    assert_equal(23, m.get('qux.quxx'))
+    assert_equal(23, m['qux.quxx'])
+    assert_equal(nil, m['qux.nope'])
+    assert_raise(ArgumentError) { m.get('qux.nope') }
   end
 
 end
@@ -160,8 +172,8 @@ class JournalBrokerTest < Test::Unit::TestCase
     received = []
     journal = JournalBroker.new
 
-    # subscribe to messages:
-    sub = journal.subscribe(Query.define { topic 'foo' }) do |message|
+    # subscribe to messages for topic foo:
+    sub = journal.subscribe('foo') do |message|
       received << message
     end
 
@@ -186,47 +198,17 @@ class JournalBrokerTest < Test::Unit::TestCase
 
 end
 
-class JournalStoragePostgresTest < Test::Unit::TestCase
+module JournalStorageTestMixin
 
   include Irc::Bot::Journal
 
-  def setup
-    @storage = Storage::PostgresStorage.new(
-      uri: ENV['DB_URI'] || 'postgresql://localhost/rbot_journal',
-      drop: true)
-  end
-
   def teardown
     @storage.drop
   end
 
-  def test_query_to_sql
-    q = Query.define do
-      id 'foo'
-      id 'bar', 'baz'
-      topic 'log.irc.*'
-      topic 'log.core', 'baz'
-      timestamp from: Time.now, to: Time.now + 60 * 10
-      payload 'action': :privmsg, 'alice': 'bob'
-      payload 'channel': '#rbot'
-      payload 'foo.bar': 'baz'
-    end
-    sql = @storage.query_to_sql(q)
-    assert_equal("(id = $1 OR id = $2 OR id = $3) AND (topic ILIKE $4 OR topic ILIKE $5 OR topic ILIKE $6) AND (timestamp >= $7 AND timestamp <= $8) AND (payload->>'action' = $9 OR payload->>'alice' = $10 OR payload->>'channel' = $11 OR payload->'foo'->>'bar' = $12)", sql[0])
-    q = Query.define do
-      id 'foo'
-    end
-    assert_equal('(id = $1)', @storage.query_to_sql(q)[0])
-    q = Query.define do
-      topic 'foo.*.bar'
-    end
-    assert_equal('(topic ILIKE $1)', @storage.query_to_sql(q)[0])
-    assert_equal(['foo.%.bar'], @storage.query_to_sql(q)[1])
-  end
-
   def test_operations
     # insertion
-    m = JournalMessage.create('log.core', {foo: {bar: 'baz'}})
+    m = JournalMessage.create('log.core', {foo: {bar: 'baz', qux: 42}})
     @storage.insert(m)
 
     # query by id
@@ -239,7 +221,7 @@ class JournalStoragePostgresTest < Test::Unit::TestCase
                  res.first.timestamp.strftime('%Y-%m-%d %H:%M:%S%z'))
 
     # check if payload was returned correctly:
-    assert_equal({'foo' => {'bar' => 'baz'}}, res.first.payload)
+    assert_equal({'foo' => {'bar' => 'baz', 'qux' => 42}}, res.first.payload)
 
     # query by topic
     assert_equal(m, @storage.find(Query.define { topic('log.core') }).first)
@@ -263,6 +245,31 @@ class JournalStoragePostgresTest < Test::Unit::TestCase
     assert_equal(m, @storage.find.first)
   end
 
+  def test_find
+    # tests limit/offset and block parameters of find()
+    @storage.insert(JournalMessage.create('irclogs', {message: 'foo'}))
+    @storage.insert(JournalMessage.create('irclogs', {message: 'bar'}))
+    @storage.insert(JournalMessage.create('irclogs', {message: 'baz'}))
+    @storage.insert(JournalMessage.create('irclogs', {message: 'qux'}))
+
+    msgs = []
+    @storage.find(Query.define({topic: 'irclogs'}), 2, 1) do |m|
+      msgs << m
+    end
+    assert_equal(2, msgs.length)
+    assert_equal('bar', msgs.first['message'])
+    assert_equal('baz', msgs.last['message'])
+
+    msgs = []
+    @storage.find(Query.define({topic: 'irclogs'})) do |m|
+      msgs << m
+    end
+    assert_equal(4, msgs.length)
+    assert_equal('foo', msgs.first['message'])
+    assert_equal('qux', msgs.last['message'])
+
+  end
+
   def test_operations_multiple
     # test operations on multiple messages
     # insert a bunch:
@@ -270,7 +277,7 @@ class JournalStoragePostgresTest < Test::Unit::TestCase
     @storage.insert(JournalMessage.create('test.topic', {name: 'two'}))
     @storage.insert(JournalMessage.create('test.topic', {name: 'three'}))
     @storage.insert(JournalMessage.create('archived.topic', {name: 'four'},
-                                          timestamp: Time.now - DAY*100))
+      timestamp: Time.now - DAY*100))
     @storage.insert(JournalMessage.create('complex', {name: 'five', country: {
       name: 'Italy'
     }}))
@@ -278,16 +285,149 @@ class JournalStoragePostgresTest < Test::Unit::TestCase
       name: 'Austria'
     }}))
 
+    # query by topic
+    assert_equal(3, @storage.find(Query.define { topic 'test.*' }).length)
+    # query by payload
+    assert_equal(1, @storage.find(Query.define {
+      payload('country.name' => 'Austria') }).length)
+    # query by timestamp range
+    assert_equal(1, @storage.find(Query.define {
+      timestamp(from: Time.now - DAY*150, to: Time.now - DAY*50) }).length)
+
+    # count with query
+    assert_equal(2, @storage.count(Query.define { topic('complex') }))
+    assert_equal(6, @storage.count)
+    @storage.remove(Query.define { topic('archived.*') })
+    assert_equal(5, @storage.count)
+    @storage.remove
+    assert_equal(0, @storage.count)
+  end
+
+  def test_broker_interface
+    journal = JournalBroker.new(storage: @storage) 
+
+    journal.publish 'irclogs', message: 'foo'
+    journal.publish 'irclogs', message: 'bar'
+    journal.publish 'irclogs', message: 'baz'
+    journal.publish 'irclogs', message: 'qux'
+
+    # wait for messages to be consumed:
+    sleep 0.1
 
+    msgs = []
+    journal.find({topic: 'irclogs'}, 2, 1) do |m|
+      msgs << m
+    end
+    assert_equal(2, msgs.length)
+    assert_equal('bar', msgs.first['message'])
+    assert_equal('baz', msgs.last['message'])
+
+    journal.ensure_payload_index('foo.bar.baz')
   end
 
-  def test_journal
-    received = []
-    # this journal persists messages in the test storage:
-    journal = JournalBroker.new(storage: @storage)
+  NUM=100 # 1_000_000
+  def test_benchmark
+    puts
+
+    assert_equal(0, @storage.count)
+    # prepare messages to insert, we benchmark the storage backend not ruby
+    num = 0
+    messages = (0...NUM).map do
+      num += 1
+      JournalMessage.create(
+            'test.topic.num_'+num.to_s, {answer: {number: '42', word: 'forty-two'}})
+    end
 
+    # iter is the number of operations performed WITHIN block
+    def benchmark(label, iter, &block)
+      time = Benchmark.realtime do
+        yield
+      end
+      puts label + ' %d iterations, duration: %.3fms (%.3fms / iteration)' % [iter, time*1000, (time*1000) / iter]
+    end
+
+    benchmark(@storage.class.to_s+'~insert', messages.length) do
+      messages.each { |m|
+        @storage.insert(m)
+      }
+    end
 
+    benchmark(@storage.class.to_s+'~find_by_id', messages.length) do
+      messages.each { |m|
+        @storage.find(Query.define { id m.id })
+      }
+    end
+    benchmark(@storage.class.to_s+'~find_by_topic', messages.length) do
+      messages.each { |m|
+        @storage.find(Query.define { topic m.topic })
+      }
+    end
+    benchmark(@storage.class.to_s+'~find_by_topic_wildcard', messages.length) do
+      messages.each { |m|
+        @storage.find(Query.define { topic m.topic.gsub('topic', '*') })
+      }
+    end
   end
 
 end
 
+begin
+  require 'rbot/journal/postgres.rb'
+  if ENV['PG_URI']
+  class JournalStoragePostgresTest < Test::Unit::TestCase
+
+    include JournalStorageTestMixin
+
+    def setup
+      @storage = Storage::PostgresStorage.new(
+        uri: ENV['PG_URI'] || 'postgresql://localhost/rbot_journal',
+        drop: true)
+    end
+
+    def test_query_to_sql
+      q = Query.define do
+        id 'foo'
+        id 'bar', 'baz'
+        topic 'log.irc.*'
+        topic 'log.core', 'baz'
+        timestamp from: Time.now, to: Time.now + 60 * 10
+        payload 'action': :privmsg, 'alice': 'bob'
+        payload 'channel': '#rbot'
+        payload 'foo.bar': 'baz'
+      end
+      sql = @storage.query_to_sql(q)
+      assert_equal("(id = $1 OR id = $2 OR id = $3) AND (topic ILIKE $4 OR topic ILIKE $5 OR topic ILIKE $6) AND (timestamp >= $7 AND timestamp <= $8) AND (payload->>'action' = $9 OR payload->>'alice' = $10 OR payload->>'channel' = $11 OR payload->'foo'->>'bar' = $12)", sql[0])
+      q = Query.define do
+        id 'foo'
+      end
+      assert_equal('(id = $1)', @storage.query_to_sql(q)[0])
+      q = Query.define do
+        topic 'foo.*.bar'
+      end
+      assert_equal('(topic ILIKE $1)', @storage.query_to_sql(q)[0])
+      assert_equal(['foo.%.bar'], @storage.query_to_sql(q)[1])
+    end
+
+  end
+  else
+    puts 'NOTE: Set PG_URI environment variable to test postgresql storage.'
+  end
+rescue Exception; end
+
+begin
+  require 'rbot/journal/mongo.rb'
+  if ENV['MONGO_URI']
+  class JournalStorageMongoTest < Test::Unit::TestCase
+
+    include JournalStorageTestMixin
+
+    def setup
+      @storage = Storage::MongoStorage.new(
+        uri: ENV['MONGO_URI'] || 'mongodb://127.0.0.1:27017/rbot',
+        drop: true)
+    end
+  end
+  else
+    puts 'NOTE: Set MONGO_URI environment variable to test postgresql storage.'
+  end
+rescue Exception; end