diff options
| -rw-r--r-- | lib/mauve/alert.rb | 24 | ||||
| -rw-r--r-- | test/tc_mauve_database_peculiarities.rb | 85 |
2 files changed, 109 insertions, 0 deletions
diff --git a/lib/mauve/alert.rb b/lib/mauve/alert.rb index 6b08f82..a5cb885 100644 --- a/lib/mauve/alert.rb +++ b/lib/mauve/alert.rb @@ -75,6 +75,20 @@ module Mauve def size; 99; end include DataMapper::Resource + + # + # If a string matches this regex, it is valid UTF8. + # + UTF8_REGEXP = Regexp.new(/^(?:#{[ + "[\x00-\x7F]", # ASCII + "[\xC2-\xDF][\x80-\xBF]", # non-overlong 2-byte + "\xE0[\xA0-\xBF][\x80-\xBF]", # excluding overlongs + "[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}", # straight 3-byte + "\xED[\x80-\x9F][\x80-\xBF]", # excluding surrogates + "\xF0[\x90-\xBF][\x80-\xBF]{2}", # planes 1-3 + "[\xF1-\xF3][\x80-\xBF]{3}", # planes 4-15 + "\xF4[\x80-\x8F][\x80-\xBF]{2}" # plane 16 + ].join("|")})*$/) property :id, Serial property :alert_id, String, :required => true, :unique_index => :alert_index, :length=>256, :lazy => false @@ -618,6 +632,7 @@ module Mauve # @return [String] def remove_html(str, conf = Sanitize::Config::DEFAULT) raise ArgumentError, "Expected a string, got a #{str.class}" unless str.is_a?(String) + str = clean_utf8(str) if str =~ /<[^0-9 <&.-]/ Sanitize.clean( str, conf ) @@ -631,9 +646,18 @@ module Mauve # @param [String] str String to clean # @return [String] def clean_html(str) + str = clean_utf8(str) remove_html(str, Sanitize::Config::RELAXED.merge({:remove_contents => true})) end + def clean_utf8(str) + unless UTF8_REGEXP.match(str) + str.gsub(/[^\x00-\x7F]/,'?') + else + str + end + end + # All alerts currently raised # # @return [Array] diff --git a/test/tc_mauve_database_peculiarities.rb b/test/tc_mauve_database_peculiarities.rb new file mode 100644 index 0000000..9eb612e --- /dev/null +++ b/test/tc_mauve_database_peculiarities.rb @@ -0,0 +1,85 @@ +$:.unshift "../lib" + +require 'th_mauve' +require 'mauve/datamapper' +require 'mauve/server' +require 'mauve/configuration' +require 'mauve/configuration_builder' +require 'mauve/configuration_builders' +require 'iconv' + +class TcMauveDatabasePeculiarities < Mauve::UnitTest + include Mauve + + def setup + super + setup_database + @temp_db = "mauve_test.#{10.times.collect{ rand(36).to_s(36) }.join}" + end + + def teardown + teardown_database + super + end + + def test_encoding + # + # Don't test unless the DB URL has been set. + # + return unless @db_url + + config=<<EOF +server { + database "#{@db_url}" +} +EOF + + Configuration.current = ConfigurationBuilder.parse(config) + Server.instance.setup + + x = Hash.new + x["en"] = "Please rush me my portable walrus polishing kit!" + x["fi"] = "Ole hyvä kiirehtiä minulle kannettavan mursu kiillotukseen pakki!" + x["jp"] = "私に私のポータブルセイウチの研磨キットを急いでください！" + + %w(UTF-8 WINDOWS-1252 SHIFT-JIS).each do |enc| + x.each do |lang, str| + assert_nothing_raised("Failed to use iconv to convert to #{enc}") { str = Iconv.conv(enc+"//IGNORE", "utf8", str) } + + alert = Alert.new( + :alert_id => "#{lang}:#{enc}", + :source => "test", + :subject => str + ) + + assert_nothing_raised("failed to insert #{enc}") { alert.save } + end + end + end +end + + + +class TcMauveDatabasePostgresPeculiarities < TcMauveDatabasePeculiarities + def setup + super + system("createdb #{@temp_db} --encoding UTF8") || flunk("Failed to create #{@temp_db}") + # @pg_conn = PGconn.open(:dbname => @temp_db) + @db_url = "postgres:///#{@temp_db}" + end + + def teardown + # @pg_conn.finish if @pg_conn.is_a?(PGconn) and @pg_conn.status == PGconn::CONNECTION_OK + super + (system("dropdb #{@temp_db}") || puts("Failed to drop #{@temp_db}")) if @temp_db + end +end + +class TcMauveDatabaseSqlite3Peculiarities < TcMauveDatabasePeculiarities + def setup + super + # @pg_conn = PGconn.open(:dbname => @temp_db) + @db_url = "sqlite3::memory:" + end +end + |