add unicode string support

lib/marcel/tables.rb

@@ -2521,7 +2521,7 @@ module Marcel
     ['text/html', [[0, b['(?i)<(html|head|body|title|div)[ >]']], [0, b['(?i)<h[123][ >]']]]],
     ['image/svg+xml', [[0..4096, b['<svg']]]],
     ['video/x-msvideo', [[0, b['RIFF'], [[8, b['AVI ']]]], [8, b['AVI ']]]],
-    ['video/x-ms-wmv', [[0..8192, b['Windows Media Video']], [0..8192, b['VC-1 Advanced Profile']], [0..8192, b['wmv2']]]],
+    ['video/x-ms-wmv', [[0..8192, b["W\000i\000n\000d\000o\000w\000s\000 \000M\000e\000d\000i\000a\000 \000V\000i\000d\000e\000o\000"]], [0..8192, b["V\000C\000-\0001\000 \000A\000d\000v\000a\000n\000c\000e\000d\000 \000P\000r\000o\000f\000i\000l\000e\000"]], [0..8192, b["w\000m\000v\0002\000"]]]],
     ['video/mp4', [[4, b['ftypmp41']], [4, b['ftypmp42']]]],
     ['audio/mp4', [[4, b['ftypM4A ']], [4, b['ftypM4B ']], [4, b['ftypF4A ']], [4, b['ftypF4B ']]]],
     ['video/quicktime', [[4, b["moov\000"]], [4, b["mdat\000"]], [4, b["free\000"]], [4, b["skip\000"]], [4, b["pnot\000"]], [4, b['ftyp']], [0, b["\000\000\000\bwide"]]]],
@@ -2831,7 +2831,7 @@ module Marcel
     ['audio/x-flac', [[0, b['fLaC']]]],
     ['audio/x-mod', [[0, b['Extended Module:']], [21, b['BMOD2STM']], [1080, b['M.K.']], [1080, b['M!K!']], [1080, b['FLT4']], [1080, b['FLT8']], [1080, b['4CHN']], [1080, b['6CHN']], [1080, b['8CHN']], [1080, b['CD81']], [1080, b['OKTA']], [1080, b['16CN']], [1080, b['32CN']], [0, b['IMPM']]]],
     ['audio/x-mpegurl', [[0, b["#EXTM3U\r\n"]]]],
-    ['audio/x-ms-wma', [[0..8192, b['Windows Media Audio']]]],
+    ['audio/x-ms-wma', [[0..8192, b["W\000i\000n\000d\000o\000w\000s\000 \000M\000e\000d\000i\000a\000 \000A\000u\000d\000i\000o\000"]]]],
     ['audio/x-pn-realaudio', [[0, b[".ra\375"]]]],
     ['audio/x-psf', [[0, b['PSF'], [[3, b["\001"]], [3, b["\002"]], [3, b["\021"]], [3, b["\022"]], [3, b["\023"]], [3, b['!']], [3, b["\""]], [3, b['#']], [3, b['A']]]]]],
     ['audio/x-sap', [[0, b["SAP\r\n"]]]],

script/generate_tables.rb

@@ -65,6 +65,10 @@ def get_matches(mime, parent)
 
     offset = offset.size == 2 ? offset[0]..offset[1] : offset[0]
     case type
+      when 'unicodeLE', 'unicodeBE' # Unicode string types (UTF-16 Little/Big Endian)
+        value.gsub!(/\A0x([0-9a-f]+)\z/i) { [$1].pack('H*') }
+        encoding = type == 'unicodeLE' ? Encoding::UTF_16LE : Encoding::UTF_16BE
+        value = value.encode(encoding).force_encoding(Encoding::BINARY)
     when 'string', 'stringignorecase'
       value.gsub!(/\A0x([0-9a-f]+)\z/i) { [$1].pack('H*') }
       value.gsub!(/\\(x[\dA-Fa-f]{1,2}|0\d{1,3}|\d{1,3}|.)/) { eval("\"\\#{$1}\"") }

test/fixtures/magic/audio/x-pn-realaudio/sample.ra

@@ -0,0 +1 @@
+.ra�