Commit 1d18663b authored by Alex Vandiver's avatar Alex Vandiver
Browse files

Standardize on the stricter Encode::encode("UTF-8", ...) everywhere

This is not only for code consistency, but also for consistency of
output.  Encode::encode_utf8(...) is equivalent to
Encode::encode("utf8",...) which is the non-"strict" form of UTF-8.
Strict UTF-8 encoding differs in that (from `perldoc Encode`):

    ...its range is much narrower (0 ..  0x10_FFFF to cover only 21 bits
    instead of 32 or 64 bits) and some sequences are not allowed, like
    those used in surrogate pairs, the 31 non-character code points
    0xFDD0 .. 0xFDEF, the last two code points in any plane (0xXX_FFFE
    and 0xXX_FFFF), all non-shortest encodings, etc.

RT deals with interchange with databases, email, and other systems.  In
dealing with encodings, it should ensure that it does not produce byte
sequences that are invalid according to official Unicode standards.
parent ba110857
...@@ -72,7 +72,7 @@ foreach my $set ( 'ru', 'latin1' ) { ...@@ -72,7 +72,7 @@ foreach my $set ( 'ru', 'latin1' ) {
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$set}{test}/ $subject =~ /$string{$set}{test}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
} }
...@@ -101,7 +101,7 @@ diag "ascii subject with non-ascii subject tag"; ...@@ -101,7 +101,7 @@ diag "ascii subject with non-ascii subject tag";
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$tag_set}{support}/ $subject =~ /$string{$tag_set}{support}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
} }
...@@ -122,7 +122,7 @@ foreach my $set ( 'ru', 'latin1' ) { ...@@ -122,7 +122,7 @@ foreach my $set ( 'ru', 'latin1' ) {
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$tag_set}{support}/ $subject =~ /$string{$tag_set}{support}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
$subject =~ /$string{$set}{test}/ $subject =~ /$string{$set}{test}/
...@@ -171,7 +171,7 @@ diag "ascii subject with non-ascii subject prefix in template"; ...@@ -171,7 +171,7 @@ diag "ascii subject with non-ascii subject prefix in template";
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$prefix_set}{autoreply}/ $subject =~ /$string{$prefix_set}{autoreply}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
} }
...@@ -192,7 +192,7 @@ foreach my $set ( 'ru', 'latin1' ) { ...@@ -192,7 +192,7 @@ foreach my $set ( 'ru', 'latin1' ) {
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$prefix_set}{autoreply}/ $subject =~ /$string{$prefix_set}{autoreply}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
$subject =~ /$string{$set}{test}/ $subject =~ /$string{$set}{test}/
...@@ -222,7 +222,7 @@ foreach my $set ( 'ru', 'latin1' ) { ...@@ -222,7 +222,7 @@ foreach my $set ( 'ru', 'latin1' ) {
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$prefix_set}{autoreply}/ $subject =~ /$string{$prefix_set}{autoreply}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
$subject =~ /$string{$tag_set}{support}/ $subject =~ /$string{$tag_set}{support}/
...@@ -275,7 +275,7 @@ foreach my $set ( 'ru', 'latin1' ) { ...@@ -275,7 +275,7 @@ foreach my $set ( 'ru', 'latin1' ) {
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$set}{test}/ $subject =~ /$string{$set}{test}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
} }
...@@ -303,7 +303,7 @@ foreach my $set ( 'ru', 'latin1' ) { ...@@ -303,7 +303,7 @@ foreach my $set ( 'ru', 'latin1' ) {
my $status = 1; my $status = 1;
foreach my $mail ( @mails ) { foreach my $mail ( @mails ) {
my $entity = parse_mail( $mail ); my $entity = parse_mail( $mail );
my $subject = Encode::decode_utf8( $entity->head->get('Subject') ); my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
$subject =~ /$string{$set}{test}/ $subject =~ /$string{$set}{test}/
or do { $status = 0; diag "wrong subject: $subject" }; or do { $status = 0; diag "wrong subject: $subject" };
$subject =~ /$string{$tag_set}{support}/ $subject =~ /$string{$tag_set}{support}/
......
...@@ -504,8 +504,7 @@ EOF ...@@ -504,8 +504,7 @@ EOF
is ($tick->Id, $id, "correct ticket"); is ($tick->Id, $id, "correct ticket");
is ($tick->Subject , 'This is a test of I18N ticket creation', "Created the ticket - ". $tick->Subject); is ($tick->Subject , 'This is a test of I18N ticket creation', "Created the ticket - ". $tick->Subject);
my $unistring = "\303\241\303\251\303\255\303\263\303\272"; my $unistring = Encode::decode("UTF-8","\303\241\303\251\303\255\303\263\303\272");
Encode::_utf8_on($unistring);
is ( is (
$tick->Transactions->First->Content, $tick->Transactions->First->Content,
$tick->Transactions->First->Attachments->First->Content, $tick->Transactions->First->Attachments->First->Content,
...@@ -542,8 +541,7 @@ EOF ...@@ -542,8 +541,7 @@ EOF
is ($tick->Id, $id, "correct ticket"); is ($tick->Id, $id, "correct ticket");
is ($tick->Subject , 'This is a test of I18N ticket creation', "Created the ticket"); is ($tick->Subject , 'This is a test of I18N ticket creation', "Created the ticket");
my $unistring = "\303\241\303\251\303\255\303\263\303\272"; my $unistring = Encode::decode("UTF-8","\303\241\303\251\303\255\303\263\303\272");
Encode::_utf8_on($unistring);
ok ( ok (
$tick->Transactions->First->Content =~ $unistring, $tick->Transactions->First->Content =~ $unistring,
...@@ -573,8 +571,7 @@ EOF ...@@ -573,8 +571,7 @@ EOF
my $tick = RT::Test->last_ticket; my $tick = RT::Test->last_ticket;
is ($tick->Id, $id, "correct ticket"); is ($tick->Id, $id, "correct ticket");
my $content = $tick->Transactions->First->Content; my $content = Encode::encode("UTF-8",$tick->Transactions->First->Content);
Encode::_utf8_off($content);
like $content, qr{informaci\303\263n confidencial}; like $content, qr{informaci\303\263n confidencial};
like $content, qr{informaci\357\277\275n confidencial}; like $content, qr{informaci\357\277\275n confidencial};
......
...@@ -10,7 +10,7 @@ my ($baseurl, $m) = RT::Test->started_ok; ...@@ -10,7 +10,7 @@ my ($baseurl, $m) = RT::Test->started_ok;
diag "Testing non-ASCII latin1 in From: header"; diag "Testing non-ASCII latin1 in From: header";
{ {
my $mail = encode( 'iso-8859-1', <<'.' ); my $mail = Encode::encode( 'iso-8859-1', <<'.' );
From: <René@example.com> From: <René@example.com>
Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com> Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
Subject: testing non-ASCII From Subject: testing non-ASCII From
...@@ -34,7 +34,7 @@ here's some content ...@@ -34,7 +34,7 @@ here's some content
diag "Testing non-ASCII latin1 in From: header with MIME-word-encoded phrase"; diag "Testing non-ASCII latin1 in From: header with MIME-word-encoded phrase";
{ {
my $mail = encode( 'iso-8859-1', <<'.' ); my $mail = Encode::encode( 'iso-8859-1', <<'.' );
From: =?iso-8859-1?Q?Ren=E9?= <René@example.com> From: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com> Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
Subject: testing non-ASCII From Subject: testing non-ASCII From
......
...@@ -44,7 +44,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE}; ...@@ -44,7 +44,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
{ {
my $file = my $file =
File::Spec->catfile( RT::Test->temp_directory, encode_utf8 '附件.txt' ); File::Spec->catfile( RT::Test->temp_directory, Encode::encode("UTF-8",'附件.txt') );
open( my $fh, '>', $file ) or die $!; open( my $fh, '>', $file ) or die $!;
binmode $fh, ':utf8'; binmode $fh, ':utf8';
print $fh '附件'; print $fh '附件';
...@@ -59,7 +59,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE}; ...@@ -59,7 +59,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
); );
$m->content_like( qr/Ticket \d+ created/i, 'created the ticket' ); $m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
$m->content_contains( '附件.txt', 'attached filename' ); $m->content_contains( '附件.txt', 'attached filename' );
$m->content_lacks( encode_utf8 '附件.txt', 'no double encoded attached filename' ); $m->content_lacks( Encode::encode("UTF-8",'附件.txt'), 'no double encoded attached filename' );
$m->follow_link_ok( { text => 'with headers' }, $m->follow_link_ok( { text => 'with headers' },
'-> /Ticket/Attachment/WithHeaders/...' ); '-> /Ticket/Attachment/WithHeaders/...' );
......
...@@ -27,7 +27,7 @@ my $url = $agent->rt_base_url; ...@@ -27,7 +27,7 @@ my $url = $agent->rt_base_url;
$agent->goto_create_ticket(1); $agent->goto_create_ticket(1);
is ($agent->status, 200, "Loaded Create.html"); is ($agent->status, 200, "Loaded Create.html");
$agent->form_name('TicketCreate'); $agent->form_name('TicketCreate');
my $string = Encode::decode_utf8("I18N Web Testing æøå"); my $string = Encode::decode("UTF-8","I18N Web Testing æøå");
$agent->field('Subject' => "Ticket with utf8 body"); $agent->field('Subject' => "Ticket with utf8 body");
$agent->field('Content' => $string); $agent->field('Content' => $string);
ok($agent->submit, "Created new ticket with $string as Content"); ok($agent->submit, "Created new ticket with $string as Content");
...@@ -49,7 +49,7 @@ my $url = $agent->rt_base_url; ...@@ -49,7 +49,7 @@ my $url = $agent->rt_base_url;
is ($agent->status, 200, "Loaded Create.html"); is ($agent->status, 200, "Loaded Create.html");
$agent->form_name('TicketCreate'); $agent->form_name('TicketCreate');
my $string = Encode::decode_utf8("I18N Web Testing æøå"); my $string = Encode::decode( "UTF-8","I18N Web Testing æøå");
$agent->field('Subject' => $string); $agent->field('Subject' => $string);
$agent->field('Content' => "Ticket with utf8 subject"); $agent->field('Content' => "Ticket with utf8 subject");
ok($agent->submit, "Created new ticket with $string as Content"); ok($agent->submit, "Created new ticket with $string as Content");
......
...@@ -61,7 +61,7 @@ diag('test real mail outgoing') if $ENV{TEST_VERBOSE}; ...@@ -61,7 +61,7 @@ diag('test real mail outgoing') if $ENV{TEST_VERBOSE};
# $mail is utf8 encoded # $mail is utf8 encoded
my ($mail) = RT::Test->fetch_caught_mails; my ($mail) = RT::Test->fetch_caught_mails;
$mail = decode_utf8 $mail; $mail = Encode::decode("UTF-8", $mail );
like( $mail, qr/你好.*你好/s, 'mail has éèà€' ); like( $mail, qr/你好.*你好/s, 'mail has éèà€' );
like( $mail, qr/éèà€.*éèà€/s, 'mail has éèà€' ); like( $mail, qr/éèà€.*éèà€/s, 'mail has éèà€' );
like( $mail, qr/标题.*标题/s, 'mail has ticket subject 标题' ); like( $mail, qr/标题.*标题/s, 'mail has ticket subject 标题' );
......
...@@ -32,8 +32,7 @@ Text: $text"; ...@@ -32,8 +32,7 @@ Text: $text";
$m->post("$baseurl/REST/1.0/ticket/new", [ $m->post("$baseurl/REST/1.0/ticket/new", [
user => 'root', user => 'root',
pass => 'password', pass => 'password',
# error message from HTTP::Message: content must be bytes content => Encode::encode( "UTF-8", $content),
content => Encode::encode_utf8($content),
], Content_Type => 'form-data' ); ], Content_Type => 'form-data' );
my ($id) = $m->content =~ /Ticket (\d+) created/; my ($id) = $m->content =~ /Ticket (\d+) created/;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment