Commit 7548587b authored by Alex Vandiver's avatar Alex Vandiver
Browse files

Remove remaining cases of "use utf8"

All remaining cases of "use utf8" lie in the testsuite.  As "use utf8"
changes the semantics of dealing with Unicode strings, remove it to
allow programmers to always assume that literals are interpreted as
bytestrings, not characters.  To do otherwise means that one must always
ask if "use utf8" is in scope before performing operations on any
literals; instead, simply make the encodings and decodings explicit.

Note that wide characters may appear in editors, and that the encoding
of the characters on _disk_ will always be UTF-8.  The removal of "use
utf8" merely means that perl will generate a two-byte string from "é",
and not a one-character string.
parent ed0458d7
......@@ -51,7 +51,6 @@
use strict;
use warnings;
use utf8;
use open qw/ :std :encoding(UTF-8) /;
use File::Find;
......@@ -194,7 +193,7 @@ sub extract_strings_from_code {
$seen{$line}++;
unless ( defined $str ) {
print "\n" unless $errors++;
print " Couldn't process loc at $filename:$line:\n str«$str»\n";
print " Couldn't process loc at $filename:$line:\n $str\n";
next;
}
my $interp = (substr($str,0,1) eq '"' ? 1 : 0);
......@@ -238,7 +237,7 @@ sub extract_strings_from_code {
$seen{$line}++;
unless ( defined $str ) {
print "\n" unless $errors++;
print " Couldn't process loc_qw at $filename:$line:\n str«$str»\n";
print " Couldn't process loc_qw at $filename:$line:\n $str\n";
next;
}
foreach my $value (split ' ', $str) {
......@@ -255,7 +254,7 @@ sub extract_strings_from_code {
$seen{$line}++;
unless ( defined $key ) {
print "\n" unless $errors++;
print " Couldn't process loc_left_pair at $filename:$line:\n key«$key»\n";
print " Couldn't process loc_left_pair at $filename:$line:\n $key\n";
next;
}
my $interp = (substr($key,0,1) eq '"' ? 1 : 0);
......@@ -272,7 +271,7 @@ sub extract_strings_from_code {
$seen{$line}++;
unless ( defined $key && defined $val ) {
print "\n" unless $errors++;
print " Couldn't process loc_pair at $filename:$line:\n key«$key»\n val«$val»\n";
print " Couldn't process loc_pair at $filename:$line:\n $key\n $val\n";
next;
}
my $interp_key = (substr($key,0,1) eq '"' ? 1 : 0);
......@@ -293,7 +292,7 @@ sub extract_strings_from_code {
$line += ( $all =~ tr/\n/\n/ );
$seen{$line}++;
unless ( defined $key && defined $val ) {
warn "Couldn't process loc_pair at $filename:$line:\n key«$key»\n val«$val»\n";
warn "Couldn't process loc_pair at $filename:$line:\n $key\n $val\n";
next;
}
$val = substr($val, 1, -1); # dequote always quoted string
......
......@@ -3,8 +3,7 @@ use warnings;
use RT::Test;
use Digest::MD5;
use Encode 'encode_utf8';
use utf8;
use Encode;
my $default = "bcrypt";
......@@ -55,9 +54,9 @@ like($root->__Value("Password"), qr/^\!$default\!/, "And is now upgraded to salt
# Non-ASCII salted truncated SHA-256
my $non_ascii_trunc = MIME::Base64::encode_base64(
"salt" . substr(Digest::SHA::sha256("salt".Digest::MD5::md5(Encode::encode("UTF-8","áěšý"))),0,26),
"salt" . substr(Digest::SHA::sha256("salt".Digest::MD5::md5("áěšý")),0,26),
""
);
$root->_Set( Field => "Password", Value => $non_ascii_trunc);
ok($root->IsPassword("áěšý"), "Unsalted MD5 base64 works");
ok($root->IsPassword(Encode::decode("UTF-8", "áěšý")), "Unsalted MD5 base64 works");
like($root->__Value("Password"), qr/^\!$default\!/, "And is now upgraded to salted $default");
......@@ -13,10 +13,11 @@ $m->content_contains('<html lang="en">');
$m->add_header('Accept-Language' => 'zh-tw,zh;q=0.8,en-gb;q=0.5,en;q=0.3');
$m->get_ok('/');
use utf8;
Encode::_utf8_on($m->{content});
$m->title_is('登入', 'Page title properly translated to chinese');
$m->content_contains('密碼','Password properly translated');
$m->title_is( Encode::decode("UTF-8",'登入'),
'Page title properly translated to chinese');
$m->content_contains( Encode::decode("UTF-8",'密碼'),
'Password properly translated');
{
local $TODO = "We fail to correctly advertise the langauage in the <html> block";
$m->content_contains('<html lang="zh-tw">');
......
......@@ -6,8 +6,6 @@ use RT::Test tests => undef;
plan skip_all => 'GD required'
unless GD->require;
use utf8;
my $root = RT::Test->load_or_create_user( Name => 'root' );
my ( $baseurl, $m ) = RT::Test->started_ok;
......@@ -15,11 +13,11 @@ ok( $m->login, 'logged in' );
my $ticket = RT::Ticket->new( $RT::SystemUser );
$ticket->Create(
Queue => 'General',
Subject => 'test äöü',
Subject => Encode::decode("UTF-8",'test äöü'),
);
ok( $ticket->id, 'created ticket' );
$m->get_ok(q{/Search/Chart.html?Query=Subject LIKE 'test äöü'});
$m->get_ok(Encode::decode("UTF-8", q{/Search/Chart.html?Query=Subject LIKE 'test äöü'}));
$m->submit_form(
form_name => 'SaveSearch',
fields => {
......@@ -52,7 +50,7 @@ $m->field( 'Hour' => '06:00' );
$m->click_button( name => 'Save' );
$m->content_contains('Subscribed to dashboard dashboard foo');
my $c = $m->get(q{/Search/Chart?Query=Subject LIKE 'test äöü'});
my $c = $m->get(Encode::decode("UTF-8",q{/Search/Chart?Query=Subject LIKE 'test äöü'}));
my $image = $c->content;
RT::Test->run_and_capture(
command => $RT::SbinPath . '/rt-email-dashboards', all => 1
......
use strict;
use warnings;
use utf8;
use RT::Test tests => 18;
......
......@@ -3,14 +3,13 @@ use warnings;
use RT::Test tests => 12;
use Test::Warn;
use utf8;
use Encode;
my ($baseurl, $m) = RT::Test->started_ok;
diag "Testing non-ASCII latin1 in From: header";
{
my $mail = Encode::encode( 'iso-8859-1', <<'.' );
my $mail = Encode::encode( 'iso-8859-1', Encode::decode( "UTF-8", <<'.') );
From: <René@example.com>
Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
Subject: testing non-ASCII From
......@@ -34,7 +33,7 @@ here's some content
diag "Testing non-ASCII latin1 in From: header with MIME-word-encoded phrase";
{
my $mail = Encode::encode( 'iso-8859-1', <<'.' );
my $mail = Encode::encode( 'iso-8859-1', Encode::decode( "UTF-8", <<'.') );
From: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
Subject: testing non-ASCII From
......
use strict;
use warnings;
use utf8;
use RT::Test tests => undef;
use RT::Test::Email;
......
use strict;
use warnings;
use utf8;
use RT::Test tests => undef;
my ($baseurl, $m) = RT::Test->started_ok;
ok $m->login, 'logged in as root';
......@@ -20,7 +19,7 @@ diag "encoded attachment filename with parameter continuations";
ok( $id, "Created ticket" );
$m->get_ok("/Ticket/Display.html?id=$id");
$m->content_contains("新しいテキスト ドキュメント.txt", "found full filename");
$m->content_contains(Encode::decode("UTF-8","新しいテキスト ドキュメント.txt"), "found full filename");
}
undef $m;
......
use strict;
use warnings;
use utf8;
use RT::Test tests => 22;
RT->Config->Set( NotifyActor => 1 );
......
......@@ -3,9 +3,8 @@ use warnings;
use RT::Test nodb => 1, tests => undef;
use_ok('RT::I18N');
use utf8;
use Encode;
my $test_string = 'À';
my $test_string = Encode::decode("UTF-8", 'À');
my $encoded_string = encode( 'iso-8859-1', $test_string );
my $mime = MIME::Entity->build(
"Subject" => $encoded_string,
......
......@@ -7,10 +7,12 @@ use Encode;
my ( $baseurl, $m ) = RT::Test->started_ok;
ok $m->login, 'logged in as root';
use utf8;
use File::Spec;
my $subject = Encode::decode("UTF-8",'标题');
my $content = Encode::decode("UTF-8",'测试');
my $filename = Encode::decode("UTF-8",'附件.txt');
diag 'test without attachments' if $ENV{TEST_VERBOSE};
{
......@@ -19,13 +21,13 @@ diag 'test without attachments' if $ENV{TEST_VERBOSE};
$m->form_name('TicketModify');
$m->submit_form(
form_number => 3,
fields => { Subject => '标题', Content => '测试' },
fields => { Subject => $subject, Content => $content },
);
$m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
$m->follow_link_ok( { text => 'with headers' },
'-> /Ticket/Attachment/WithHeaders/...' );
$m->content_contains( '标题', 'has subject 标题' );
$m->content_contains( '测试', 'has content 测试' );
$m->content_contains( $subject, "has subject $subject" );
$m->content_contains( $content, "has content $content" );
my ( $id ) = $m->uri =~ /(\d+)$/;
ok( $id, 'found attachment id' );
......@@ -35,8 +37,8 @@ diag 'test without attachments' if $ENV{TEST_VERBOSE};
ok( $attachment->SetHeader( 'X-RT-Original-Encoding' => 'gbk' ),
'set original encoding to gbk' );
$m->get( $m->uri );
$m->content_contains( '标题', 'has subject 标题' );
$m->content_contains( '测试', 'has content 测试' );
$m->content_contains( $subject, "has subject $subject" );
$m->content_contains( $content, "has content $content" );
}
diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
......@@ -44,10 +46,10 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
{
my $file =
File::Spec->catfile( RT::Test->temp_directory, Encode::encode("UTF-8",'附件.txt') );
File::Spec->catfile( RT::Test->temp_directory, Encode::encode("UTF-8",$filename) );
open( my $fh, '>', $file ) or die $!;
binmode $fh, ':utf8';
print $fh '附件';
print $fh $filename;
close $fh;
$m->get_ok( $baseurl . '/Ticket/Create.html?Queue=1' );
......@@ -55,17 +57,17 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
$m->form_name('TicketModify');
$m->submit_form(
form_number => 3,
fields => { Subject => '标题', Content => '测试', Attach => $file },
fields => { Subject => $subject, Content => $content, Attach => $file },
);
$m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
$m->content_contains( '附件.txt', 'attached filename' );
$m->content_lacks( Encode::encode("UTF-8",'附件.txt'), 'no double encoded attached filename' );
$m->content_contains( $filename, 'attached filename' );
$m->content_lacks( Encode::encode("UTF-8",$filename), 'no double encoded attached filename' );
$m->follow_link_ok( { text => 'with headers' },
'-> /Ticket/Attachment/WithHeaders/...' );
# subject is in the parent attachment, so there is no 标题
$m->content_lacks( '标题', 'does not have content 标题' );
$m->content_contains( '测试', 'has content 测试' );
$m->content_lacks( $subject, "does not have content $subject" );
$m->content_contains( $content, "has content $content" );
my ( $id ) = $m->uri =~ /(\d+)$/;
ok( $id, 'found attachment id' );
......@@ -75,15 +77,15 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
ok( $attachment->SetHeader( 'X-RT-Original-Encoding' => 'gbk' ),
'set original encoding to gbk' );
$m->get( $m->uri );
$m->content_lacks( '标题', 'does not have content 标题' );
$m->content_contains( '测试', 'has content 测试' );
$m->content_lacks( $subject, "does not have content $subject" );
$m->content_contains( $content, "has content $content" );
$m->back;
$m->back;
$m->follow_link_ok( { text => 'Download 附件.txt' },
$m->follow_link_ok( { text => "Download $filename" },
'-> /Ticket/Attachment/...' );
$m->content_contains( '附件', 'has content 附件' );
$m->content_contains( $filename, "has file content $filename" );
( $id ) = $m->uri =~ m{/(\d+)/[^/]+$};
ok( $id, 'found attachment id' );
......@@ -94,7 +96,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
ok( $attachment->SetHeader( 'X-RT-Original-Encoding' => 'gbk' ),
'set original encoding to gbk' );
$m->get( $m->uri );
$m->content_contains( '附件', 'has content 附件' );
$m->content_contains( $filename, "has content $filename" );
unlink $file;
}
......
use strict;
use warnings;
use utf8;
# trs: I'd write a quick t/web/caching-headers.t file which loops the available
# endpoints checking for the right headers.
......
......@@ -2,16 +2,17 @@
use strict;
use warnings;
use RT::Test tests => 19;
use RT::Test tests => undef;
use Encode;
my ( $baseurl, $m ) = RT::Test->started_ok;
ok $m->login, 'logged in as root';
use utf8;
diag('make Autoreply template a html one and add utf8 chars')
if $ENV{TEST_VERBOSE};
my $template = Encode::decode("UTF-8", "你好 éèà€");
my $subject = Encode::decode("UTF-8", "标题");
my $content = Encode::decode("UTF-8", "测试");
{
$m->follow_link_ok( { id => 'admin-global-templates' }, '-> Templates' );
$m->follow_link_ok( { text => 'Autoreply in HTML' }, '-> Autoreply in HTML' );
......@@ -19,20 +20,20 @@ diag('make Autoreply template a html one and add utf8 chars')
$m->submit_form(
form_name => 'ModifyTemplate',
fields => {
Content => <<'EOF',
Subject: AutoReply: {$Ticket->Subject}
Content => <<EOF,
Subject: AutoReply: {\$Ticket->Subject}
Content-Type: text/html
你好 éèà€
{$Ticket->Subject}
$template
{\$Ticket->Subject}
-------------------------------------------------------------------------
{$Transaction->Content()}
{\$Transaction->Content()}
EOF
},
);
$m->content_like( qr/Content updated/, 'content is changed' );
$m->content_contains( '你好', 'content is really updated' );
$m->content_contains( $template, 'content is really updated' );
}
diag('create a ticket to see the autoreply mail') if $ENV{TEST_VERBOSE};
......@@ -42,17 +43,16 @@ diag('create a ticket to see the autoreply mail') if $ENV{TEST_VERBOSE};
$m->submit_form(
form_name => 'TicketCreate',
fields => { Subject => '标题', Content => '<h1>测试</h1>',
fields => { Subject => $subject, Content => "<h1>$content</h1>",
ContentType => 'text/html' },
);
$m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
$m->follow_link( text => 'Show' );
$m->content_contains( '你好', 'html has 你好' );
$m->content_contains( 'éèà€', 'html has éèà€' );
$m->content_contains( '标题',
'html has ticket subject 标题' );
$m->content_contains( '&lt;h1&gt;测试&lt;/h1&gt;',
'html has ticket html content 测试' );
$m->content_contains( $template, "html has $template" );
$m->content_contains( $subject,
"html has ticket subject $subject" );
$m->content_contains( "&lt;h1&gt;$content&lt;/h1&gt;",
"html has ticket html content $content" );
}
diag('test real mail outgoing') if $ENV{TEST_VERBOSE};
......@@ -62,10 +62,11 @@ diag('test real mail outgoing') if $ENV{TEST_VERBOSE};
# $mail is utf8 encoded
my ($mail) = RT::Test->fetch_caught_mails;
$mail = Encode::decode("UTF-8", $mail );
like( $mail, qr/你好.*你好/s, 'mail has éèà€' );
like( $mail, qr/éèà€.*éèà€/s, 'mail has éèà€' );
like( $mail, qr/标题.*标题/s, 'mail has ticket subject 标题' );
like( $mail, qr/测试.*测试/s, 'mail has ticket content 测试' );
like( $mail, qr!<h1>测试</h1>!, 'mail has ticket html content 测试' );
like( $mail, qr/$template.*$template/s, 'mail has template content $template twice' );
like( $mail, qr/$subject.*$subject/s, 'mail has ticket subject $sujbect twice' );
like( $mail, qr/$content.*$content/s, 'mail has ticket content $content twice' );
like( $mail, qr!<h1>$content</h1>!, 'mail has ticket html content <h1>$content</h1>' );
}
undef $m;
done_testing;
use strict;
use warnings;
use File::Spec;
use utf8;
$ENV{RT_TEST_WEB_HANDLER} = 'plack+rt-server';
use RT::Test
......@@ -32,10 +31,10 @@ $m->get_ok($url);
is( $m->uri, $url . '/Install/index.html', 'install page' );
$m->select( 'Lang', 'zh-cn' );
$m->click('ChangeLang');
$m->content_contains( '语言', 'select chinese' );
$m->content_contains( Encode::decode("UTF-8",'语言'), 'select chinese' );
$m->click('Run');
$m->content_contains( '数据库', 'select db type in chinese' );
$m->content_contains( Encode::decode("UTF-8",'数据库'), 'select db type in chinese' );
$m->back;
$m->select( 'Lang', 'en' );
......
use strict;
use warnings;
use utf8;
use RT::Test tests => 9;
my ( $url, $m ) = RT::Test->started_ok;
......@@ -11,8 +10,8 @@ $m->form_with_fields('Lang');
$m->field(Lang => 'zh_TW');
$m->submit;
$m->text_contains("並讓現存的 iCal feeds不再能用", "successfully updated to zh_TW");
$m->text_contains("使用語言 的值從 (無) 改為 'zh_TW'", "when updating to language zh_TW, results are in zh_TW");
$m->text_contains(Encode::decode("UTF-8","並讓現存的 iCal feeds不再能用"), "successfully updated to zh_TW");
$m->text_contains(Encode::decode("UTF-8","使用語言 的值從 (無) 改為 'zh_TW'"), "when updating to language zh_TW, results are in zh_TW");
$m->form_with_fields('Lang');
$m->field(Lang => 'en_us');
......
use strict;
use warnings;
use utf8;
use RT::Test tests => undef;
......@@ -14,10 +13,10 @@ diag "create a ticket via the API";
my $ticket = RT::Ticket->new( RT->SystemUser );
my ($id, $txn, $msg) = $ticket->Create(
Queue => 'General',
Subject => 'bad subject‽',
Subject => Encode::decode("UTF-8",'bad subject‽'),
);
ok $id, 'created a ticket #'. $id or diag "error: $msg";
is $ticket->Subject, 'bad subject‽', 'correct subject';
is $ticket->Subject, Encode::decode("UTF-8",'bad subject‽'), 'correct subject';
push @tickets, $id;
}
......@@ -29,10 +28,10 @@ diag "create a ticket via the web";
}, 'create ticket in Queue');
$m->submit_form_ok({
with_fields => {
Subject => 'bad subject #2‽',
Subject => Encode::decode("UTF-8",'bad subject #2‽'),
},
}, 'create ticket');
$m->content_contains('bad subject #2‽', 'correct subject');
$m->content_contains(Encode::decode("UTF-8",'bad subject #2‽'), 'correct subject');
push @tickets, 2;
}
......@@ -58,12 +57,12 @@ for my $tid (@tickets) {
$m->follow_link_ok({ id => 'page-actions-reply' }, "Actions -> Reply");
$m->submit_form_ok({
with_fields => {
UpdateSubject => 'bad subject‽ without attachment',
UpdateSubject => Encode::decode("UTF-8",'bad subject‽ without attachment'),
UpdateContent => 'testing unicode txn subjects',
},
button => 'SubmitTicket',
}, 'submit reply');
$m->content_contains('bad subject‽ without attachment', "found txn subject");
$m->content_contains(Encode::decode("UTF-8",'bad subject‽ without attachment'), "found txn subject");
}
diag "add a reply which adds to the subject with an attachment";
......@@ -72,13 +71,13 @@ for my $tid (@tickets) {
$m->follow_link_ok({ id => 'page-actions-reply' }, "Actions -> Reply");
$m->submit_form_ok({
with_fields => {
UpdateSubject => 'bad subject‽ with attachment',
UpdateSubject => Encode::decode("UTF-8",'bad subject‽ with attachment'),
UpdateContent => 'testing unicode txn subjects',
Attach => RT::Test::get_relocatable_file('bpslogo.png', '..', 'data'),
},
button => 'SubmitTicket',
}, 'submit reply');
$m->content_contains('bad subject‽ with attachment', "found txn subject");
$m->content_contains(Encode::decode("UTF-8",'bad subject‽ with attachment'), "found txn subject");
}
}
......
use strict;
use warnings;
use utf8;
use RT::Test tests => undef;
my ( $url, $m ) = RT::Test->started_ok;
......@@ -9,8 +8,8 @@ ok( $m->login(), 'logged in' );
$m->follow_link_ok({text => 'About me'});
$m->submit_form_ok({ with_fields => { Lang => 'ja'} },
"Change to Japanese");
$m->text_contains("Langは「(値なし)」から「'ja'」に変更されました");
$m->text_contains("実名", "Page content is japanese");
$m->text_contains(Encode::decode("UTF-8","Langは「(値なし)」から「'ja'」に変更されました"));
$m->text_contains(Encode::decode("UTF-8","実名"), "Page content is japanese");
# we only changed one field, and it wasn't the default, so this feedback is
# spurious and annoying
......@@ -29,13 +28,12 @@ $m->content_lacks("That is already the current value");
# Ensure that we can change the language back to the default.
$m->submit_form_ok({ with_fields => { Lang => 'ja'} },
"Back briefly to Japanese");
$m->text_contains("Langは「'en_us'」から「'ja'」に変更されました");
$m->text_contains("実名", "Page content is japanese");
$m->text_contains(Encode::decode("UTF-8","Langは「'en_us'」から「'ja'」に変更されました"));
$m->text_contains(Encode::decode("UTF-8","実名"), "Page content is japanese");
$m->submit_form_ok({ with_fields => { Lang => ''} },
"And set to the default");
$m->text_contains("Lang changed from 'ja' to (no value)");
$m->text_contains("Real Name", "Page content is english");
undef $m;
done_testing;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment