Ensure all MIME::Entity bodies are UTF-8 encoded bytes

Placing wide characters into MIME::Entity objects can lead to
double-encoding.  Always treat them as byte stores, encoding as UTF-8
and noting their character set.

In the case of Approvals/index.html, there was no need for an explicit
MIME::Entity object; ->Correspond creates one as needed from a "Content"
argument.

lib/RT/Action/CreateTickets.pm

@@ -736,10 +736,10 @@ sub ParseLines {
     );
 
     if ( $args{content} ) {
-        my $mimeobj = MIME::Entity->new();
-        $mimeobj->build(
-            Type => $args{'contenttype'} || 'text/plain',
-            Data => $args{'content'}
+        my $mimeobj = MIME::Entity->build(
+            Type    => $args{'contenttype'} || 'text/plain',
+            Charset => 'UTF-8',
+            Data    => [ map {Encode::encode( "UTF-8", $_ )} @{$args{'content'}} ],
         );
         $ticketargs{MIMEObj} = $mimeobj;
         $ticketargs{UpdateType} = $args{'updatetype'} || 'correspond';

lib/RT/Attachment.pm

@@ -150,12 +150,9 @@ sub Create {
     my $content;
     unless ( $head->get('Content-Length') ) {
         my $length = 0;
-        if ( defined $Attachment->bodyhandle ) {
-            $content = $Attachment->bodyhandle->as_string;
-            utf8::encode( $content ) if utf8::is_utf8( $content );
-            $length = length $content;
-        }
-        $head->replace( 'Content-Length' => $length );
+        $length = length $Attachment->bodyhandle->as_string
+            if defined $Attachment->bodyhandle;
+        $head->replace( 'Content-Length' => Encode::encode( "UTF-8", $length ) );
     }
     $head = $head->as_string;
 

lib/RT/Dashboard/Mailer.pm

@@ -386,9 +386,14 @@ sub BuildEmail {
 
             $cid_of{$uri} = time() . $$ . int(rand(1e6));
 
-            # downgrade non-text strings, because all strings are utf8 by
-            # default, which is wrong for non-text strings.
-            if ( $mimetype !~ m{text/} ) {
+            # Encode textual data in UTF-8, and downgrade (treat
+            # codepoints as codepoints, and ensure the UTF-8 flag is
+            # off) everything else.
+            my @extra;
+            if ( $mimetype =~ m{text/} ) {
+                $data = Encode::encode( "UTF-8", $data );
+                @extra = ( Charset => "UTF-8" );
+            } else {
                 utf8::downgrade( $data, 1 ) or $RT::Logger->warning("downgrade $data failed");
             }
 
@@ -400,6 +405,7 @@ sub BuildEmail {
                 Disposition  => 'inline',
                 Name         => RT::Interface::Email::EncodeToMIME( String => $filename ),
                 'Content-Id' => $cid_of{$uri},
+                @extra,
             );
 
             return "cid:$cid_of{$uri}";

lib/RT/Interface/Email.pm

@@ -263,7 +263,11 @@ sub MailError {
     my $entity = MIME::Entity->build(%entity_args);
     SetInReplyTo( Message => $entity, InReplyTo => $args{'MIMEObj'} );
 
-    $entity->attach( Data => $args{'Explanation'} . "\n" );
+    $entity->attach(
+        Type    => "text/plain",
+        Charset => "UTF-8",
+        Data    => Encode::encode( "UTF-8", $args{'Explanation'} . "\n" ),
+    );
 
     if ( $args{'MIMEObj'} ) {
         $args{'MIMEObj'}->sync_headers;
@@ -271,7 +275,7 @@ sub MailError {
     }
 
     if ( $args{'Attach'} ) {
-        $entity->attach( Data => $args{'Attach'}, Type => 'message/rfc822' );
+        $entity->attach( Data => Encode::encode( "UTF-8", $args{'Attach'} ), Type => 'message/rfc822' );
 
     }
 

lib/RT/Interface/Web.pm

@@ -2477,7 +2477,7 @@ sub MakeMIMEEntity {
         $Message->attach(
             Type    => $args{'Type'} || 'text/plain',
             Charset => 'UTF-8',
-            Data    => $args{'Body'},
+            Data    => Encode::encode( "UTF-8", $args{'Body'} ),
         );
     }
 
@@ -2500,7 +2500,7 @@ sub MakeMIMEEntity {
             $Message->attach(
                 Type     => $uploadinfo->{'Content-Type'},
                 Filename => $filename,
-                Data     => \@content,
+                Data     => \@content, # Bytes, as read directly from the file, above
             );
             if ( !$args{'Subject'} && !( defined $args{'Body'} && length $args{'Body'} ) ) {
                 $Message->head->replace( 'Subject' => $filename );

lib/RT/Template.pm

@@ -678,6 +678,7 @@ sub _DowngradeFromHTML {
     # need to decode_utf8, see the doc of MIMEObj method
     $body = Encode::decode_utf8( $body );
     my $html = RT::Interface::Email::ConvertHTMLToText( $body );
+    $html = Encode::encode( "UTF-8", $html );
     return unless defined $html;
 
     $new_entity->bodyhandle(MIME::Body::InCore->new( \$html ));

lib/RT/Test.pm

@@ -869,7 +869,9 @@ sub create_ticket {
         $args{'MIMEObj'} = MIME::Entity->build(
             From    => $args{'Requestor'},
             Subject => $args{'Subject'},
-            Data    => $content,
+            Type    => "text/plain",
+            Charset => "UTF-8",
+            Data    => Encode::encode( "UTF-8", $content ),
         );
     }
 

lib/RT/Ticket.pm

@@ -1564,8 +1564,11 @@ sub _RecordNote {
     }
 
     unless ( $args{'MIMEObj'} ) {
+        my $data = ref $args{'Content'}? $args{'Content'} : [ $args{'Content'} ];
         $args{'MIMEObj'} = MIME::Entity->build(
-            Data => ( ref $args{'Content'}? $args{'Content'}: [ $args{'Content'} ] )
+            Type    => "text/plain",
+            Charset => "UTF-8",
+            Data    => [ map {Encode::encode("UTF-8", $_)} @{$data} ],
         );
     }
 
@@ -1656,7 +1659,7 @@ sub DryRun {
         Type    => 'text/plain',
         Subject => defined $args{UpdateSubject} ? Encode::encode_utf8( $args{UpdateSubject} ) : "",
         Charset => 'UTF-8',
-        Data    => $args{'UpdateContent'} || "",
+        Data    => Encode::encode("UTF-8", $args{'UpdateContent'} || ""),
     );
 
     my ( $Transaction, $Description, $Object ) = $self->$action(
@@ -1686,12 +1689,12 @@ sub DryRunCreate {
     my $self = shift;
     my %args = @_;
     my $Message = MIME::Entity->build(
-        Type    => 'text/plain',
         Subject => defined $args{Subject} ? Encode::encode_utf8( $args{'Subject'} ) : "",
         (defined $args{'Cc'} ?
              ( Cc => Encode::encode_utf8( $args{'Cc'} ) ) : ()),
+        Type    => 'text/plain',
         Charset => 'UTF-8',
-        Data    => $args{'Content'} || "",
+        Data    => Encode::encode( "UTF-8", $args{'Content'} || ""),
     );
 
     my ( $Transaction, $Object, $Description ) = $self->Create(
@@ -3057,7 +3060,7 @@ sub Forward {
     my $mime = MIME::Entity->build(
         Subject => $args{Subject},
         Type    => $args{ContentType},
-        Data    => $args{Content},
+        Data    => Encode::encode( "UTF-8", $args{Content} ),
     );
 
     $mime->head->replace(

share/html/Approvals/index.html

@@ -72,12 +72,9 @@ foreach my $arg ( keys %ARGS ) {
     next if $skip_update;
 
     if ( $ARGS{ "Approval-" . $ticket->Id . "-Notes" } ) {
-        my $notes = MIME::Entity->build(
-            Data => [ $ARGS{ "Approval-" . $ticket->Id . "-Notes" } ]
+        my ( $notesval, $notesmsg ) = $ticket->Correspond(
+            Content => $ARGS{ "Approval-" . $ticket->Id . "-Notes" }
         );
-        RT::I18N::SetMIMEEntityToUTF8($notes); # convert text parts into utf-8
-
-        my ( $notesval, $notesmsg ) = $ticket->Correspond( MIMEObj => $notes );
         if ($notesval) {
                 push ( @actions, loc("Approval #[_1]: Notes recorded",$ticket->Id ));
         } else {

share/html/REST/1.0/Forms/ticket/comment

@@ -91,8 +91,9 @@ my $ent = MIME::Entity->build(
     'X-RT-Interface' => 'REST',
 );
 $ent->attach(
-    'Content-Type' => $changes{'Content-Type'} || 'text/plain',
-    Data => $changes{Text},
+    Type    => $changes{'Content-Type'} || 'text/plain',
+    Charset => "UTF-8",
+    Data    => Encode::encode("UTF-8", $changes{Text} ),
 ) if $changes{Text};
 
 

share/html/REST/1.0/Forms/ticket/default

@@ -198,8 +198,9 @@ else {
                     'X-RT-Interface' => 'REST',
                 );
             $v{MIMEObj}->attach(
-                Data => $text,
-                'Content-Type' => $v{'Content-Type'} || 'text/plain',
+                Type    => $v{'Content-Type'} || 'text/plain',
+                Charset => "UTF-8",
+                Data    => Encode::encode( "UTF-8", $text ),
             ) if $text;
             my ($status, $msg) = process_attachments($v{'MIMEObj'}, @atts);
             unless ($status) {

share/html/REST/1.0/ticket/comment

@@ -108,7 +108,11 @@ my $ent = MIME::Entity->build(
     Type => "multipart/mixed",
     'X-RT-Interface' => 'REST',
 );
-$ent->attach(Data => $k->{Text}) if $k->{Text};
+$ent->attach(
+    Type    => "text/plain",
+    Charset => "UTF-8",
+    Data    => Encode::encode( "UTF-8", $k->{Text} ),
+) if $k->{Text};
 
 {
     my ($res, $msg) = process_attachments($ent, @atts);