Merge branch '1.0.x' into prod/1.0.x

README.md

@@ -11,12 +11,18 @@ This can be a local file directory containing the source site
   (e.g. /var/www/docroot), or the site address (e.q. http://example.com).
 * `uw_migrate_site_path` represents the path to the site directory relative
   to Drupal root. Defaults to 'sites/default/files'.
+* `uw_migrate_private_files` represents the path to the private site files
+  directory relative to Drupal root.
 
 Example:
 
 ```
+// Remote source supports public files only.
 $settings['uw_migrate_source'] = 'https://uwaterloo.ca/web-resources';
+// Local directory source supports both public and private files.
+$settings['uw_migrate_source'] = '/var/www/docroot';
 $settings['uw_migrate_site_path'] = 'sites/ca.web-resources/files';
+$settings['uw_migrate_private_files'] = '../files';
 ```
 
 In this example, the source site URL is https://uwaterloo.ca/web-resources

migrations/block_content/uw_cbl_copy_text.yml

@@ -43,6 +43,7 @@ process:
         uw_ct_profile: {}
         uw_ct_web_page: {}
     - plugin: uw_dom_inline_image_handler
+    - plugin: uw_dom_inline_file_handler
     - plugin: dom
       method: export
   field_uw_copy_text/format:

migrations/block_content/uw_cbl_special_alert.yml

@@ -8,7 +8,7 @@ migration_tags:
   - Block
 source:
   plugin: uw_node
-  node_type: uw_ct_special_alert
+  node_type: uw_special_alert
 process:
   plugin:
     plugin: default_value
@@ -20,7 +20,10 @@ process:
     plugin: uw_block_visibility
     source: field_block_visibility/0/value
   settings/display_option: status
-  settings/message/value: body/0/value
+  settings/message/value:
+    plugin: callback
+    callable: trim
+    source: body/0/value
   settings/message/format:
     plugin: default_value
     default_value: uw_tf_basic

migrations/layout_builder/uw_field_body_no_summary_lb.yml

@@ -17,7 +17,7 @@ source:
   conditions:
     - field: bundle
       value: uw_promotional_item
-      operator: !=
+      operator: '!='
   id_fields:
     entity_id:
       type: integer

migrations/layout_builder/uw_field_body_no_summary_sections_lb.yml

@@ -16,7 +16,7 @@ source:
   conditions:
     - field: bundle
       value: uw_promotional_item
-      operator: !=
+      operator: '!='
   id_fields:
     entity_id:
       type: integer

migrations/uw_file.yml → migrations/media/uw_file.yml

 id: uw_file
-label: Public files
+label: Files (public and private)
 migration_group: uw
 audit: true
 migration_tags:
@@ -9,7 +9,6 @@ migration_tags:
   - Media library
 source:
   plugin: uw_file
-  scheme: public
   batch_size: 200
   constants:
     # The value of this constant will be assigned from uw_migrate_source setting,

migrations/paragraph/uw_para_call_to_action.yml

@@ -36,8 +36,8 @@ process:
     -
       plugin: static_map
       map:
-        TRUE: big
-        FALSE: small
+        1: big
+        0: small
   field_uw_cta_text_details/1/text_value:
     -
       plugin: skip_on_empty

migrations/uw_menu_links.yml

@@ -12,6 +12,9 @@ source:
   constants:
     bundle: menu_link_content
 process:
+  # IDs are available only for default links.
+  # See: \Drupal\uw_migrate\Plugin\migrate\source\UwMenuLink::prepareRow().
+  id: uw_mlid
   bundle: 'constants/bundle'
   title: link_title
   description: description
@@ -50,6 +53,11 @@ process:
 destination:
   plugin: entity:menu_link_content
   no_stub: true
+  overwrite_properties:
+    - weight
+    - expanded
+    - enabled
+    - parent
 migration_dependencies:
   optional:
     - uw_ct_blog

migrations/webform/uw_webform_submission.yml

@@ -40,7 +40,12 @@ process:
   sticky: {}
   notes: {}
   serial: {}
-  data: webform_data
+  data:
+    - plugin: single_value
+      source: webform_data
+    - plugin: uw_webform_file_lookup
+      migration: uw_file
+      no_stub: true
 destination:
   plugin: entity:webform_submission
 migration_dependencies:

src/EventSubscriber/MigrateSubscriber.php

@@ -81,7 +81,7 @@ class MigrateSubscriber implements EventSubscriberInterface {
     $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id);
 
     // Force generating entity path alias, but do not save it.
-    $pathauto_alias = \Drupal::service('pathauto.generator')->updateEntityAlias($entity, 'return', ['force' => TRUE]);
+    $pathauto_alias = $this->pathautoGenerator->updateEntityAlias($entity, 'return', ['force' => TRUE]);
     $migrated_alias = $row->getDestinationProperty('alias');
 
     // Determine if migrated alias matches the configured pattern.

src/Plugin/migrate/destination/UwEntityContentBase.php

@@ -21,7 +21,9 @@ class UwEntityContentBase extends EntityContentBase {
     if ($entity->hasField('changed')) {
       $entity->changed->preserve = TRUE;
     }
-    $entity->path->pathauto = PathautoState::SKIP;
+    if ($entity->hasField('path')) {
+      $entity->path->pathauto = PathautoState::SKIP;
+    }
     return parent::save($entity, $old_destination_id_values);
   }
 

src/Plugin/migrate/process/UwDomInlineFileHandler.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\Component\Utility\Html;
+use Drupal\media\MediaInterface;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Row;
+
+/**
+ * Takes inline ref docs from rich text and makes them into media entities.
+ *
+ * Examples:
+ *
+ * @code
+ * process:
+ *   'body/value':
+ *     -
+ *       plugin: dom
+ *       method: import
+ *       source: 'body/0/value'
+ *     -
+ *       plugin: uw_dom_inline_file_handler
+ *     -
+ *       plugin: dom
+ *       method: export
+ * @endcode
+ *
+ * @see \Drupal\migrate\Plugin\MigrateProcessInterface
+ * @see \Drupal\migrate_plus\Plugin\migrate\process\DomProcessBase
+ *
+ * @MigrateProcessPlugin(
+ *   id = "uw_dom_inline_file_handler"
+ * )
+ */
+class UwDomInlineFileHandler extends UwFileDomProcessBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    $allowed_extensions = [
+      'bib', 'csv', 'doc', 'docx', 'gz', 'key', 'm', 'mw', 'mp3', 'rtf', 'pdf',
+      'pot', 'potx', 'pps', 'ppt', 'pptx', 'psd', 'r', 'tar', 'tex', 'txt',
+      'wav', 'xls', 'xlsx', 'zip',
+    ];
+
+    // Initialize DOM handling on this value.
+    $this->init($value, $destination_property);
+
+    // Loop through images to make into file/media entities and replace.
+    $links = $this->xpath->query('//a');
+    foreach ($links as $html_node) {
+      // Get href attribute from the <a>.
+      $href = $html_node->getAttribute('href');
+
+      // Try to retrieve file extension.
+      $path_info = pathinfo($href);
+
+      // Make sure the href is to a .pdf.
+      if (empty($path_info['extension']) || !\in_array($path_info['extension'], $allowed_extensions, TRUE)) {
+        continue;
+      }
+
+      $file_uri = $this->getFileUri($href);
+      $file = $this->loadFileByUri($file_uri);
+
+      if (!$file instanceof MediaInterface) {
+        continue;
+      }
+
+      // Create a new DOM element for the document link in the text.
+      $new_node = $this->document->createElement('drupal-media', '');
+
+      $attributes = [
+        'data-entity-uuid' => $file->uuid(),
+        'data-entity-type' => 'media',
+        'data-caption' => $html_node->nodeValue,
+      ];
+
+      foreach ($attributes as $attr => $val) {
+        if (empty($val)) {
+          continue;
+        }
+        $dom_att = $this->document->createAttribute($attr);
+        $dom_att->value = Html::escape($val);
+        $new_node->appendChild($dom_att);
+      }
+      // Replace the <a href> with <drupal-media>.
+      $html_node->parentNode->replaceChild($new_node, $html_node);
+    }
+    return $this->document;
+  }
+
+}

src/Plugin/migrate/process/UwDomInlineImageHandler.php

@@ -3,17 +3,10 @@
 namespace Drupal\uw_migrate\Plugin\migrate\process;
 
 use Drupal\Component\Utility\Html;
-use Drupal\Component\Utility\UrlHelper;
-use Drupal\Core\Entity\EntityTypeManagerInterface;
-use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
-use Drupal\Core\Site\Settings;
 use Drupal\file\FileInterface;
-use Drupal\file\FileUsage\FileUsageInterface;
 use Drupal\media\MediaInterface;
 use Drupal\migrate\MigrateExecutableInterface;
 use Drupal\migrate\Row;
-use Drupal\migrate_plus\Plugin\migrate\process\DomProcessBase;
-use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * Takes inline images from rich text and makes them into media entities.
@@ -38,43 +31,7 @@ use Symfony\Component\DependencyInjection\ContainerInterface;
  *   id = "uw_dom_inline_image_handler"
  * )
  */
-class UwDomInlineImageHandler extends DomProcessBase implements ContainerFactoryPluginInterface {
-
-  /**
-   * The entity type manager.
-   *
-   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
-   */
-  protected $entityTypeManager;
-
-  /**
-   * The file usage service.
-   *
-   * @var \Drupal\file\FileUsage\FileUsageInterface
-   */
-  protected $fileUsage;
-
-  /**
-   * {@inheritdoc}
-   */
-  public function __construct(array $configuration, $plugin_id, array $plugin_definition, EntityTypeManagerInterface $entity_type_manager, FileUsageInterface $file_usage) {
-    parent::__construct($configuration, $plugin_id, $plugin_definition);
-    $this->entityTypeManager = $entity_type_manager;
-    $this->fileUsage = $file_usage;
-  }
-
-  /**
-   * {@inheritdoc}
-   */
-  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
-    return new static(
-      $configuration,
-      $plugin_id,
-      $plugin_definition,
-      $container->get('entity_type.manager'),
-      $container->get('file.usage')
-    );
-  }
+class UwDomInlineImageHandler extends UwFileDomProcessBase {
 
   /**
    * {@inheritdoc}
@@ -87,31 +44,10 @@ class UwDomInlineImageHandler extends DomProcessBase implements ContainerFactory
     /** @var \DOMElement $html_node */
     foreach ($this->xpath->query('//img') as $html_node) {
       // Get attributes from the image.
-      $src = $html_node->getAttribute('src');
-      $image_path = UrlHelper::parse($src)['path'];
-
-      // Remove the source site from the path.
-      $source = Settings::get('uw_migrate_source', '');
-      // Remove base url if the path is absolute.
-      $source = str_replace('https://uwaterloo.ca', '', $source);
-      if (!empty($source)) {
-        $image_path = str_replace($source, '', $image_path);
-        // The image path usually starts from the /site-name base url.
-        // It should be removed.
-        $parts = parse_url($source);
-        if (!empty($parts['path'])) {
-          $image_path = str_replace($parts['path'], '', $image_path);
-        }
-      }
+      $image_src = $html_node->getAttribute('src');
+      $file_uri = $this->getFileUri($image_src);
+      $image = $this->loadFileByUri($file_uri);
 
-      // Remove site path.
-      $image_path = preg_replace('/.*sites\/.*\/files/', '', $image_path);
-      // Remove image style path prefix.
-      $image_path = preg_replace('/.*styles\/.*\/public/', '', $image_path);
-      // Trim remaining slashes and append scheme.
-      $image_path = 'public://' . urldecode(trim($image_path, '/'));
-
-      $image = $this->loadImageByUrl($image_path);
       if ($image instanceof MediaInterface) {
         // Create a new DOM element for the image in the text.
         $new_node = $this->document->createElement('drupal-media', '');
@@ -173,46 +109,4 @@ class UwDomInlineImageHandler extends DomProcessBase implements ContainerFactory
     return $this->document;
   }
 
-  /**
-   * Loads media or file entity by file URI.
-   *
-   * @param string $uri
-   *   The File URI to search in destination.
-   *
-   * @return \Drupal\media\MediaInterface|\Drupal\file\FileInterface|false
-   *   Media entity or FALSE if not found.
-   */
-  public function loadImageByUrl($uri) {
-    $files = $this->entityTypeManager
-      ->getStorage('file')
-      ->loadByProperties(['uri' => $uri]);
-
-    // Report the following cases:
-    // - there are no files at all;
-    // - there is more than 1 file.
-    if (empty($files)) {
-      // @todo Add log entry about not found file.
-      return FALSE;
-    }
-    $file = current($files);
-
-    // Get file usage to see if there is a media entity.
-    // If there is a media, then return media entity.
-    // if there is a file, then return the file.
-    if ($file instanceof FileInterface) {
-      $file_usages = $this->fileUsage->listUsage($file);
-      if (!empty($file_usages['file']['media'])) {
-        /** @var \Drupal\media\MediaInterface $media */
-        $media = $this->entityTypeManager
-          ->getStorage('media')
-          ->load(key($file_usages['file']['media']));
-        return !empty($media) ? $media : $file;
-      }
-      else {
-        return $file;
-      }
-    }
-    return FALSE;
-  }
-
 }

src/Plugin/migrate/process/UwFileDomProcessBase.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\Component\Utility\UrlHelper;
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\Core\Site\Settings;
+use Drupal\file\FileInterface;
+use Drupal\file\FileUsage\FileUsageInterface;
+use Drupal\migrate_plus\Plugin\migrate\process\DomProcessBase;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Custom base class for file-to-media process plugins.
+ */
+abstract class UwFileDomProcessBase extends DomProcessBase implements ContainerFactoryPluginInterface {
+
+  /**
+   * The entity type manager.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
+   */
+  protected $entityTypeManager;
+
+  /**
+   * The file usage service.
+   *
+   * @var \Drupal\file\FileUsage\FileUsageInterface
+   */
+  protected $fileUsage;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function __construct(array $configuration, $plugin_id, array $plugin_definition, EntityTypeManagerInterface $entity_type_manager, FileUsageInterface $file_usage) {
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+    $this->entityTypeManager = $entity_type_manager;
+    $this->fileUsage = $file_usage;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
+    return new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $container->get('entity_type.manager'),
+      $container->get('file.usage')
+    );
+  }
+
+  /**
+   * Converts source file path into expected destination URI.
+   *
+   * @param string $src
+   *   Original source file path.
+   *
+   * @return string
+   *   The File URI to search in destination.
+   */
+  protected function getFileUri($src) {
+    $uri = UrlHelper::parse($src)['path'];
+
+    // Remove the source site from the path.
+    $source = Settings::get('uw_migrate_source', '');
+    // Remove base url if the path is absolute.
+    $source = str_replace('https://uwaterloo.ca', '', $source);
+
+    if (!empty($source)) {
+      $uri = str_replace($source, '', $uri);
+      // The image path usually starts from the /site-name base url.
+      // It should be removed.
+      $parts = parse_url($source);
+      if (!empty($parts['path'])) {
+        $uri = str_replace($parts['path'], '', $uri);
+      }
+    }
+
+    // Remove site path.
+    $uri = preg_replace('/.*sites\/.*\/files/U', '', $uri);
+    // Remove image style path prefix.
+    $uri = preg_replace('/.*styles\/.*\/public/U', '', $uri);
+    // Trim remaining slashes and append scheme.
+    $uri = 'public://' . urldecode(trim($uri, '/'));
+
+    return $uri;
+  }
+
+  /**
+   * Loads media or file entity by file URI.
+   *
+   * @param string $uri
+   *   The File URI to search in destination.
+   *
+   * @return \Drupal\media\MediaInterface|\Drupal\file\FileInterface|false
+   *   Media entity or FALSE if not found.
+   */
+  public function loadFileByUri($uri) {
+    $files = $this->entityTypeManager
+      ->getStorage('file')
+      ->loadByProperties(['uri' => $uri]);
+
+    // Report the following cases:
+    // - there are no files at all;
+    // - there is more than 1 file.
+    if (empty($files)) {
+      // @todo Add log entry about not found file.
+      return FALSE;
+    }
+    $file = current($files);
+
+    // Get file usage to see if there is a media entity.
+    // If there is a media, then return media entity.
+    // if there is a file, then return the file.
+    if ($file instanceof FileInterface) {
+      $file_usages = $this->fileUsage->listUsage($file);
+      if (!empty($file_usages['file']['media'])) {
+        /** @var \Drupal\media\MediaInterface $media */
+        $media = $this->entityTypeManager
+          ->getStorage('media')
+          ->load(key($file_usages['file']['media']));
+        return !empty($media) ? $media : $file;
+      }
+      return $file;
+    }
+    return FALSE;
+  }
+
+}

src/Plugin/migrate/process/UwMigrationLookupPath.php

@@ -42,9 +42,12 @@ class UwMigrationLookupPath extends UwNodeLookup {
    */
   public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
     if (isset($value['url'])) {
+      // The following line is required due to a bug in phpcs.
+      // phpcs:ignore DrupalPractice.CodeAnalysis.VariableAnalysis.UnusedVariable
       $uri = &$value['url'];
     }
     elseif (is_array($value) && isset($value[0])) {
+      // phpcs:ignore DrupalPractice.CodeAnalysis.VariableAnalysis.UnusedVariable
       $uri = &$value[0];
     }
     else {

src/Plugin/migrate/process/UwWebformFileLookup.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Plugin\migrate\process\MigrationLookup;
+use Drupal\migrate\Row;
+
+/**
+ * Custom lookup plugin for files in the webform submitted data.
+ *
+ * Example:
+ *
+ * @code
+ * process:
+ *   type:
+ *     plugin: uw_webform_file_lookup
+ *     migration: uw_file
+ *     source: webform_data
+ * @endcode
+ *
+ * @MigrateProcessPlugin(
+ *   id = "uw_webform_file_lookup"
+ * )
+ */
+class UwWebformFileLookup extends MigrationLookup {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    $file_components = array_keys($row->getDestinationProperty('webform_components'), 'file');
+
+    // Lookup destination files in the specified migration.
+    foreach ($file_components as $form_key) {
+      $value[$form_key] = parent::transform($value[$form_key], $migrate_executable, $row, $destination_property);
+    }
+    return $value;
+  }
+
+}

src/Plugin/migrate/source/UmMenuLink.php

@@ -87,31 +87,23 @@ class UmMenuLink extends MenuLink {
   public function prepareRow(Row $row) {
     $source_path = $row->getSourceProperty('router_path');
 
-    // This is a bit hacky, but we want to edit the status of existing links
-    // provided by installation profile, based on their D7 settings.
+    // Create a list of default links and their IDs.
+    // See: _uw_sites_all_get_menu_items().
+    // @Todo Consider load the list from _uw_sites_all_get_menu_items().
     $default_links = [
-      'people-profiles' => 'views_view:views.uw_view_profiles.profile_page',
-      'contacts' => 'views_view:views.uw_view_contacts.contact_page',
-      'blog' => 'uw_main_menu.blog',
-      'events' => 'uw_main_menu.events',
-      'news' => 'uw_main_menu.news',
+      'contacts' => 1,
+      'people-profiles' => 2,
+      'blog' => 3,
+      'events' => 4,
+      'news' => 5,
+      'catalogs' => 6,
     ];