Merge branch '1.0.x' into prod/1.0.x

README.md

@@ -11,8 +11,6 @@ This can be a local file directory containing the source site
   (e.g. /var/www/docroot), or the site address (e.q. http://example.com).
 * `uw_migrate_site_path` represents the path to the site directory relative
   to Drupal root. Defaults to 'sites/default/files'.
-* `uw_migrate_private_files` represents the path to the private site files
-  directory relative to Drupal root.
 
 Example:
 
@@ -22,7 +20,8 @@ $settings['uw_migrate_source'] = 'https://uwaterloo.ca/web-resources';
 // Local directory source supports both public and private files.
 $settings['uw_migrate_source'] = '/var/www/docroot';
 $settings['uw_migrate_site_path'] = 'sites/ca.web-resources/files';
-$settings['uw_migrate_private_files'] = '../files';
+// Use session cookie (admin user) to download private files from remote server.
+$settings['uw_migrate_cookies'] = ['SESS...' => 'hello'];
 ```
 
 In this example, the source site URL is https://uwaterloo.ca/web-resources

migrations/media/uw_file.yml

@@ -11,23 +11,22 @@ source:
   plugin: uw_file
   batch_size: 200
   constants:
-    # The value of this constant will be assigned from uw_migrate_source setting,
-    # which should be supplied in settings.php file.
+    # The value of this constant will be assigned from uw_migrate_source
+    # setting, which should be supplied in settings.php file.
     # @see: uw_migrate/README.md file for details.
     source_base_path: ''
 process:
   filename: filename
   source_full_path:
     -
-      plugin: concat
-      delimiter: /
+      plugin: uw_file_source
       source:
         - constants/source_base_path
         - filepath
     -
       plugin: urlencode
   uri:
-    plugin: file_copy
+    plugin: uw_file_copy
     source:
       - '@source_full_path'
       - uri

migrations/webform/uw_webform_submission.yml

@@ -43,6 +43,7 @@ process:
   data:
     - plugin: single_value
       source: webform_data
+    - plugin: uw_webform_uwaddress
     - plugin: uw_webform_file_lookup
       migration: uw_file
       no_stub: true

src/Plugin/migrate/process/UwDomInlineFileHandler.php

@@ -4,6 +4,7 @@ namespace Drupal\uw_migrate\Plugin\migrate\process;
 
 use Drupal\Component\Utility\Html;
 use Drupal\file\Entity\File;
+use Drupal\file\FileInterface;
 use Drupal\media\MediaInterface;
 use Drupal\migrate\MigrateExecutableInterface;
 use Drupal\migrate\Row;
@@ -64,19 +65,23 @@ class UwDomInlineFileHandler extends UwFileDomProcessBase {
       }
 
       $file_uri = $this->getFileUri($href);
-      $media = $this->loadFileByUri($file_uri);
+      $file = $this->loadFileByUri($file_uri);
 
-      if (!$media instanceof MediaInterface) {
+      // $file could be either MediaInterface or FileInterface. Load the file if
+      // this is a MediaInterface.
+      if ($file instanceof MediaInterface) {
+        $file = $file->getSource()->getSourceFieldValue($file);
+        $file = File::load($file);
+      }
+
+      // Do nothing if there is no file.
+      if (!$file instanceof FileInterface) {
         continue;
       }
 
       // Create a new <a href> with an updated URL.
       $new_node = $this->document->createElement('a', $html_node->nodeValue);
 
-      // Load the file for this media item.
-      $file = $media->getSource()->getSourceFieldValue($media);
-      $file = File::load($file);
-
       // Attributes for the 'a' element.
       $attributes = [
         'href' => $file->createFileUrl(),

src/Plugin/migrate/process/UwDownload.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\Core\File\FileSystemInterface;
+use Drupal\Core\Site\Settings;
+use Drupal\migrate\Plugin\migrate\process\Download;
+use GuzzleHttp\Client;
+use GuzzleHttp\Cookie\CookieJar;
+
+/**
+ * Custom download plugin to pass cookies jar into HTTP client requests.
+ *
+ * @MigrateProcessPlugin(
+ *   id = "uw_download"
+ * )
+ */
+class UwDownload extends Download {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function __construct(array $configuration, $plugin_id, array $plugin_definition, FileSystemInterface $file_system, Client $http_client) {
+    $cookies = Settings::get('uw_migrate_cookies', []);
+    $source = Settings::get('uw_migrate_source', '');
+
+    if (!empty($cookies) && !empty($source)) {
+      $domain = parse_url($source, PHP_URL_HOST);
+      $jar = CookieJar::fromArray($cookies, $domain);
+      $configuration += [
+        'guzzle_options' => [
+          'cookies' => $jar,
+        ],
+      ];
+    }
+    parent::__construct($configuration, $plugin_id, $plugin_definition, $file_system, $http_client);
+  }
+
+}

src/Plugin/migrate/process/UwFileCopy.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\migrate\Plugin\migrate\process\FileCopy;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Custom file copy plugin to swap download plugin with a custom one.
+ *
+ * @MigrateProcessPlugin(
+ *   id = "uw_file_copy"
+ * )
+ */
+class UwFileCopy extends FileCopy {
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
+    return new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $container->get('stream_wrapper_manager'),
+      $container->get('file_system'),
+      $container->get('plugin.manager.migrate.process')->createInstance('uw_download', $configuration)
+    );
+  }
+
+}

src/Plugin/migrate/process/UwFileSource.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\ProcessPluginBase;
+use Drupal\migrate\Row;
+
+/**
+ * Custom process plugin to contact (or not) file path pieces based on scheme.
+ *
+ * @code
+ * process:
+ *   type:
+ *     plugin: uw_file_source
+ *     source:
+ *       - source_base_path
+ *       - value
+ * @endcode
+ *
+ * @MigrateProcessPlugin(
+ *   id = "uw_file_source"
+ * )
+ */
+class UwFileSource extends ProcessPluginBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    $file_uri = $row->getSourceProperty('uri');
+
+    // Append "system/files" for private files.
+    if (strpos($file_uri, 'private://') === 0) {
+      array_splice($value, 1, 0, 'system/files');
+      // Make sure there are no extra slashes.
+      $value = array_map(static function ($item) {
+        return trim($item, '/');
+      }, $value);
+    }
+    return implode('/', $value);
+  }
+
+}

src/Plugin/migrate/process/UwMigrationLookupPath.php

@@ -60,6 +60,7 @@ class UwMigrationLookupPath extends UwNodeLookup {
     static $uw_migrate_settings;
     if (!isset($uw_migrate_settings)) {
       $uw_migrate_settings['source'] = parse_url(Settings::get('uw_migrate_source'));
+      $uw_migrate_settings['source']['host'] = $uw_migrate_settings['source']['host'] ?? NULL;
       // Normalize by removing trailing slash, if any.
       $uw_migrate_settings['source']['path'] = $uw_migrate_settings['source']['path'] ?? NULL;
       $uw_migrate_settings['source']['path'] = rtrim($uw_migrate_settings['source']['path'], '/');

src/Plugin/migrate/process/UwWebformUwaddress.php

+<?php
+
+namespace Drupal\uw_migrate\Plugin\migrate\process;
+
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Plugin\migrate\process\MigrationLookup;
+use Drupal\migrate\Row;
+
+/**
+ * Custom plugin to migrate Webform component uwaddress.
+ *
+ * Example:
+ *
+ * @code
+ * process:
+ *   type:
+ *     plugin: uw_webform_uwaddress
+ * @endcode
+ *
+ * @MigrateProcessPlugin(
+ *   id = "uw_webform_uwaddress"
+ * )
+ */
+class UwWebformUwaddress extends MigrationLookup {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    // Get form keys for all uwaddress components.
+    $address_components = array_keys($row->getDestinationProperty('webform_components'), 'uwaddress');
+
+    // Map the address sub-components to the keys used in Webform.
+    $address_mapping = [
+      'address' => 'addr_1',
+      'address_2' => 'addr_2',
+      'city' => 'city',
+      'state_province' => 'province',
+      'postal_code' => 'postal_code',
+      'country' => 'country',
+    ];
+    foreach ($address_components as $form_key) {
+      $new_value = [];
+      foreach ($address_mapping as $to => $from) {
+        $new_value[$to] = $value[$form_key][$from];
+      }
+      $value[$form_key] = $new_value;
+    }
+
+    return $value;
+  }
+
+}

src/Plugin/migrate/source/UwFile.php

@@ -70,7 +70,6 @@ class UwFile extends File {
   protected function initializeIterator() {
     $iterator = parent::initializeIterator();
     $this->publicPath = Settings::get('uw_migrate_site_path', $this->publicPath);
-    $this->privatePath = Settings::get('uw_migrate_private_files', $this->privatePath);
     return $iterator;
   }
 

src/Plugin/migrate/source/UwMenuLink.php

@@ -86,6 +86,8 @@ class UwMenuLink extends MenuLink {
    */
   public function prepareRow(Row $row) {
     $source_path = $row->getSourceProperty('router_path');
+    $menu_name = $row->getSourceProperty('menu_name');
+    $link_path = $row->getSourceProperty('link_path');
 
     // Create a list of default links and their IDs.
     // See: _uw_sites_all_get_menu_items().
@@ -99,10 +101,16 @@ class UwMenuLink extends MenuLink {
       'catalogs' => 6,
     ];
 
+    // Skip the home page link as the link is now an icon and has already
+    // been created.
+    if ($link_path == '<front>' && $menu_name == 'main-menu') {
+      return FALSE;
+    }
+
     // Assign an actual ID to update (not create) the link for selected paths.
     // We're using a separate uw_mlid property to preserve the original mlid
     // value.
-    if ($source_path && isset($default_links[$source_path]) && $row->getSourceProperty('menu_name') === 'main-menu') {
+    if ($source_path && isset($default_links[$source_path]) && $menu_name === 'main-menu') {
       $row->setSourceProperty('uw_mlid', $default_links[$source_path]);
     }
     return parent::prepareRow($row);

src/Plugin/migrate/source/UwWebformSubmission.php

@@ -56,7 +56,12 @@ class UwWebformSubmission extends D7WebformSubmission {
     foreach ($wf_submissions as $wf_submission) {
       $extra = unserialize($wf_submission['extra']);
       $destination_form_key = $element['form_key'] = $this->getDestinationFormKey($wf_submission['form_key'], $wf_submission['pid']);
-      $is_multiple = !empty($extra['multiple']) || $wf_submission['type'] === 'grid';
+
+      $multi_value_types = [
+        'grid',
+        'uwaddress',
+      ];
+      $is_multiple = !empty($extra['multiple']) || in_array($wf_submission['type'], $multi_value_types, TRUE);
 
       $item = $is_multiple
         ? $submitted_data[$destination_form_key] ?? []

tests/src/Kernel/UwMigrateTest.php

@@ -42,7 +42,7 @@ class UwMigrateTest extends MigrateDrupal7TestBase {
   /**
    * {@inheritdoc}
    */
-  public static $modules = [
+  protected static $modules = [
     'datetime', 'image', 'link', 'node', 'taxonomy', 'user', 'text', 'file',
     'simplify_menu', 'image', 'media', 'field', 'block', 'block_content',
     'layout_builder', 'paragraphs', 'migrate_plus', 'path', 'crop', 'views',
@@ -65,7 +65,7 @@ class UwMigrateTest extends MigrateDrupal7TestBase {
   /**
    * {@inheritdoc}
    */
-  protected function setUp() {
+  protected function setUp(): void {
     parent::setUp();
 
     $this->installSchema('system', ['sequences', 'key_value_expire']);
@@ -772,6 +772,9 @@ class UwMigrateTest extends MigrateDrupal7TestBase {
     $this->assertWebform('webform_50', [
       'title' => 'Program change form',
     ]);
+    $this->assertWebform('webform_102', [
+      'title' => 'Contact us webform',
+    ]);
 
     $this->executeMigration('uw_webform_submission');
     $this->assertWebformSubmission(1, 'webform_49', [
@@ -790,6 +793,18 @@ class UwMigrateTest extends MigrateDrupal7TestBase {
       'reason_for_change' => "It's old school.",
       'student_id_number' => '12345678',
     ]);
+    $this->assertWebformSubmission(4, 'webform_102', [
+      'address' => [
+        'address' => 'ffg',
+        'address_2' => 'hjjj',
+        'city' => 'Kitchener',
+        'country' => 'CA',
+        'postal_code' => 'N1N 1N1',
+        'state_province' => 'ON',
+      ],
+      'email' => 'Thedog@gmail.com',
+      'phone_number' => '5194952956',
+    ]);
 
     // Run layout builder migration for webform blocks.
     $this->executeMigration('uw_webform_lb');

tests/src/Kernel/process/UwSmartDateTest.php

@@ -42,7 +42,7 @@ class UwSmartDateTest extends KernelTestBase {
   /**
    * {@inheritdoc}
    */
-  protected function setUp() {
+  protected function setUp(): void {
     parent::setUp();
 
     $this->installEntitySchema('node');