Issue #1147734 by philipnorton42, znerol, David_Rothstein, twistor | johnv:...

Issue #1147734 by philipnorton42, znerol, David_Rothstein, twistor | johnv: Added Import files in public://feeds/ vs. private://feeds/.

feeds.module

@@ -1178,3 +1178,43 @@ function feeds_get_feed_nid_entity_callback($entity, array $options, $name, $ent
   }
   return $feed_nid;
 }
+
+/**
+ * Implements hook_file_download().
+ */
+function feeds_file_download($uri) {
+  $id = db_query("SELECT id FROM {feeds_source} WHERE source = :uri", array(':uri' => $uri))->fetchField();
+
+  if (!$id) {
+    // File is not associated with a feed.
+    return;
+  }
+
+   // Get the file record based on the URI. If not in the database just return.
+  $files = file_load_multiple(array(), array('uri' => $uri));
+  foreach ($files as $item) {
+    // Since some database servers sometimes use a case-insensitive comparison
+    // by default, double check that the filename is an exact match.
+    if ($item->uri === $uri) {
+      $file = $item;
+      break;
+    }
+  }
+  if (!isset($file)) {
+    return;
+  }
+
+  // Check if this file belongs to Feeds.
+  $usage_list = file_usage_list($file);
+  if (!isset($usage_list['feeds'])) {
+    return;
+  }
+
+  if (!feeds_access('import', $id)) {
+    // User does not have permission to import this feed.
+    return -1;
+  }
+
+  // Return file headers.
+  return file_get_content_headers($file);
+}

feeds.pages.inc

@@ -341,7 +341,12 @@ function theme_feeds_upload($variables) {
     $wrapper = file_stream_wrapper_get_instance_by_uri($file->uri);
     $description .= '<div class="file-info">';
     $description .= '<div class="file-name">';
-    $description .= l($file->filename, $wrapper->getExternalUrl());
+    if ($wrapper) {
+      $description .= l($file->filename, $wrapper->getExternalUrl());
+    }
+    else {
+      $description .= t('URI scheme %scheme not available.', array('%scheme' =>  file_uri_scheme($uri)));
+    }
     $description .= '</div>';
     $description .= '<div class="file-size">';
     $description .= format_size($file->filesize);

plugins/FeedsFileFetcher.inc

@@ -19,7 +19,7 @@ class FeedsFileFetcherResult extends FeedsFetcherResult {
   }
 
   /**
-   * Overrides parent::getRaw();
+   * Overrides parent::getRaw().
    */
   public function getRaw() {
     return $this->sanitizeRaw(file_get_contents($this->file_path));
@@ -69,14 +69,14 @@ class FeedsFileFetcher extends FeedsFetcher {
   }
 
   /**
-   * Return an array of files in a directory.
+   * Returns an array of files in a directory.
    *
-   * @param $dir
+   * @param string $dir
    *   A stream wreapper URI that is a directory.
    *
-   * @return
-   *   An array of stream wrapper URIs pointing to files. The array is empty
-   *   if no files could be found. Never contains directories.
+   * @return array
+   *   An array of stream wrapper URIs pointing to files. The array is empty if
+   *   no files could be found. Never contains directories.
    */
   protected function listFiles($dir) {
     $dir = file_stream_wrapper_uri_normalize($dir);
@@ -118,7 +118,7 @@ class FeedsFileFetcher extends FeedsFetcher {
       $form['source'] = array(
         '#type' => 'textfield',
         '#title' => t('File'),
-        '#description' => t('Specify a path to a file or a directory. Path must start with @scheme://', array('@scheme' => file_default_scheme())),
+        '#description' => t('Specify a path to a file or a directory. Prefix the path with a scheme. Available schemes: @schemes.', array('@schemes' => implode(', ', $this->config['allowed_schemes']))),
         '#default_value' => empty($source_config['source']) ? '' : $source_config['source'],
       );
     }
@@ -126,34 +126,53 @@ class FeedsFileFetcher extends FeedsFetcher {
   }
 
   /**
-   * Override parent::sourceFormValidate().
+   * Overrides parent::sourceFormValidate().
    */
   public function sourceFormValidate(&$values) {
     $values['source'] = trim($values['source']);
 
-    $feed_dir = 'public://feeds';
-    file_prepare_directory($feed_dir, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS);
+    if (empty($this->config['direct'])) {
 
-    // If there is a file uploaded, save it, otherwise validate input on
-    // file.
-    // @todo: Track usage of file, remove file when removing source.
-    if ($file = file_save_upload('feeds', array('file_validate_extensions' => array(0 => $this->config['allowed_extensions'])), $feed_dir)) {
-      $values['source'] = $file->uri;
-      $values['file'] = $file;
-    }
-    elseif (empty($values['source'])) {
-      form_set_error('feeds][source', t('Upload a file first.'));
+      $feed_dir = $this->config['directory'];
+
+      if (!file_prepare_directory($feed_dir, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS)) {
+        if (user_access('administer feeds')) {
+          $plugin_key = feeds_importer($this->id)->config[$this->pluginType()]['plugin_key'];
+          $link = url('admin/structure/feeds/' . $this->id . '/settings/' . $plugin_key);
+          form_set_error('feeds][FeedsFileFetcher][source', t('Upload failed. Please check the upload <a href="@link">settings.</a>', array('@link' => $link)));
+        }
+        else {
+          form_set_error('feeds][FeedsFileFetcher][source', t('Upload failed. Please contact your site administrator.'));
+        }
+        watchdog('feeds', 'The upload directory %directory required by a feed could not be created or is not accessible. A newly uploaded file could not be saved in this directory as a consequence, and the upload was canceled.', array('%directory' => $feed_dir));
+      }
+      // Validate and save uploaded file.
+      elseif ($file = file_save_upload('feeds', array('file_validate_extensions' => array(0 => $this->config['allowed_extensions'])), $feed_dir)) {
+        $values['source'] = $file->uri;
+        $values['file'] = $file;
+      }
+      elseif (empty($values['source'])) {
+        form_set_error('feeds][FeedsFileFetcher][source', t('Please upload a file.'));
+      }
+      else {
+        // File present from previous upload. Nothing to validate.
+      }
     }
-    // If a file has not been uploaded and $values['source'] is not empty, make
-    // sure that this file is within Drupal's files directory as otherwise
-    // potentially any file that the web server has access to could be exposed.
-    elseif (strpos($values['source'], file_default_scheme()) !== 0) {
-      form_set_error('feeds][source', t('File needs to reside within the site\'s file directory, its path needs to start with @scheme://.', array('@scheme' => file_default_scheme())));
+    else {
+      // Check if chosen url scheme is allowed.
+      $scheme = file_uri_scheme($values['source']);
+      if (!$scheme || !in_array($scheme, $this->config['allowed_schemes'])) {
+        form_set_error('feeds][FeedsFileFetcher][source', t("The file needs to reside within the site's files directory, its path needs to start with scheme://. Available schemes: @schemes.", array('@schemes' => implode(', ', $this->config['allowed_schemes']))));
+      }
+      // Check wether the given path exists.
+      elseif (!file_exists($values['source'])) {
+        form_set_error('feeds][FeedsFileFetcher][source', t('The specified file or directory does not exist.'));
+      }
     }
   }
 
   /**
-   * Override parent::sourceSave().
+   * Overrides parent::sourceSave().
    */
   public function sourceSave(FeedsSource $source) {
     $source_config = $source->getConfigFor($this);
@@ -176,7 +195,7 @@ class FeedsFileFetcher extends FeedsFetcher {
   }
 
   /**
-   * Override parent::sourceDelete().
+   * Overrides parent::sourceDelete().
    */
   public function sourceDelete(FeedsSource $source) {
     $source_config = $source->getConfigFor($this);
@@ -186,17 +205,22 @@ class FeedsFileFetcher extends FeedsFetcher {
   }
 
   /**
-   * Override parent::configDefaults().
+   * Overrides parent::configDefaults().
    */
   public function configDefaults() {
+    $schemes = $this->getSchemes();
+    $scheme = in_array('private', $schemes) ? 'private' : 'public';
+
     return array(
       'allowed_extensions' => 'txt csv tsv xml opml',
       'direct' => FALSE,
+      'directory' => $scheme . '://feeds',
+      'allowed_schemes' => $schemes,
     );
   }
 
   /**
-   * Override parent::configForm().
+   * Overrides parent::configForm().
    */
   public function configForm(&$form_state) {
     $form = array();
@@ -214,16 +238,112 @@ class FeedsFileFetcher extends FeedsFetcher {
         are already on the server.'),
       '#default_value' => $this->config['direct'],
     );
+    $form['directory'] = array(
+      '#type' => 'textfield',
+      '#title' => t('Upload directory'),
+      '#description' => t('Directory where uploaded files get stored. Prefix the path with a scheme. Available schemes: @schemes.', array('@schemes' => implode(', ', $this->getSchemes()))),
+      '#default_value' => $this->config['directory'],
+      '#states' => array(
+        'visible' => array(':input[name="direct"]' => array('checked' => FALSE)),
+        'required' => array(':input[name="direct"]' => array('checked' => FALSE)),
+      ),
+    );
+    if ($options = $this->getSchemeOptions()) {
+      $form['allowed_schemes'] = array(
+        '#type' => 'checkboxes',
+        '#title' => t('Allowed schemes'),
+        '#default_value' => $this->config['allowed_schemes'],
+        '#options' => $options,
+        '#description' => t('Select the schemes you want to allow for direct upload.'),
+        '#states' => array(
+          'visible' => array(':input[name="direct"]' => array('checked' => TRUE)),
+        ),
+      );
+    }
+
     return $form;
   }
 
   /**
-   * Helper. Deletes a file.
+   * Overrides parent::configFormValidate().
+   *
+   * Ensure that the chosen directory is accessible.
+   */
+  public function configFormValidate(&$values) {
+
+    $values['directory'] = trim($values['directory']);
+    $values['allowed_schemes'] = array_filter($values['allowed_schemes']);
+
+    if (!$values['direct']) {
+      // Ensure that the upload directory field is not empty when not in
+      // direct-mode.
+      if (!$values['directory']) {
+        form_set_error('directory', t('Please specify an upload directory.'));
+        // Do not continue validating the directory if none was specified.
+        return;
+      }
+
+      // Validate the URI scheme of the upload directory.
+      $scheme = file_uri_scheme($values['directory']);
+      if (!$scheme || !in_array($scheme, $this->getSchemes())) {
+        form_set_error('directory', t('Please enter a valid scheme into the directory location.'));
+
+        // Return here so that attempts to create the directory below don't
+        // throw warnings.
+        return;
+      }
+
+      // Ensure that the upload directory exists.
+      if (!file_prepare_directory($values['directory'], FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS)) {
+        form_set_error('directory', t('The chosen directory does not exist and attempts to create it failed.'));
+      }
+    }
+  }
+
+  /**
+   * Deletes a file.
+   *
+   * @param int $fid
+   *   The file id.
+   * @param int $feed_nid
+   *   The feed node's id, or 0 if a standalone feed.
+   *
+   * @return bool|array
+   *   TRUE for success, FALSE in the event of an error, or an array if the file
+   *   is being used by any modules.
+   *
+   * @see file_delete()
    */
   protected function deleteFile($fid, $feed_nid) {
     if ($file = file_load($fid)) {
       file_usage_delete($file, 'feeds', get_class($this), $feed_nid);
-      file_delete($file);
+      return file_delete($file);
     }
+    return FALSE;
   }
+
+  /**
+   * Returns available schemes.
+   *
+   * @return array
+   *   The available schemes.
+   */
+  protected function getSchemes() {
+    return array_keys(file_get_stream_wrappers(STREAM_WRAPPERS_WRITE_VISIBLE));
+  }
+
+  /**
+   * Returns available scheme options for use in checkboxes or select list.
+   *
+   * @return array
+   *   The available scheme array keyed scheme => description
+   */
+  protected function getSchemeOptions() {
+    $options = array();
+    foreach (file_get_stream_wrappers(STREAM_WRAPPERS_WRITE_VISIBLE) as $scheme => $info) {
+      $options[$scheme] = check_plain($scheme . ': ' . $info['description']);
+    }
+    return $options;
+  }
+
 }

tests/feeds_fetcher_file.test

@@ -17,11 +17,10 @@ class FeedsFileFetcherTestCase extends FeedsWebTestCase {
     );
   }
 
-
   /**
    * Test scheduling on cron.
    */
-  public function test() {
+  public function testPublicFiles() {
     // Set up an importer.
     $this->createImporterConfiguration('Node import', 'node');
     // Set and configure plugins and mappings.
@@ -37,15 +36,19 @@ class FeedsFileFetcherTestCase extends FeedsWebTestCase {
     $this->addMappings('node', $mappings);
     // Straight up upload is covered in other tests, focus on direct mode
     // and file batching here.
-    $this->setSettings('node', 'FeedsFileFetcher', array('direct' => TRUE));
+    $settings = array(
+      'direct' => TRUE,
+      'directory' => 'public://feeds',
+    );
+    $this->setSettings('node', 'FeedsFileFetcher', $settings);
 
     // Verify that invalid paths are not accepted.
-    foreach (array('private://', '/tmp/') as $path) {
+    foreach (array('/tmp/') as $path) {
       $edit = array(
         'feeds[FeedsFileFetcher][source]' => $path,
       );
       $this->drupalPost('import/node', $edit, t('Import'));
-      $this->assertText("File needs to reside within the site's file directory, its path needs to start with public://.");
+      $this->assertText("The file needs to reside within the site's files directory, its path needs to start with scheme://. Available schemes:");
       $count = db_query("SELECT COUNT(*) FROM {feeds_source} WHERE feed_nid = 0")->fetchField();
       $this->assertEqual($count, 0);
     }
@@ -64,4 +67,48 @@ class FeedsFileFetcherTestCase extends FeedsWebTestCase {
     $this->drupalPost('import/node', $edit, t('Import'));
     $this->assertText('Created 18 nodes');
   }
+
+  /**
+   * Test uploading private files.
+   */
+  public function testPrivateFiles() {
+    // Set up an importer.
+    $this->createImporterConfiguration('Node import', 'node');
+    // Set and configure plugins and mappings.
+    $edit = array(
+      'content_type' => '',
+    );
+    $this->drupalPost('admin/structure/feeds/node/settings', $edit, 'Save');
+    $this->setPlugin('node', 'FeedsFileFetcher');
+    $this->setPlugin('node', 'FeedsCSVParser');
+    $mappings = array(
+      '0' => array(
+        'source' => 'title',
+        'target' => 'title',
+      ),
+    );
+    $this->addMappings('node', $mappings);
+    // Straight up upload is covered in other tests, focus on direct mode
+    // and file batching here.
+    $settings = array(
+      'direct' => TRUE,
+      'directory' => 'private://feeds',
+    );
+    $this->setSettings('node', 'FeedsFileFetcher', $settings);
+
+    // Verify batching through directories.
+    // Copy directory of files.
+    $dir = 'private://batchtest';
+    $this->copyDir($this->absolutePath() . '/tests/feeds/batch', $dir);
+
+    // Ingest directory of files. Set limit to 5 to force processor to batch,
+    // too.
+    variable_set('feeds_process_limit', 5);
+    $edit = array(
+      'feeds[FeedsFileFetcher][source]' => $dir,
+    );
+    $this->drupalPost('import/node', $edit, t('Import'));
+    $this->assertText('Created 18 nodes');
+  }
+
 }