Initial commit of feeds. Not in a usable shape yet.

README.txt

+$Id$
+
+FEEDS
+
+Work in progress. Contact alex_b http://drupal.org/user/53995 for details.
\ No newline at end of file

feeds.info

+;$Id$
+name = Feeds
+description = API for importing feeds of data.
+package = Feeds
+dependencies[] = ctools
+core = 6.x

feeds.install

+<?php
+// $Id$
+/**
+ * @file
+ */
+
+/**
+ * Implementation of hook_schema().
+ */
+function feeds_schema() {
+  $schema['feeds_configuration'] = array(
+    'description' => 'Configuration of feeds objects.',
+    'export' => array(
+      'key' => 'name',
+      'identifier' => 'feed_object',
+      'default hook' => 'feeds',  // Function hook name.
+      'api' => array(
+        'owner' => 'data',
+        'api' => 'feeds_default',  // Base name for api include files.
+        'minimum_version' => 1,
+        'current_version' => 1,
+      ),
+    ),
+    'fields' => array(
+      'id' => array(
+        'type' => 'varchar',
+        'length' => '255',
+        'not null' => TRUE,
+        'default' => '',
+        'description' => 'Id of the fields object.',
+      ),
+      'class' => array(
+        'type' => 'varchar',
+        'length' => '255',
+        'not null' => TRUE,
+        'default' => '',
+        'description' => 'Class name of the feeds object.',
+      ),
+      'config' => array(
+        'type' => 'text',
+        'not null' => FALSE,
+        'description' => 'Configuration of the feeds object.',
+        'serialize' => TRUE,
+      ),
+    ),
+    'indexes' => array(
+        'id' => array('id'),
+        'class' => array('class'),
+      ),
+  );
+  return $schema;
+}
+
+/**
+ * Implementation of hook_install().
+ */
+function feeds_install() {
+  // Create tables.
+  drupal_install_schema('feeds');
+}
+
+/**
+ * Implementation of hook_uninstall().
+ */
+function feeds_uninstall() {
+  // Remove tables.
+  drupal_uninstall_schema('feeds');
+}

feeds.module

+<?php
+// $Id$
+/**
+ * @file
+ * Feeds module.
+ */
+
+/**
+ * Implementation of hook_cron().
+ */
+function feeds_cron() {
+  feeds_include('queue');
+  feeds_queue_refresh();
+}
+
+/**
+ * Implementation of hook_perm().
+ */
+function feeds_perm() {
+  return array('use feeds', 'administer feeds');
+}
+
+/**
+ * Implementation of hook_feeds_plugin().
+ */
+function feeds_feeds_plugin() {
+  return array(
+    'FeedsHttpFetcher' => array(
+      'name' => t('HTTP fetcher'),
+      'file' => drupal_get_path('module', 'feeds') .'/plugins/FeedsHttpFetcher.inc',
+      'parent' => 'FeedsFetcher',
+    ),
+    'FeedsSyndicationParser' => array(
+      'name' => t('Common syndication parser'),
+      'file' => drupal_get_path('module', 'feeds') .'/plugins/FeedsSyndicationParser.inc',
+      'parent' => 'FeedsParser',
+    ),
+    'FeedsNodeProcessor' => array(
+      'name' => t('Node processor'),
+      'file' => drupal_get_path('module', 'feeds') .'/plugins/FeedsNodeProcessor.inc',
+      'parent' => 'FeedsProcessor',
+    ),
+  );
+}
+
+/**
+ * Get all available plugins.
+ * 
+ * @todo: more powerful tracing of parents.
+ */
+function feeds_get_plugins() {
+  feeds_include('feed');
+
+  $result = array();
+  $plugins = module_invoke_all('feeds_plugin');
+  foreach ($plugins as $plugin => $info) {
+    if ($info['parent'] == 'FeedsFetcher') { // @todo: walk tree.
+      $result['fetcher'][$plugin] = $info['name'] ? $info['name'] : $plugin;
+    }
+    elseif ($info['parent'] == 'FeedsParser') {
+      $result['parser'][$plugin] = $info['name'] ? $info['name'] : $plugin;
+    }
+    elseif ($info['parent'] == 'FeedsProcessor') {
+      $result['processor'][$plugin] = $info['name'] ? $info['name'] : $plugin;
+    }
+  }
+  return $result;
+}
+
+/**
+ * Load a plugin's file.
+ */
+function feeds_require_plugin($plugin) {
+  static $required;
+  $plugins = module_invoke_all('feeds_plugin');
+  if ($plugins[$plugin]['file'] && empty($required[$plugin])) {
+    require($plugins[$plugin]['file']);
+    $required[$plugin] = TRUE;
+  }
+}
+
+/**
+ * Menu loader. Loads a feeds configuration.
+ */
+function feeds_load($id) {
+  return feeds_get_feed($id);
+}
+
+/**
+ * Load all feeds.
+ */
+function feeds_load_all() {
+  ctools_include('export');
+  $configs = ctools_export_load_object('feeds_configuration', 'conditions', array('class' => 'Feed'));
+  foreach ($configs as $config) {
+    $feeds[$config->id] = feeds_get_feed($config->id);
+  }
+  return $feeds;
+}
+
+/**
+ * Singleton function.
+ */
+function feeds_get_feed($id) {
+  static $instantiated;
+  feeds_include('feed');
+  if (!isset($instantiated[$id])) {
+    $instantiated[$id] = new Feed($id);
+  }
+  return $instantiated[$id];
+}
+
+/**
+ * Create a configuration object.
+ */
+function feeds_create_configuration($id) {
+  feeds_include('configuration');
+}
+
+/**
+ * Get a configuration object for a given id.
+ */
+function feeds_get_configuration($id) {
+  feeds_include('configuration');
+}
+
+/**
+ * Delete a configuration object.
+ */
+function feeds_delete_configuration($id) {
+
+}
+
+/**
+ * Attach a configuration to a content type.
+ */
+function feeds_attach_configuration($id, $content_type) {
+  
+}
+
+/**
+ * Detach a configuration from a content type.
+ */
+function feeds_detach_configuration($content_type) {
+  
+}
+
+/**
+ * Create a feed.
+ */
+function feeds_create_feed($configuration) {
+  feeds_include('configuration');
+  feeds_include('feed');
+  $feed = new Feed($configuration);
+  return $feed;
+}
+
+/**
+ * Retrieve a feed object that is attached to a feed node.
+ * @todo.
+ */
+function feeds_get_feed_node($nid) {
+  $node = node_load($nid);
+  $configuration = feeds_get_configuration($node->type, TRUE);
+}
+
+/**
+ * Fetches, parses and processes a feed.
+ * 
+ * @return 
+ *   TRUE if the feed could be imported completely,
+ *   FALSE if a timeout occurred.
+ */
+function feeds_import($id, FeedsResource $source, $timeout = NULL) {
+  $importer = new FeedImporter($id, $source, $timeout);
+  return $importer->import();
+}
+
+/**
+ * Determines whether execution time is up.
+ */
+function feeds_timeout($timeout = NULL) {
+  $timeout_time = 0;
+  if ($timeout) {
+    $timeout_time = time() + $timeout;
+  }
+  if (time() > $timeout_time) {
+    return FALSE;
+  }
+  return TRUE;
+}
+
+/**
+ * Includes a feeds module include file.
+ */
+function feeds_include($file) {
+  static $included = array();
+  if (!isset($included[$file])) {
+    require_once './' . drupal_get_path('module', 'feeds') . "/includes/$file.inc";
+  }
+
+  $included[$file] = TRUE;
+}

feeds_ui/feeds_ui.admin.inc

+<?php
+// $Id$
+/**
+ * @file
+ */
+
+/**
+ * Build overview of available configurations.
+ */
+function feeds_ui_build_overview() {
+  $rows = array();
+  if ($feeds = feeds_load_all()) {
+    foreach ($feeds as $feed) {
+      $rows[] = array(
+        $feed->getId(),
+        l(t('Edit'), 'admin/build/feeds/edit/'. $feed->getId()) .' | '. 
+        l(t('Delete'), 'admin/build/feeds/delete/'. $feed->getId()),
+      );
+    }
+  }
+  $rows[] = array(
+    l(t('New configuration'), 'admin/build/feeds/create'),
+    '&nbsp;',
+  );
+  $header = array(
+    t('Configurations'),
+    t('Operations'),
+  );
+  return theme('table', $header, $rows);
+}
+
+/**
+ * Create a new configuration.
+ */
+function feeds_ui_build_create_form(&$form_state) {
+  $form = array();
+  if (!$form_state['storage']['id']) {
+    $form['id'] = array(
+      '#type' => 'textfield',
+      '#title' => t('Id'),
+      '#description' => t('A unique identifier for this preset.'),
+    );
+    $form['submit'] = array(
+      '#type' => 'submit',
+      '#value' => t('Next'),
+    );
+  }
+  else {
+    $form += _feeds_ui_config_form();
+    $form['submit'] = array(
+      '#type' => 'submit',
+      '#value' => t('Create'),
+    );
+  }
+  return $form;
+}
+
+/**
+ * Submit handler for feeds_build_create_form().
+ */
+function feeds_ui_build_create_form_submit($form, &$form_state) {
+  if ($form_state['values']['id']) {
+    $form_state['storage']['id'] = $form_state['values']['id'];
+  }
+  else {
+    // Build and save feed.
+    feeds_include('feed');
+    $feed = new Feed($form_state['storage']['id'], $form_state['values']);
+    $feed->save();
+    drupal_set_message(t('Created configuration'));
+
+    // Unset storage to enable redirect.
+    unset($form_state['storage']);
+    $form_state['redirect'] = 'admin/build/feeds';
+  }
+}
+
+/**
+ * Edit an existing configuration.
+ */
+function feeds_ui_build_edit_form(&$form_state, $feed) {
+  $form = _feeds_ui_config_form ($feed);
+  $form['submit'] = array(
+    '#type' => 'submit',
+    '#value' => t('Save'),
+  );
+  return $form;
+}
+
+/**
+ * Edit an existing configuration.
+ */
+function feeds_ui_build_mapping_form(&$form_state, $feed) {
+  $form = array();
+  $form['submit'] = array(
+    '#type' => 'submit',
+    '#value' => t('Save'),
+  );
+  return $form;
+}
+
+/**
+ * Edit an existing configuration.
+ */
+function feeds_ui_build_advanced_form(&$form_state, $feed) {
+  $form = array();
+  $form['submit'] = array(
+    '#type' => 'submit',
+    '#value' => t('Save'),
+  );
+  return $form;
+}
+
+/**
+ * Helper, build edit UI for a feed. Used for editing and creating.
+ */
+function _feeds_ui_config_form($feed = NULL) {
+  $form = array();
+  $plugins = feeds_get_plugins();
+  $form['plugins'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Plugins'),
+  );
+  $form['plugins']['fetcher'] = array(
+    '#type' => 'radios',
+    '#title' => t('Fetcher'),
+    '#options' => $plugins['fetcher'],
+    '#description' => t('Select a fetcher for this preset'),
+    '#default_value' => empty($feed) ? key($plugins['fetcher']) : get_class($feed->fetcher),
+  );
+  $form['plugins']['parser'] = array(
+    '#type' => 'radios',
+    '#title' => t('Parser'),
+    '#options' => $plugins['parser'],
+    '#description' => t('Select a parser for this preset'),
+    '#default_value' =>  empty($feed) ? key($plugins['parser']) : get_class($feed->parser),
+  );
+  if (empty($feed)) {
+    $default = array(key($plugins['processor']));
+  }
+  else {
+    foreach ($feed->processors as $processor) {
+      $default[get_class($processor)] = get_class($processor);
+    }
+  }
+  $form['plugins']['processors'] = array(
+    '#type' => 'checkboxes',
+    '#title' => t('Processors'),
+    '#options' => $plugins['processor'],
+    '#description' => t('Select processors for this preset'),
+    '#default_value' => $default,
+  );
+  $form['settings'] = array(
+    '#type' => 'fieldset',
+    '#title' => t('Basic settings'),
+  );
+  $form['settings']['content_type'] = array(
+    '#type' => 'select',
+    '#title' => t('Attach to content type'),
+    '#description' => t('When you attach a feeds configuration to a node, you can use nodes for creating feeds on your site.'),
+    '#options' => array('' => t('None')) + node_get_types('names'),
+    '#default_value' => '',
+  );
+  $form['settings']['update'] = array(
+    '#type' => 'radios',
+    '#title' => t('Update existing'),
+    '#options' => array(1 => t('Yes'), 0 => t('No')),
+    '#default_value' => 0, // @todo
+  );
+  $period = drupal_map_assoc(array(1, 900, 1800, 3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 3628800, 4838400, 7257600, 15724800, 31536000), 'format_interval');
+  $period[FEEDAPI_CRON_NEVER_REFRESH] = t('Never refresh');
+  $period[1] = t('As often as possible');
+  $form['settings']['refresh_period'] = array(
+    '#type' => 'select',
+    '#title' => t('Minimum refresh period'),
+    '#options' => $period,
+    '#default_value' => 0, // @todo
+  );
+  return $form;
+}
\ No newline at end of file

feeds_ui/feeds_ui.info

+;$Id$
+name = Feeds Admin UI
+description = Administrative UI for Feeds module.
+package = Feeds
+dependencies[] = feeds
+core = 6.x
\ No newline at end of file

feeds_ui/feeds_ui.module

+<?php
+// $Id$
+/**
+ * @file
+ */
+
+/**
+ * Implementation of hook_menu().
+ */
+function feeds_ui_menu() {
+  $items = array();
+  $items['admin/content/feeds'] = array(
+    'title' => 'Feeds',
+    'page callback' => 'feeds_ui_content_overview',
+    'access arguments' => array('use feeds'),
+    'file' => 'feeds_ui.admin.inc',
+  );
+  $items['admin/content/feeds/%feeds'] = array(
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('feeds_ui_content_feed', 3),
+    'access arguments' => array('use feeds'),
+    'file' => 'feeds_ui.admin.inc',
+  );
+  $items['admin/build/feeds'] = array(
+    'title' => 'Feeds',
+    'page callback' => 'feeds_ui_build_overview',
+    'access arguments' => array('administer feeds'),
+    'file' => 'feeds_ui.admin.inc',
+  );
+  $items['admin/build/feeds/create'] = array(
+    'title' => t('Create preset'),
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('feeds_ui_build_create_form'),
+    'access arguments' => array('administer feeds'),
+    'file' => 'feeds_ui.admin.inc',
+    'type' => MENU_CALLBACK,
+  );
+  $items['admin/build/feeds/edit/%feeds'] = array(
+    'title callback' => 'feed_ui_title',
+    'title arguments' => array(4),
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('feeds_ui_build_edit_form', 4),
+    'access arguments' => array('administer feeds'),
+    'file' => 'feeds_ui.admin.inc',
+    'type' => MENU_CALLBACK,
+  );
+  $items['admin/build/feeds/edit/%feeds/edit'] = array(
+    'title' => t('Configuration'),
+    'type' => MENU_DEFAULT_LOCAL_TASK,
+  );
+  $items['admin/build/feeds/edit/%feeds/map'] = array(
+    'title' => t('Mapping'),
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('feeds_ui_build_mapping_form', 4),
+    'access arguments' => array('administer feeds'),
+    'file' => 'feeds_ui.admin.inc',
+    'type' => MENU_LOCAL_TASK,
+  );
+  $items['admin/build/feeds/edit/%feeds/advanced'] = array(
+    'title' => t('Advanced'),
+    'page callback' => 'drupal_get_form',
+    'page arguments' => array('feeds_ui_build_advanced_form', 4),
+    'access arguments' => array('administer feeds'),
+    'file' => 'feeds_ui.admin.inc',
+    'type' => MENU_LOCAL_TASK,
+    'weight' => 10,
+  );
+  return $items;
+}
+
+/**
+ * Title callback.
+ */
+function feed_ui_title($feed) {
+  return t('Edit !feed', array('!feed' => $feed->getId()));
+}
\ No newline at end of file

includes/feed.inc

+<?php
+// $Id$
+/**
+ * @file
+ */
+
+/**
+ * Importer class.
+ * 
+ * After fetch() has been invoked, it can be cached and revoked in a subsequent 
+ * page callback until it is reset().
+ */
+class Feed extends FeedsConfigurable {
+
+  public $fetcher, $parser, $processors; 
+
+  /**
+   * Instantiate class variables.
+   */
+  public function __construct($id, $config = NULL) {
+    parent::__construct($id, $config);
+
+    feeds_require_plugin($this->config['fetcher']);
+    $this->fetcher = new $this->config['fetcher']($id);
+
+    feeds_require_plugin($this->config['parser']);
+    $this->parser = new $this->config['parser']($id);
+
+    foreach ($this->config['processors'] as $processor) {
+      feeds_require_plugin($processor);
+      $this->processors[$processor] = new $processor($id);
+    }
+  }
+
+  /**
+   * Import feed by using configured fetchers, parsers, processors.
+   */
+  public function import() {
+    $raw = $this->fetcher->fetch($this->source);
+    $parsed = $this->parser->parse($raw);
+    foreach ($this->processors as $processor) {
+      $processor->process($parsed);
+    }
+  }
+
+  /**
+   * Set active fetcher. Does not save fetcher to configuration.
+   *
+   * @param $fetcher
+   *   String that is the class name of the fetcher.
+   */
+  public function setFetcher($fetcher) {
+    unset($this->fetcher);
+    feeds_require_plugin($fetcher);
+    $this->config['fetcher'] = $fetcher;
+    $this->fetcher = new $fetcher($this->id);
+  }
+
+  /**
+   * Set active parser. Does not save parser to configuration.
+   *
+   * @param $parser
+   *   String that is the class name of the parser.
+   */
+  public function setParser($parser) {
+    unset($this->parser);
+    feeds_require_plugin($parser);
+    $this->config['parser'] = $parser;
+    $this->save();
+    $this->parser = new $parser($this->id);
+  }
+
+  /**
+   * Set active processors.
+   *
+   * @param $processors
+   *  Array of strings that are the processors of this feed.
+   */
+  public function setProcessors($processors) {
+    unset($this->processors);
+    $this->processors = array();
+    $this->config['processors'] = $processors;
+    $this->save();
+    foreach ($processors as $processor) {
+      feeds_require_plugin($processor);
+      $this->processors[] = new $processor($this->id);
+    }
+  }
+}
+
+/**
+ * Base class for configurable, peristent objects.
+ */
+class FeedsConfigurable {
+  protected $config;
+  protected $id;
+
+  /**
+   * Constructor.
+   *
+   * @param $id
+   *   String identifier of this object.
+   * @param $config
+   *   Configuration of this object. If not available, will attempt to load from database.
+   */
+  public function __construct($id, $config = NULL) {
+    $this->id = $id;
+    if (empty($config)) {
+      $this->load();
+    }
+    else {
+      $this->config = $config;
+    }
+  }
+
+  /**
+   * Save configuration.
+   */
+  public function save() {
+    $save = new stdClass();
+    $save->id = $this->id;
+    $save->class = get_class($this);
+    $save->config = $this->config;
+    db_query('DELETE FROM {feeds_configuration} WHERE id = "%s"', $save->id);
+    drupal_write_record('feeds_configuration', $save);
+  }
+
+  /**
+   * Load configuration and unpack.
+   */
+  public function load() {
+    ctools_include('export');
+    if ($config = ctools_export_load_object('feeds_configuration', 'conditions', array('id' => $this->id, 'class' => get_class($this)))) {
+      $config = array_shift($config);
+      $this->config = $config->config;
+      return TRUE;
+    }
+    return FALSE;
+  }
+
+  public function getConfig($fallback = TRUE) {
+    if ($fallback) {
+      return empty($this->config) ? $this->getDefaultConfig() : $this->config;
+    }
+    return $this->config;
+  }
+
+  public function setConfig($config) {
+    $this->config = $config;
+  }
+
+  public function getDefaultConfig() {
+    return array();
+  }
+
+  public function getId() {
+    return $this->id;
+  }
+
+  public function form() {
+    return array();
+  }
+}
+
+/**
+ * Abstract class, defines interface for fetchers.
+ * 
+ * Not using interfaces because we need a simple inheritence tree for determining the 
+ * plugin type. See hook_feeds_plugin().
+ */
+class FeedsFetcher extends FeedsConfigurable {
+  public function fetch($source) {
+    return NULL;
+  }
+}
+
+/**
+ * Abstract class, defines interface for parsers.
+ */
+class FeedsParser extends FeedsConfigurable {
+  public function parse($raw) {
+    return NULL;
+  }
+  public function getMappingSources() {
+    return NULL;
+  }
+}
+
+/**
+ * Abstract class, defines interface for processors.
+ */
+class FeedsProcessor extends FeedsConfigurable {
+  public function process($feed) {
+    return NULL;
+  }
+  public function getMappingTargets() {
+    return NULL;
+  }
+
+  /**
+   * Build the mapping form.
+   */
+  public function mappingForm() {
+    return array();
+  }
+
+  /**
+   * Return TRUE if processor requires that the feed configuration be attached to a feed node.
+   */
+  public function requiresFeedAsNode() {
+    return TRUE;
+  }
+}
\ No newline at end of file

includes/queue.inc

+<?php
+// $Id$
+/**
+ * Refreshes scheduled feeds.
+ */
+function feeds_queue_refresh() {
+  
+}
\ No newline at end of file

plugins/FeedsHttpFetcher.inc

+<?php
+// $Id$
+/**
+ * @file
+ * Home of the FeedsHttpFetcher.
+ *
+ */
+
+/**
+ * Fetches data via HTTP.
+ */
+class FeedsHttpFetcher extends FeedsFetcher {
+
+  /**
+   * Fetch a resource via http.
+   * 
+   * @todo: Use Curl, fall back to drupal_http_request.
+   * @todo: Add caching.
+   *
+   * @param $resource
+   *   A resource description of type FeedsResource.
+   * 
+   * @return 
+   *   A string from the requested location if successful, or FALSE if not.
+   */
+  public function fetch($resource) {
+    if ($resource->type == 'http') {
+      $result = drupal_http_request($resource->identifier);
+      $count = 0;
+      while ($result->redirect_url) {
+        $result = drupal_http_request($result->redirect_url);
+        $count++;
+        if ($count > 100) {
+          return FALSE;
+        }
+      }
+      if (!$result->error) {
+        return $result->response;
+      }
+      return FALSE;
+    }
+  }
+}
\ No newline at end of file

plugins/FeedsNodeProcessor.inc

+<?php
+// $Id$
+
+class FeedsNodeProcessor extends FeedsProcessor {
+
+  function process($feed) {
+    foreach ($feed->items as $item) {
+      if (feeds_unique('FeedsNodeProcessor', $item)) {
+        $node = feeds_map('FeedsNodeProcessor', $item);
+        node_save($node);
+      }
+    }
+  }
+
+  function getMappingTargets() {
+    $config = $this->getConfiguration();
+    $default = array('title', 'body', 'status');
+    $default += module_invoke('feedapi_mapper', 'list', 'FeedsNodeProcessor', $config->content_type);
+    return $default;
+  }
+
+  function map() {
+    
+  }
+
+  function unique() {
+    
+  }
+}
\ No newline at end of file

plugins/FeedsSyndicationParser.inc

+<?php
+// $Id$
+
+class FeedsSyndicationParser extends FeedsParser {
+
+  public static function getName() {
+    return t('Syndication Parser');
+  }
+
+  public static function getDescription() {
+    return t('Parses RSS and Atom feeds.');
+  }
+
+  /**
+   * Parses a raw string and returns a Feed object from it.
+   */
+  public function parse($raw) {
+    
+  }
+
+  /**
+   * Return mapping sources.
+   */
+  public function getMappingSources() {
+    return array('title', 'description', 'url', 'guid', 'tags');
+  }
+}
\ No newline at end of file