From b250842af20e81bac72332541c8fbcd60632a793 Mon Sep 17 00:00:00 2001 From: Alex Barth <alex_b@53995.no-reply.drupal.org> Date: Tue, 12 Jan 2010 16:28:02 +0000 Subject: [PATCH] #624088: mongolito404, David Goode, alex_b: Imagefield/filefield mapper, formalize feed elements. --- CHANGELOG.txt | 2 + feeds.plugins.inc | 2 +- mappers/filefield.inc | 77 +++++++++++++ plugins/FeedsParser.inc | 41 +++++++ plugins/FeedsSimplePieParser.inc | 36 +++++- plugins/FeedsSyndicationParser.inc | 61 +++++++++++ tests/feeds/flickr.xml | 169 +++++++++++++++++++++++++++++ tests/feeds_mapper_filefield.test | 105 ++++++++++++++++++ 8 files changed, 486 insertions(+), 7 deletions(-) create mode 100644 mappers/filefield.inc create mode 100644 tests/feeds/flickr.xml create mode 100644 tests/feeds_mapper_filefield.test diff --git a/CHANGELOG.txt b/CHANGELOG.txt index ca63a85d..37d88faa 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -3,6 +3,8 @@ Feeds 6.x 1.0 XXXXXXX, 20XX-XX-XX --------------------------------- +- #624088: mongolito404, David Goode, alex_b: Imagefield/filefield mapper, + formalize feed elements. - #584034: aaroncouch, mongolito404: Views integration. - Redirect to node or import form after manual import or delete. - #663830 Aron Novak, alex_b: When download of URL failed, node w/ empty title diff --git a/feeds.plugins.inc b/feeds.plugins.inc index e1e7e497..0bc096d2 100644 --- a/feeds.plugins.inc +++ b/feeds.plugins.inc @@ -113,7 +113,7 @@ function _feeds_feeds_plugins() { 'description' => 'Parse RSS and Atom feeds.', 'help' => 'Use <a href="http://simplepie.org">SimplePie</a> to parse XML feeds in RSS 1, RSS 2 and Atom format.', 'handler' => array( - 'parent' => 'FeedsParser', + 'parent' => 'FeedsSyndicationParser', 'class' => 'FeedsSimplePieParser', 'file' => 'FeedsSimplePieParser.inc', 'path' => $path, diff --git a/mappers/filefield.inc b/mappers/filefield.inc new file mode 100644 index 00000000..7e1d8e85 --- /dev/null +++ b/mappers/filefield.inc @@ -0,0 +1,77 @@ +<?php +// $Id$ + +/** + * @file + * On behalf implementation of Feeds mapping API for filefield.module (CCK). + * + * @todo Support <em>list</em> subfields (how to provide default value?) + * @todo Support <em>data</em> subfields (or its own subfieds?) + */ + +/** + * Implementation of hook_feeds_node_processor_targets_alter() + */ +function filefield_feeds_node_processor_targets_alter($targets, $content_type) { + $info = content_types($content_type); + $fields = array(); + if (isset($info['fields']) && count($info['fields'])) { + foreach ($info['fields'] as $field_name => $field) { + if ($field['type'] == 'filefield') { + $name = isset($field['widget']['label']) ? $field['widget']['label'] : $field_name; + $targets[$field_name] = array( + 'name' => $name, + 'callback' => $field['type'].'_feeds_set_target', + 'description' => t('The URL for the CCK !name field of the node.', array('!name' => $name)), + ); + } + } + } +} + +/** + * Implementation of hook_feeds_set_target(). + * + * @param $node + * The target node. + * @param $field_name + * The name of field on the target node to map to. + * @param $value + * The value to be mapped. Should contain a URL or an array of URLs; a + * FeedsEnclosure or an array of FeedsEnclosures. + * + * @todo: should we support $object->url again? + */ +function filefield_feeds_set_target($node, $field_name, $value) { + // Normalize $value, create an array of FeedsEnclosures of it. + $enclosures = array(); + if (!is_array($value)) { + $value = array($value); + } + foreach ($value as $k => $v) { + if ($v instanceof FeedsEnclosure) { + $enclosures[] = $v; + } + elseif (valid_url($v)) { + $enclosures[$k] = new FeedsEnclosure($v, 'application/octet-stream'); + } + } + + // Map enclosures. + $items = isset($node->$field_name) ? $node->$field_name : array(); + foreach ($enclosures as $enclosure) { + if ($file = $enclosure->getFile()) { + $field = content_fields($field_name, $node->type); + $target_dir = filefield_widget_file_path($field, user_load($node->uid)); + $info = field_file_save_file($enclosure->getFile(), array(), $target_dir); + if ($info) { + if ($field['list_field']) { + $info['list'] = $field['list_default']; + } + $items[] = $info; + $error = false; + } + } + } + $node->$field_name = $items; +} \ No newline at end of file diff --git a/plugins/FeedsParser.inc b/plugins/FeedsParser.inc index d4340a28..117d14ed 100644 --- a/plugins/FeedsParser.inc +++ b/plugins/FeedsParser.inc @@ -1,6 +1,47 @@ <?php // $Id$ +/** + * Defines an element of a parsed result. Such an element can be a simple type, + * a complex type (derived from FeedsElement) or an array of either. + * + * @see FeedsEnclosure + */ +class FeedsElement { + // The standard value of this element. This value can contain be a simple type, + // a FeedsElement or an array of either. + protected $value; + + /** + * Constructor. + */ + public function __construct($value) { + $this->value = $value; + } + + /** + * @return + * Standard value of this FeedsElement. + */ + public function getValue() { + return $this->value; + } + + /** + * @return + * A string representation of this element. + */ + public function __toString() { + if (is_array($this->value)) { + return 'Array'; + } + if (is_object($this->value)) { + return 'Object'; + } + return (string) $this->value; + } +} + /** * Abstract class, defines interface for parsers. */ diff --git a/plugins/FeedsSimplePieParser.inc b/plugins/FeedsSimplePieParser.inc index f5b97ef6..3cae8cf0 100644 --- a/plugins/FeedsSimplePieParser.inc +++ b/plugins/FeedsSimplePieParser.inc @@ -1,6 +1,34 @@ <?php // $Id$ +/** + * Adapter to present SimplePie_Enclosure as FeedsEnclosure object. + */ +class FeedsSimplePieEnclosure extends FeedsEnclosure { + protected $simplepie_enclosure; + + /** + * Constructor requires SimplePie enclosure object. + */ + function __construct(SimplePie_Enclosure $enclosure) { + $this->simplepie_enclosure = $enclosure; + } + + /** + * Override parent::getValue(). + */ + public function getValue() { + return $this->simplepie_enclosure->get_link(); + } + + /** + * Override parent::getMIMEType(). + */ + public function getMIMEType() { + return $this->simplepie_enclosure->get_real_type(); + } +} + /** * Class definition for Common Syndication Parser. * @@ -13,7 +41,7 @@ class FeedsSimplePieParser extends FeedsParser { */ public function parse(FeedsImportBatch $batch, FeedsSource $source) { feeds_include_library('simplepie.inc', 'simplepie'); - + // Initialize SimplePie. $parser = new SimplePie(); $parser->set_raw_data($batch->getRaw()); @@ -54,11 +82,7 @@ class FeedsSimplePieParser extends FeedsParser { $enclosures = $simplepie_item->get_enclosures(); if (is_array($enclosures)) { foreach ($enclosures as $enclosure) { - $mime = $enclosure->get_real_type(); - if ($mime != '') { - list($type, $subtype) = split('/', $mime); - $item['enclosures'][$type][$subtype][] = $enclosure; - } + $item['enclosures'][] = new FeedsSimplePieEnclosure($enclosure); } } // Location diff --git a/plugins/FeedsSyndicationParser.inc b/plugins/FeedsSyndicationParser.inc index 1561bce2..08473e83 100644 --- a/plugins/FeedsSyndicationParser.inc +++ b/plugins/FeedsSyndicationParser.inc @@ -1,6 +1,67 @@ <?php // $Id$ +/** + * Enclosure element, can be part of the result array. + */ +class FeedsEnclosure extends FeedsElement { + protected $mime_type; + protected $file; + + /** + * Constructor, requires MIME type. + */ + public function __construct($value, $mime_type) { + parent::__construct($value); + $this->mime_type = $mime_type; + } + + /** + * @return + * MIME type of return value of getValue(). + */ + public function getMIMEType() { + return $this->mime_type; + } + + /** + * @return + * The content of the referenced resource. + */ + public function getContent() { + feeds_include_library('http_request.inc', 'http_request'); + $result = http_request_get($this->getValue()); + if ($result->code != 200) { + throw new Exception(t('Download of @url failed with code !code.', array('@url' => $this->getValue(), '!code' => $result->code))); + } + return $result->data; + } + + /** + * @return + * The file path to the downloaded resource referenced by the enclosure. + * Downloads resource if not downloaded yet. + * + * @todo Get file extension from mime_type. + * @todo This is not concurrency safe. + */ + public function getFile() { + if(!isset($this->file)) { + $dest = file_destination(file_directory_temp() .'/'. get_class($this) .'-'. basename($this->getValue()), FILE_EXISTS_RENAME); + if (ini_get('allow_url_fopen')) { + $this->file = copy($this->getValue(), $dest) ? $dest : 0; + } + else { + $this->file = file_save_data($this->getContent(), $dest); + } + if ($this->file === 0) { + throw new Exception(t('Cannot write content to %dest', array('%dest' => $dest))); + } + } + return $this->file; + } +} + /** * Class definition for Common Syndication Parser. * diff --git a/tests/feeds/flickr.xml b/tests/feeds/flickr.xml new file mode 100644 index 00000000..33ef7f2e --- /dev/null +++ b/tests/feeds/flickr.xml @@ -0,0 +1,169 @@ +<?xml version="1.0" encoding="utf-8" standalone="yes"?> +<feed xmlns="http://www.w3.org/2005/Atom" + xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:flickr="urn:flickr:" xmlns:media="http://search.yahoo.com/mrss/"> + + <title>Content from My picks</title> + <link rel="self" href="http://api.flickr.com/services/feeds/photoset.gne?set=72157603970496952&nsid=28242329@N00&lang=en-us" /> + <link rel="alternate" type="text/html" href="http://www.flickr.com/photos/a-barth/sets/72157603970496952"/> + <id>tag:flickr.com,2005:http://www.flickr.com/photos/28242329@N00/sets/72157603970496952</id> + <icon>http://farm1.static.flickr.com/42/86410049_bd6dcdd5f9_s.jpg</icon> + <subtitle>Some of my shots I like best in random order.</subtitle> + <updated>2009-07-09T21:48:04Z</updated> + <generator uri="http://www.flickr.com/">Flickr</generator> + + <entry> + <title>Tubing is awesome</title> + <link rel="alternate" type="text/html" href="http://www.flickr.com/photos/a-barth/3596408735/in/set-72157603970496952/"/> + <id>tag:flickr.com,2005:/photo/3596408735/in/set-72157603970496952</id> + <published>2009-07-09T21:48:04Z</published> + <updated>2009-07-09T21:48:04Z</updated> + <dc:date.Taken>2009-05-01T00:00:00-08:00</dc:date.Taken> + <content type="html"><p><a href="http://www.flickr.com/people/a-barth/">Alex Barth</a> posted a photo:</p> + +<p><a href="http://www.flickr.com/photos/a-barth/3596408735/" title="Tubing is awesome"><img src="http://farm4.static.flickr.com/3599/3596408735_ce2f0c4824_m.jpg" width="240" height="161" alt="Tubing is awesome" /></a></p> + + +<p>Virginia, 2009</p></content> + <author> + <name>Alex Barth</name> + <uri>http://www.flickr.com/people/a-barth/</uri> + </author> + <link rel="license" type="text/html" href="http://creativecommons.org/licenses/by-nc/2.0/deed.en" /> + <link rel="enclosure" type="image/jpeg" href="http://farm4.static.flickr.com/3599/3596408735_ce2f0c4824_b.jpg" /> + + <category term="color" scheme="http://www.flickr.com/photos/tags/" /> + <category term="film" scheme="http://www.flickr.com/photos/tags/" /> + <category term="virginia" scheme="http://www.flickr.com/photos/tags/" /> + <category term="awesome" scheme="http://www.flickr.com/photos/tags/" /> + <category term="ishootfilm" scheme="http://www.flickr.com/photos/tags/" /> + <category term="va" scheme="http://www.flickr.com/photos/tags/" /> + <category term="badge" scheme="http://www.flickr.com/photos/tags/" /> + <category term="tubing" scheme="http://www.flickr.com/photos/tags/" /> + <category term="fuji160c" scheme="http://www.flickr.com/photos/tags/" /> + <category term="anfamiliebarth" scheme="http://www.flickr.com/photos/tags/" /> + <category term="canon24l" scheme="http://www.flickr.com/photos/tags/" /> + </entry> + <entry> + <title>Jeff vs Tom</title> + <link rel="alternate" type="text/html" href="http://www.flickr.com/photos/a-barth/2640019371/in/set-72157603970496952/"/> + <id>tag:flickr.com,2005:/photo/2640019371/in/set-72157603970496952</id> + <published>2009-07-09T21:45:50Z</published> + <updated>2009-07-09T21:45:50Z</updated> + <dc:date.Taken>2008-06-01T00:00:00-08:00</dc:date.Taken> + <content type="html"><p><a href="http://www.flickr.com/people/a-barth/">Alex Barth</a> posted a photo:</p> + +<p><a href="http://www.flickr.com/photos/a-barth/2640019371/" title="Jeff vs Tom"><img src="http://farm4.static.flickr.com/3261/2640019371_495c3f51a2_m.jpg" width="240" height="159" alt="Jeff vs Tom" /></a></p> + + +</content> + <author> + <name>Alex Barth</name> + <uri>http://www.flickr.com/people/a-barth/</uri> + </author> + <link rel="license" type="text/html" href="http://creativecommons.org/licenses/by-nc/2.0/deed.en" /> + <link rel="enclosure" type="image/jpeg" href="http://farm4.static.flickr.com/3261/2640019371_495c3f51a2_b.jpg" /> + + <category term="b" scheme="http://www.flickr.com/photos/tags/" /> + <category term="blackandwhite" scheme="http://www.flickr.com/photos/tags/" /> + <category term="bw" scheme="http://www.flickr.com/photos/tags/" /> + <category term="jeff" scheme="http://www.flickr.com/photos/tags/" /> + <category term="tom" scheme="http://www.flickr.com/photos/tags/" /> + <category term="washingtondc" scheme="http://www.flickr.com/photos/tags/" /> + <category term="blackwhite" scheme="http://www.flickr.com/photos/tags/" /> + <category term="dc" scheme="http://www.flickr.com/photos/tags/" /> + <category term="nikon" scheme="http://www.flickr.com/photos/tags/" /> + <category term="wideangle" scheme="http://www.flickr.com/photos/tags/" /> + <category term="ilfordhp5" scheme="http://www.flickr.com/photos/tags/" /> + <category term="foosball" scheme="http://www.flickr.com/photos/tags/" /> + <category term="20mm" scheme="http://www.flickr.com/photos/tags/" /> + <category term="nikonfe2" scheme="http://www.flickr.com/photos/tags/" /> + <category term="800asa" scheme="http://www.flickr.com/photos/tags/" /> + <category term="foosballtable" scheme="http://www.flickr.com/photos/tags/" /> + <category term="wuzler" scheme="http://www.flickr.com/photos/tags/" /> + <category term="wuzln" scheme="http://www.flickr.com/photos/tags/" /> + <category term="tischfusball" scheme="http://www.flickr.com/photos/tags/" /> + <category term="jeffmiccolis" scheme="http://www.flickr.com/photos/tags/" /> + <category term="ilfordhp5800asa" scheme="http://www.flickr.com/photos/tags/" /> + <category term="widean" scheme="http://www.flickr.com/photos/tags/" /> + </entry> + <entry> + <title>Attersee 1</title> + <link rel="alternate" type="text/html" href="http://www.flickr.com/photos/a-barth/3686290986/in/set-72157603970496952/"/> + <id>tag:flickr.com,2005:/photo/3686290986/in/set-72157603970496952</id> + <published>2009-07-09T21:42:01Z</published> + <updated>2009-07-09T21:42:01Z</updated> + <dc:date.Taken>2009-06-01T00:00:00-08:00</dc:date.Taken> + <content type="html"><p><a href="http://www.flickr.com/people/a-barth/">Alex Barth</a> posted a photo:</p> + +<p><a href="http://www.flickr.com/photos/a-barth/3686290986/" title="Attersee 1"><img src="http://farm4.static.flickr.com/3606/3686290986_334c427e8c_m.jpg" width="240" height="238" alt="Attersee 1" /></a></p> + + +<p>Upper Austria, 2009</p></content> + <author> + <name>Alex Barth</name> + <uri>http://www.flickr.com/people/a-barth/</uri> + </author> + <link rel="license" type="text/html" href="http://creativecommons.org/licenses/by-nc/2.0/deed.en" /> + <link rel="enclosure" type="image/jpeg" href="http://farm4.static.flickr.com/3606/3686290986_334c427e8c_b.jpg" /> + + <category term="lake" scheme="http://www.flickr.com/photos/tags/" /> + <category term="green" scheme="http://www.flickr.com/photos/tags/" /> + <category term="water" scheme="http://www.flickr.com/photos/tags/" /> + <category term="austria" scheme="http://www.flickr.com/photos/tags/" /> + <category term="holga" scheme="http://www.flickr.com/photos/tags/" /> + <category term="toycamera" scheme="http://www.flickr.com/photos/tags/" /> + <category term="ishootfilm" scheme="http://www.flickr.com/photos/tags/" /> + <category term="fujireala" scheme="http://www.flickr.com/photos/tags/" /> + <category term="badge" scheme="http://www.flickr.com/photos/tags/" /> + <category term="100asa" scheme="http://www.flickr.com/photos/tags/" /> + <category term="attersee" scheme="http://www.flickr.com/photos/tags/" /> + <category term="plasticlens" scheme="http://www.flickr.com/photos/tags/" /> + <category term="colornegative" scheme="http://www.flickr.com/photos/tags/" /> + </entry> + <entry> + <title>H Street North East</title> + <link rel="alternate" type="text/html" href="http://www.flickr.com/photos/a-barth/2640845934/in/set-72157603970496952/"/> + <id>tag:flickr.com,2005:/photo/2640845934/in/set-72157603970496952</id> + <published>2008-09-23T13:26:13Z</published> + <updated>2008-09-23T13:26:13Z</updated> + <dc:date.Taken>2008-06-01T00:00:00-08:00</dc:date.Taken> + <content type="html"><p><a href="http://www.flickr.com/people/a-barth/">Alex Barth</a> posted a photo:</p> + +<p><a href="http://www.flickr.com/photos/a-barth/2640845934/" title="H Street North East"><img src="http://farm4.static.flickr.com/3083/2640845934_85c11e5a18_m.jpg" width="240" height="159" alt="H Street North East" /></a></p> + + +<p>Washington DC 2008<br /> +<a href="http://dcist.com/2008/07/07/photo_of_the_day_july_7_2008.php">Photo of the Day July 7 on DCist</a></p></content> + <author> + <name>Alex Barth</name> + <uri>http://www.flickr.com/people/a-barth/</uri> + </author> + <link rel="license" type="text/html" href="http://creativecommons.org/licenses/by-nc/2.0/deed.en" /> + <link rel="enclosure" type="image/jpeg" href="http://farm4.static.flickr.com/3083/2640845934_85c11e5a18_b.jpg" /> + + <category term="nightphotography" scheme="http://www.flickr.com/photos/tags/" /> + <category term="b" scheme="http://www.flickr.com/photos/tags/" /> + <category term="blackandwhite" scheme="http://www.flickr.com/photos/tags/" /> + <category term="bw" scheme="http://www.flickr.com/photos/tags/" /> + <category term="night" scheme="http://www.flickr.com/photos/tags/" /> + <category term="washingtondc" scheme="http://www.flickr.com/photos/tags/" /> + <category term="blackwhite" scheme="http://www.flickr.com/photos/tags/" /> + <category term="dc" scheme="http://www.flickr.com/photos/tags/" /> + <category term="nikon" scheme="http://www.flickr.com/photos/tags/" /> + <category term="dof" scheme="http://www.flickr.com/photos/tags/" /> + <category term="wideangle" scheme="http://www.flickr.com/photos/tags/" /> + <category term="explore" scheme="http://www.flickr.com/photos/tags/" /> + <category term="ilfordhp5" scheme="http://www.flickr.com/photos/tags/" /> + <category term="badge" scheme="http://www.flickr.com/photos/tags/" /> + <category term="dcist" scheme="http://www.flickr.com/photos/tags/" /> + <category term="20mm" scheme="http://www.flickr.com/photos/tags/" /> + <category term="hstreet" scheme="http://www.flickr.com/photos/tags/" /> + <category term="nikonfe2" scheme="http://www.flickr.com/photos/tags/" /> + <category term="800asa" scheme="http://www.flickr.com/photos/tags/" /> + <category term="hstreetne" scheme="http://www.flickr.com/photos/tags/" /> + <category term="anfamiliebarth" scheme="http://www.flickr.com/photos/tags/" /> + <category term="ilfordhp5800asa" scheme="http://www.flickr.com/photos/tags/" /> + <category term="hstreetbynight" scheme="http://www.flickr.com/photos/tags/" /> + <category term="forlaia" scheme="http://www.flickr.com/photos/tags/" /> + </entry> +</feed> \ No newline at end of file diff --git a/tests/feeds_mapper_filefield.test b/tests/feeds_mapper_filefield.test new file mode 100644 index 00000000..1f705745 --- /dev/null +++ b/tests/feeds_mapper_filefield.test @@ -0,0 +1,105 @@ +<?php +// $Id$ + +require_once(drupal_get_path('module', 'feeds') . '/tests/feeds_mapper_test.inc'); + +/** + * Class for testing Feeds FileField mapper. + * + * @todo Add a test for enclosures returned as array + * @todo Add a test for enclosures returned as string + * @todo Add a test for enclosures using local file + */ +class FeedsMapperFileFieldTestCase extends FeedsMapperTestCase { + + public static function getInfo() { + return array( + 'name' => t('Mapper: FileField'), + 'description' => t('Test Feeds Mapper support for FileField CCK fields'), + 'group' => t('Feeds'), + ); + } + + /** + * Set up the + */ + public function setUp() { + // Call parent setup with the required module + parent::setUp( + 'devel', 'feeds', 'feeds_ui', 'ctools', 'content', + 'filefield' + ); + + // Create user and login + $this->drupalLogin($this->drupalCreateUser( + array( + 'administer content types', + 'administer feeds', + 'administer nodes', + 'administer site configuration', + 'access devel information' + ) + )); + } + + /** + * Basic test loading a single entry CSV file. + */ + public function test() { + $static_title = $this->randomName(); + //Create content type + $typename = $this->createContentType(NULL, array( + 'files' => array( + 'type' => 'filefield', + 'settings' => array( + 'multiple' => '1', + 'file_extensions' => 'jpg' + ), + ), + )); + + //Create importer configuration + $this->createFeedConfiguration(); //Create a default importer configuration + $this->setPlugin('syndication', 'FeedsSimplePieParser'); + $this->setSettings('syndication', 'FeedsNodeProcessor', array('content_type' => $typename)); //Processor settings + $this->addMappings('syndication', array( + array( + 'source' => 'title', + 'target' => 'title' + ), + array( + 'source' => 'timestamp', + 'target' => 'created' + ), + array( + 'source' => 'description', + 'target' => 'body' + ), + array( + 'source' => 'enclosures', + 'target' => 'field_files' + ), + )); + + $nid = $this->createFeedNode('syndication', $GLOBALS['base_url'] .'/'. drupal_get_path('module', 'feeds') . '/tests/feeds/flickr.xml'); + $this->assertText('Created 4 '.$typename.' nodes.'); + + $filename = array('3596408735_ce2f0c4824_b', '2640019371_495c3f51a2_b', '3686290986_334c427e8c_b', '2640845934_85c11e5a18_b'); + for($i = 0; $i < 4; $i++) { + $this->drupalGet('node/'.($i+2).'/edit'); + $this->assertText($filename[$i]); + } + } + + /** + * Handle file field widgets. + */ + public function selectFieldWidget($fied_name, $field_type) { + if ($field_type == 'filefield') { + return 'filefield_widget'; + } + else { + return parent::selectFieldWidget($fied_name, $field_type); + } + } +} -- GitLab