diff -u AmazonS3StreamWrapper.inc AmazonS3StreamWrapper.inc --- AmazonS3StreamWrapper.inc 2013-09-12 21:14:44.000000000 -0700 +++ AmazonS3StreamWrapper.inc 2013-09-11 16:27:12.000000000 -0700 @@ -2,124 +2,201 @@ /** * @file - * Drupal stream wrapper implementation for Amazon S3 + * Drupal stream wrapper implementation for Amazon S3. * * Implements DrupalStreamWrapperInterface to provide an Amazon S3 wrapper with - * the s3:// prefix + * the s3:// prefix. */ class AmazonS3StreamWrapper implements DrupalStreamWrapperInterface { - + /** - * @var String Instance URI referenced as "s3://bucket/key" + * @var String Instance URI referenced as "s3://key" */ protected $uri; - + /** - * @var AmazonS3 S3 connection object + * @var AmazonS3 S3 client object, shared across all instances of + * AmazonS3StreamWrapper. + */ + protected static $s3_client = null; + + /** + * @var AmazonS3 An alias for self::$s3_client. */ protected $s3 = null; - + /** * @var string S3 bucket name */ protected $bucket; - + /** - * @var string Domain we use to access files over http + * @var string Domain we use to access files over http. */ protected $domain = NULL; - + /** - * @var int Current read/write position + * @var int Current read/write position. */ protected $position = 0; - + /** - * @var int Total size of the object as returned by S3 (Content-length) + * @var int Total size of the object as returned by S3 (Content-Length) */ protected $object_size = 0; - + /** - * @var string Object read/write buffer, typically a file + * @var string Object read/write buffer, typically a file. */ - protected $buffer = null; - + protected $buffer = NULL; + /** - * @var boolean Whether $buffer is active as a write buffer + * @var boolean Whether $buffer is active as a write buffer. */ - protected $buffer_write = false; - + protected $buffer_write = FALSE; + /** - * @var int Buffer length + * @var int Buffer length. */ protected $buffer_length = 0; - + /** - * @var array directory listing + * Records the number of calls to stream_write() between each call to + * stream_flush(), for testing purposes. */ - protected $dir = array(); - + protected $sw_call_count = 0; + /** - * @var array Default map for determining file mime types + * @var array directory listing. */ - protected static $mapping = null; - - /** - * @var boolean Whether local file metadata caching is on - */ - protected $caching = FALSE; - + protected $dir = NULL; + /** * @var array Map for files that should be delivered with a torrent URL. */ protected $torrents = array(); - + /** - * @var array Map for files that should have their Content-disposition header + * @var array Map for files that should have their Content-Disposition header * set to force "save as". */ protected $saveas = array(); - + /** - * @var array Map for files that should have a URL will be created that times + * @var array Map for files that should be created with a URL that times * out in a designated number of seconds. */ protected $presigned_urls = array(); - + /** - * Object constructor + * @var boolean The constructor sets this to TRUE once it's finished. + * See the comment for _assert_constructor_called() for why this exists. + */ + protected $constructed = FALSE; + + /** + * @var array Default map for determining file mime types. + */ + protected static $mime_type_mapping = NULL; + + /** + * Static function to determine a file's media type. + * + * Uses Drupal's mimetype mapping, unless a different mapping is specified. * - * Sets the bucket name + * @return + * Returns a string representing the file's MIME type, or + * 'application/octet-stream' if no type cna be determined. + */ + public static function getMimeType($uri, $mapping=NULL) { + self::_test_log("getMimeType($uri, $mapping) called."); + // Load the default mime type map. + if (!isset(self::$mime_type_mapping)) { + include_once DRUPAL_ROOT . '/includes/file.mimetypes.inc'; + self::$mime_type_mapping = file_mimetype_mapping(); + } + + // If a mapping wasn't specified, use the default map. + if ($mapping == NULL) { + $mapping = self::$mime_type_mapping; + } + + $extension = ''; + $file_parts = explode('.', basename($uri)); + + // Remove the first part: a full filename should not match an extension. + array_shift($file_parts); + + // Iterate over the file parts, trying to find a match. + // For my.awesome.image.jpeg, we try: + // - jpeg + // - image.jpeg + // - awesome.image.jpeg + while ($additional_part = array_pop($file_parts)) { + $extension = strtolower($additional_part . ($extension ? '.' . $extension : '')); + if (isset($mapping['extensions'][$extension])) { + return $mapping['mimetypes'][$mapping['extensions'][$extension]]; + } + } + + // No mime types matches, so return the default. + return 'application/octet-stream'; + } + + /** + * Object constructor. + * + * Creates the AmazonS3 client object and activates the options from the + * admin page. */ public function __construct() { $this->bucket = $bucket = variable_get('amazons3_bucket', ''); - - // CNAME support for customising S3 URLs - if (variable_get('amazons3_cname', 0)) { + + // If it hasn't already been done in this request, load the AWSSDK library + // and create the AmazonS3 client. + if (!isset(self::$s3_client)) { + if (!libraries_load('awssdk')) { + throw new Exception(t('Unable to load the AWS SDK. Please check you have installed the library correctly and configured your S3 credentials.')); + } + else if (empty($this->bucket)) { + throw new Exception(t('AmazonS3 bucket name not configured. Please visit the !config_page.', + array('!config_page' => l(t('configuration page'), '/admin/config/media/amazons3')))); + } + else { + try { + self::$s3_client = new AmazonS3(); + // Using SSL slows down uploads significantly, but it's unsafe to disable it. + // I'm still looking for a better solution. -- coredumperror 2013/07/12 + //self::$s3_client->disable_ssl(); + $this->_test_log('Created AmazonS3 client.'); + } + catch (Exception $e) { + throw new Expcetion(t('There was a problem connecting to S3: @msg', array('@msg' => $e->getMessage()))); + } + } + } + $this->s3 = self::$s3_client; + + // CNAME support for customizing S3 URLs. + if (variable_get('amazons3_cname', FALSE)) { $domain = variable_get('amazons3_domain', ''); - if(strlen($domain) > 0) { - $this->domain = 'http://' . $domain; + if ($domain) { + $this->domain = "http://$domain"; } else { - $this->domain = 'http://' . $this->bucket; + $this->domain = "http://{$this->bucket}"; } } else { - $this->domain = 'http://' . $this->bucket . '.s3.amazonaws.com'; + $this->domain = "http://{$this->bucket}.s3.amazonaws.com"; } - - // Check whether local file caching is turned on - if (variable_get('amazons3_cache', TRUE)) { - $this->caching = TRUE; - } - + // Torrent list $torrents = explode("\n", variable_get('amazons3_torrents', '')); $torrents = array_map('trim', $torrents); $torrents = array_filter($torrents, 'strlen'); $this->torrents = $torrents; - - + // Presigned url-list $presigned_urls = explode("\n", variable_get('amazons3_presigned_urls', '')); $presigned_urls = array_map('trim', $presigned_urls); @@ -135,79 +212,85 @@ $this->presigned_urls[$presigned_url] = 60; } } - + // Force "save as" list $saveas = explode("\n", variable_get('amazons3_saveas', '')); $saveas = array_map('trim', $saveas ); - $saveas = array_filter($saveas , 'strlen'); - $this->saveas = $saveas; - + $saveas = array_filter($saveas , 'strlen'); + $this->saveas = $saveas; + + $this->constructed = TRUE; + $this->_test_log('AmazonS3StreamWrapper contructed.'); } - - + /** * Sets the stream resource URI. * - * URIs are formatted as "s3://bucket/key" + * URIs are formatted as "s3://key" * * @return * Returns the current URI of the instance. */ public function setUri($uri) { + $this->_test_log("setUri($uri) called."); $this->uri = $uri; } - + /** * Returns the stream resource URI. * - * URIs are formatted as "s3://bucket/key" + * URIs are formatted as "s3://key" * * @return * Returns the current URI of the instance. */ public function getUri() { + $this->_test_log('getUri() called.'); return $this->uri; } - + /** * Returns a web accessible URL for the resource. * - * In the format http://mybucket.amazons3.com/myfile.jpg + * In the format http://mybucket.s3.amazonaws.com/key, or something different + * as configured in the admin settings. * * @return * Returns a string containing a web accessible URL for the resource. */ public function getExternalUrl() { - - // Image styles support - // Delivers the first request to an image from the private file system - // otherwise it returns an external URL to an image that has not been - // created yet - $path = explode('/', $this->getLocalPath()); - if ($path[0] == 'styles') { - if (!$this->_amazons3_get_object($this->uri, $this->caching)) { - array_shift($path); - return url('system/files/styles/' . implode('/', $path), array('absolute' => true)); + $this->_test_log('getExternalUri() called.'); + $s3_filename = $this->_uri_to_s3_filename($this->uri); + + // Image styles support: + // If an image derivative URL (e.g. styles/thumbnail/blah.jpg) is requested + // and the file doesn't exist, return a system URL instead. Drupal will + // create the derivative when that URL gets requested. + $path_parts = explode('/', $s3_filename); + if ($path_parts[0] == 'styles') { + if (!$this->_amazons3_get_object($this->uri)) { + array_shift($path_parts); + return url('system/files/styles/' . implode('/', $path_parts), array('absolute' => true)); } } - - $local_path = $this->getLocalPath(); - + $info = array( 'download_type' => 'http', 'presigned_url' => FALSE, 'presigned_url_timeout' => 60, 'response' => array(), ); - + // Allow other modules to change the download link type. - $info = array_merge($info, module_invoke_all('amazons3_url_info', $local_path, $info)); - + $info = array_merge($info, module_invoke_all('amazons3_url_info', $s3_filename, $info)); + + /////////////// // UI overrides + /////////////// // Torrent URLs if ($info['download_type'] != 'torrent') { foreach ($this->torrents as $path) { - if (preg_match('#' . strtr($path, '#', '\#') . '#', $local_path)) { + if (preg_match('#' . strtr($path, '#', '\#') . '#', $s3_filename)) { $info['download_type'] = 'torrent'; break; } @@ -216,7 +299,7 @@ // Presigned URLs if (!$info['presigned_url']) { foreach ($this->presigned_urls as $path => $timeout) { - if (preg_match('#' . strtr($path, '#', '\#') . '#', $local_path)) { + if (preg_match('#' . strtr($path, '#', '\#') . '#', $s3_filename)) { $info['presigned_url'] = TRUE; $info['presigned_url_timeout'] = $timeout; break; @@ -226,67 +309,29 @@ // Save as if ($info['download_type'] != 'torrent') { foreach ($this->saveas as $path) { - if (preg_match('#' . strtr($path, '#', '\#') . '#', $local_path)) { - $info['response']['content-disposition'] = 'attachment; filename=' . basename($local_path); + if (preg_match('#' . strtr($path, '#', '\#') . '#', $s3_filename)) { + $filename = basename($s3_filename); + $info['response']['content-disposition'] = "attachment; filename=\"$filename\""; break; } } } - - $timeout = ($info['presigned_url']) ? time() + $info['presigned_url_timeout'] : 0; + + $timeout = $info['presigned_url'] ? time() + $info['presigned_url_timeout'] : 0; $torrent = ($info['download_type'] == 'torrent') ? TRUE : FALSE; - $response = ($info['presigned_url']) ? $info['response'] : array(); + $response = $info['presigned_url'] ? $info['response'] : array(); if ($info['presigned_url'] || $info['download_type'] != 'http' || !empty($info['response'])) { - $url = $this->getS3()->get_object_url($this->bucket, $local_path, $timeout, array('torrent' => $torrent, 'response' => $response)); + $url = $this->s3->get_object_url($this->bucket, $s3_filename, $timeout, array('torrent' => $torrent, 'response' => $response)); return $url; } - - $url = $this->domain . '/' . $local_path; + + $url = "{$this->domain}/$s3_filename"; return $url; } - - /** - * Determine a file's media type - * - * Uses Drupal's mimetype mappings. Returns 'application/octet-stream' if - * no match is found. - * - * @return - * Returns a string representing the file's MIME type - */ - public static function getMimeType($uri, $mapping = NULL) { - - // Load the default file map - if (!isset(self::$mapping)) { - include_once DRUPAL_ROOT . '/includes/file.mimetypes.inc'; - self::$mapping = file_mimetype_mapping(); - } - - $extension = ''; - $file_parts = explode('.', basename($uri)); - - // Remove the first part: a full filename should not match an extension. - array_shift($file_parts); - - // Iterate over the file parts, trying to find a match. - // For my.awesome.image.jpeg, we try: - // - jpeg - // - image.jpeg, and - // - awesome.image.jpeg - while ($additional_part = array_pop($file_parts)) { - $extension = strtolower($additional_part . ($extension ? '.' . $extension : '')); - if (isset(self::$mapping['extensions'][$extension])) { - return self::$mapping['mimetypes'][self::$mapping['extensions'][$extension]]; - } - } - - return 'application/octet-stream'; - } - + /** * Changes permissions of the resource. - * - * This doesn't do anything for the moment so just returns TRUE; + * This wrapper doesn't support the concept of filesystem permissions. * * @param $mode * Integer value for the permissions. Consult PHP chmod() documentation @@ -296,42 +341,23 @@ * Returns TRUE. */ public function chmod($mode) { - $this->assertConstructorCalled(); - /** $modes = str_split($mode); - if($modes[0] == '0') { - array_shift($modes); - } - if(count($modes) != 3) { - return FALSE; - } - - if(intval($modes[2]) >= 4) { - $acl = AmazonS3::ACL_PUBLIC; - } - else { - $acl = AmazonS3::ACL_PRIVATE; - } - - $local_path = $this->getLocalPath(); - if($this->_amazons3_is_dir() && (substr($local_path, -1) != '/')) { - $local_path .= '/'; - } - - $response = $this->getS3()->set_object_acl($this->bucket, $local_path, $acl); - return $response->isOK();**/ + $this->_assert_constructor_called(); + $this->_test_log("chmod($mode) called."); return TRUE; } - + /** * Returns canonical, absolute path of the resource. + * This wrapper does not support realpath(). * * @return - * Returns FALSE as this wrapper does not provide an implementation. + * Returns FALSE. */ public function realpath() { + $this->_test_log("realpath() called."); return FALSE; } - + /** * Gets the name of the directory from a given path. * @@ -347,23 +373,27 @@ * * @see drupal_dirname() */ - public function dirname($uri = NULL) { - list($scheme, $target) = explode('://', $uri, 2); - $target = $this->getTarget($uri); - $dirname = dirname($target); - - if ($dirname == '.') { - $dirname = ''; - } - - return $scheme . '://' . $dirname; + public function dirname($uri=NULL) { + $this->_test_log("dirname($uri) called."); + if (!isset($uri)) { + $uri = $this->uri; + } + $target = $this->getTarget($uri); + $dirname = dirname($target); + + // Special case for calls to dirname('s3://'), ensuring that recursive + // calls eventually bottom out. + if ($dirname == '.') { + $dirname = ''; + } + return "s3://$dirname"; } - + /** * Support for fopen(), file_get_contents(), file_put_contents() etc. * * @param $uri - * A string containing the URI to the file to open. + * A string containing the URI of the file to open. * @param $mode * The file mode ("r", "wb" etc.). * @param $options @@ -377,30 +407,32 @@ * @see http://php.net/manual/en/streamwrapper.stream-open.php */ public function stream_open($uri, $mode, $options, &$opened_path) { + $this->_test_log("stream_open($uri, $mode, $options, $opened_path) called."); $this->uri = $uri; - - // if this stream is being opened for writing, clear the object buffer - // Return true as we'll create the object on fflush call + + // If this stream is being opened for writing, clear the object buffer + // We'll create the object during stream_flush(). if (strpbrk($mode, 'wax')) { - $this->clearBuffer(); + $this->_clear_buffer(); $this->write_buffer = TRUE; return TRUE; } - $metadata = $this->_amazons3_get_object($uri, $this->caching); + + $metadata = $this->_amazons3_get_object($uri); if ($metadata) { - $this->clearBuffer(); + $this->_clear_buffer(); $this->write_buffer = false; $this->object_size = $metadata['filesize']; return TRUE; } - + return FALSE; } - + /** * Support for fclose(). * - * Clears the object buffer and returns TRUE + * Clears the object buffer. * * @return * TRUE @@ -408,12 +440,14 @@ * @see http://php.net/manual/en/streamwrapper.stream-close.php */ public function stream_close() { - $this->clearBuffer(); + $this->_test_log("stream_close() called."); + $this->_clear_buffer(); return TRUE; } - + /** * Support for flock(). + * flock() is not supported at this time. * * @param $operation * One of the following: @@ -424,14 +458,15 @@ * supported on Windows). * * @return - * returns TRUE if lock was successful + * returns FALSE. * * @see http://php.net/manual/en/streamwrapper.stream-lock.php */ public function stream_lock($operation) { - return false; + $this->_test_log("stream_lock($operation) called."); + return FALSE; } - + /** * Support for fread(), file_get_contents() etc. * @@ -444,7 +479,8 @@ * @see http://php.net/manual/en/streamwrapper.stream-read.php */ public function stream_read($count) { - // make sure that count doesn't exceed object size + $this->_test_log("stream_read($count) called."); + // Make sure that count doesn't exceed object size if ($count + $this->position > $this->object_size) { $count = $this->object_size - $this->position; } @@ -453,20 +489,22 @@ $range_end = $this->position + $count - 1; if ($range_end > $this->buffer_length) { $opts = array( - 'range' => $this->position . '-' . $range_end, + 'range' => "{$this->position}-$range_end", ); - $response = $this->getS3()->get_object($this->bucket, $this->getLocalPath($this->uri), $opts); + $response = $this->s3->get_object($this->bucket, $this->_uri_to_s3_filename($this->uri), $opts); if ($response->isOK()) { $this->buffer .= $response->body; $this->buffer_length += strlen($response->body); } } - $data = substr($this->buffer, $this->position, $count); - $this->position += strlen($data); + if (isset($response)) { + $data = substr($response->body, 0, min($count, $this->object_size)); + $this->position += strlen($data); + } } return $data; } - + /** * Support for fwrite(), file_put_contents() etc. * @@ -479,14 +517,15 @@ * @see http://php.net/manual/en/streamwrapper.stream-write.php */ public function stream_write($data) { + $this->sw_call_count++; + $data_length = strlen($data); $this->buffer .= $data; $this->buffer_length += $data_length; $this->position += $data_length; - return $data_length; } - + /** * Support for feof(). * @@ -496,13 +535,13 @@ * @see http://php.net/manual/en/streamwrapper.stream-eof.php */ public function stream_eof() { + $this->_test_log("stream_eof() called."); if (!$this->uri) { - return true; + return TRUE; } - return ($this->position >= $this->object_size); } - + /** * Support for fseek(). * @@ -517,6 +556,7 @@ * @see http://php.net/manual/en/streamwrapper.stream-seek.php */ public function stream_seek($offset, $whence) { + $this->_test_log("stream_seek($offset, $whence) called."); switch($whence) { case SEEK_CUR: // Set position to current location plus $offset @@ -532,14 +572,14 @@ $new_position = $offset; break; } - + $ret = ($new_position >= 0 && $new_position <= $this->object_size); if ($ret) { $this->position = $new_position; } return $ret; } - + /** * Support for fflush(). Flush current cached stream data to storage. * @@ -549,20 +589,33 @@ * @see http://php.net/manual/en/streamwrapper.stream-flush.php */ public function stream_flush() { - if($this->write_buffer) { - $response = $this->getS3()->create_object($this->bucket, $this->getLocalPath(), array( + $this->_test_log("stream_flush() called after {$this->sw_call_count} calls to stream_write()."); + $this->sw_call_count = 0; + + // TODO: To enforce per user space limits, we may want to call file_validate_size() in here before + // writing to S3. + if ($this->write_buffer) { + $opts = array( 'body' => $this->buffer, 'acl' => AmazonS3::ACL_PUBLIC, 'contentType' => AmazonS3StreamWrapper::getMimeType($this->uri), - )); - if($response->isOK()) { - return TRUE; + ); + $response = $this->s3->create_object($this->bucket, $this->_uri_to_s3_filename($this->uri), $opts); + if ($response->isOK()) { + // Get the metadata for the file we just wrote. + $metadata = NULL; + $s3_metadata = $this->s3->get_object_metadata($this->bucket, $this->_uri_to_s3_filename($this->uri)); + if ($s3_metadata) { + $metadata = _amazons3_format_metadata($this->uri, $s3_metadata); + $this->_amazons3_write_cache($metadata); + return TRUE; + } } } - $this->clearBuffer(); + $this->_clear_buffer(); return FALSE; } - + /** * Support for ftell(). * @@ -572,9 +625,10 @@ * @see http://php.net/manual/en/streamwrapper.stream-tell.php */ public function stream_tell() { + $this->_test_log("stream_tell() called."); return $this->position; } - + /** * Support for fstat(). * @@ -585,9 +639,10 @@ * @see http://php.net/manual/en/streamwrapper.stream-stat.php */ public function stream_stat() { - return $this->_stat(); + $this->_test_log("stream_stat() called."); + return $this->_stat($this->uri); } - + /** * Support for unlink(). * @@ -595,23 +650,24 @@ * A string containing the uri to the resource to delete. * * @return - * TRUE if resource was successfully deleted. + * TRUE if resource was successfully deleted, regardless of whether or not + * the file actually existed. + * FALSE if the call to S3 failed, in which case the file will not be + * removed from the cache. * * @see http://php.net/manual/en/streamwrapper.unlink.php */ public function unlink($uri) { - $this->assertConstructorCalled(); - $response = $this->getS3()->delete_object($this->bucket, $this->getLocalPath($uri)); - if($response->isOK()) { - // Delete from cache - db_delete('amazons3_file') - ->condition('uri', $uri) - ->execute(); + $this->_assert_constructor_called(); + $this->_test_log("unlink($uri) called."); + $response = $this->s3->delete_object($this->bucket, $this->_uri_to_s3_filename($uri)); + if ($response->isOK()) { + $this->_amazons3_delete_cache($uri); return TRUE; } return FALSE; } - + /** * Support for rename(). * @@ -629,21 +685,28 @@ * @see http://php.net/manual/en/streamwrapper.rename.php */ public function rename($from_uri, $to_uri) { - $this->assertConstructorCalled(); - $from = $this->getLocalPath($from_uri); - $to = $this->getLocalPath($to_uri); - $s3 = $this->getS3(); - - $response = $s3->copy_object( + $this->_assert_constructor_called(); + $this->_test_log("rename() called."); + $from = $this->_uri_to_s3_filename($from_uri); + $to = $this->_uri_to_s3_filename($to_uri); + + $response = $this->s3->copy_object( array('bucket' => $this->bucket, 'filename' => $from), array('bucket' => $this->bucket, 'filename' => $to), array('acl' => AmazonS3::ACL_PUBLIC) ); - - // Check the response and then remove the original. - return $response->isOK() && $this->unlink($from_uri); + + // If the copy was successful, cache the new file, and remove the original. + if ($response->isOK()) { + $metadata = $this->_amazons3_read_cache($from_uri); + $metadata['uri'] = $to_uri; + $this->_amazons3_write_cache($metadata); + // The rename will have failed if the unlink fails. + return $this->unlink($from_uri); + } + return FALSE; } - + /** * Returns the local writable target of the resource within the stream. * @@ -660,17 +723,16 @@ * Returns a string representing a location suitable for writing of a file, * or FALSE if unable to write to the file such as with read-only streams. */ - protected function getTarget($uri = NULL) { + protected function getTarget($uri=NULL) { if (!isset($uri)) { $uri = $this->uri; } - - list($scheme, $target) = explode('://', $uri, 2); - + $this->_test_log("getTarget($uri) called."); + $data = explode('://', $uri, 2); // Remove erroneous leading or trailing forward-slashes and backslashes. - return trim($target, '\/'); + return count($data) == 2 ? trim($data[1], '\/') : FALSE; } - + /** * Support for mkdir(). * @@ -687,19 +749,33 @@ * @see http://php.net/manual/en/streamwrapper.mkdir.php */ public function mkdir($uri, $mode, $options) { - $this->assertConstructorCalled(); - $recursive = (bool) ($options & STREAM_MKDIR_RECURSIVE); - $localpath = $this->getLocalPath($uri); - - // s3 has no concept of directories, but we emulate it by creating an - // object of size 0 with a trailing "/" - $response = $this->getS3()->create_object($this->bucket, $localpath . '/', array('body' => '')); - if($response->isOk()) { - return TRUE; + $this->_assert_constructor_called(); + $this->_test_log("mkdir($uri, $mode, $options) called."); + + // If this URI already exists in the cache, return TRUE if it's a folder + // (so that recursive calls won't improperly report failure when they + // reach an existing ancestor), or FALSE if it's a file (failure). + $test_metadata = $this->_amazons3_read_cache($uri); + if ($test_metadata) { + return (bool)$test_metadata['dir']; + } + + // S3 is a flat file system, with no concept of directories (just files + // with slashes in their names). To represent folders, we store them in the + // metadata cache, without creating anything in S3. + $metadata = _amazons3_format_metadata($uri, array()); + $metadata['timestamp'] = date('U', time()); + $this->_amazons3_write_cache($metadata); + + // If the STREAM_MKDIR_RECURSIVE option was specified, also create all the + // ancestor folders of this uri. + $parent_dir = drupal_dirname($uri); + if (($options & STREAM_MKDIR_RECURSIVE) && $parent_dir != 's3://') { + return $this->mkdir($parent_dir, $mode, $options); } - return FALSE; + return TRUE; } - + /** * Support for rmdir(). * @@ -710,33 +786,37 @@ * * @return * TRUE if directory was successfully removed. + * FALSE if the directory was not empty. * * @see http://php.net/manual/en/streamwrapper.rmdir.php */ public function rmdir($uri, $options) { - $this->assertConstructorCalled(); - $localpath = $this->getLocalPath($uri); - $s3 = $this->getS3(); - - $objects = $s3->get_object_list($this->bucket, array('prefix' => $localpath)); - if (gettype($objects) === 'array' && !empty($objects)) { - $or = db_or(); - foreach($objects as $object) { - $s3->batch()->delete_object($this->bucket, $object); - // Delete from cache - $object_uri = 's3://' . rtrim($object,'/'); - $or->condition('uri', $object_uri, '='); - } - db_delete('amazons3_file')->condition($or)->execute(); - $responses = $s3->batch()->send(); - - if($responses->areOK()) { - return TRUE; - } + $this->_assert_constructor_called(); + $this->_test_log("rmdir($uri, $options) called."); + if (!$this->_amazons3_is_dir($uri)) { + return FALSE; + } + + if ($uri[strlen($uri)-1] != '/') { + // If it needs one, add a trailing '/' to the URI, to differentiate + // from files with this folder's name as a substring. + // e.g. rmdir('s3://foo/bar') should not care about the existence of + // 's3://foo/barbell.jpg'. + $uri .= '/'; + } + + $query = db_query("SELECT * FROM {amazons3_file} WHERE uri LIKE :folder", array(':folder' => "$uri%")); + $files = $query->fetchAll(); + if (empty($files)) { + // This folder is empty, so it's elegible for deletion. + $result = db_delete('amazons3_file') + ->condition('uri', rtrim($uri, '/'), '=') + ->execute(); + return (bool)$result; } return FALSE; } - + /** * Support for stat(). * @@ -752,92 +832,50 @@ * @see http://php.net/manual/en/streamwrapper.url-stat.php */ public function url_stat($uri, $flags) { - $this->assertConstructorCalled(); + $this->_assert_constructor_called(); + $this->_test_log("url_stat($uri, $flags) called."); return $this->_stat($uri); } - + /** * Support for opendir(). * * @param $uri * A string containing the URI to the directory to open. * @param $options - * Unknown (parameter is not documented in PHP Manual). + * A boolean used to enable safe_mode, which this wrapper doesn't support. * * @return * TRUE on success. * * @see http://php.net/manual/en/streamwrapper.dir-opendir.php */ - public function dir_opendir($uri, $options) { - $this->assertConstructorCalled(); - if ($uri == null) { + public function dir_opendir($uri, $options=NULL) { + $this->_assert_constructor_called(); + $this->_test_log("dir_opendir($uri, $options) called."); + if (!$this->_amazons3_is_dir($uri)) { return FALSE; } - else if(!$this->_amazons3_is_dir($uri)) { - return FALSE; - } - + + if ($uri != 's3://') { + // If this isn't the root folder, add a trailing '/' to differentiate + // from folders with this folder's name as a substring. + $uri .= '/'; + } + + // Get the list of uris for files and folders which are in the specified + // folder, but not in any of its subfolders. + $query = db_query("SELECT uri FROM {amazons3_file} WHERE uri LIKE :folder AND uri NOT LIKE :subfolder", + array(':folder' => "$uri%", ':subfolder' => "$uri%/%")); + + // Create $this->dir as an empty array, since the folder might be empty. $this->dir = array(); - $path = $this->getLocalPath($uri); - $truncated = TRUE; - $marker = ''; - if(strlen($path) == 0) { - $prefix = $path; - } - else { - $prefix = $path . '/'; - } - $prefix_length = strlen($prefix); - - while($truncated) { - $response = $this->getS3()->list_objects($this->bucket, array( - 'delimiter' => '/', - 'prefix' => $prefix, - 'marker' => urlencode($marker), - )); - if ($response->isOK()) { - - $this->dir[] = '.'; - $this->dir[] = '..'; - - // Folders - if (isset($response->body->CommonPrefixes)) { - foreach($response->body->CommonPrefixes as $prefix) { - $marker = substr($prefix->Prefix, $prefix_length, strlen($prefix->Prefix) - $prefix_length - 1); - if(strlen($marker) > 0) { - $this->dir[] = $marker; - } - } - } - - // Files - if(isset($response->body->Contents)) { - $contents = $response->body->to_stdClass()->Contents; - if (!is_array($contents)) { - $contents = array($contents); - } - - foreach($contents as $content) { - $key = $content->Key; - if(substr_compare($key, '/', -1, 1) !== 0) { - $marker = basename($key); - $this->dir[] = $marker; - } - } - } - - if(!isset($response->body->IsTruncated) || $response->body->IsTruncated == 'false') { - $truncated = FALSE; - } - } - else { - return FALSE; - } + foreach ($query->fetchAll(PDO::FETCH_COLUMN, 0) as $uri) { + $this->dir[] = basename($uri); } return TRUE; } - + /** * Support for readdir(). * @@ -847,13 +885,11 @@ * @see http://php.net/manual/en/streamwrapper.dir-readdir.php */ public function dir_readdir() { - $filename = current($this->dir); - if ($filename !== false) { - next($this->dir); - } - return $filename; + $this->_test_log("dir_readdir() called."); + $entry = each($this->dir); + return $entry ? $entry['value'] : FALSE; } - + /** * Support for rewinddir(). * @@ -863,10 +899,11 @@ * @see http://php.net/manual/en/streamwrapper.dir-rewinddir.php */ public function dir_rewinddir() { + $this->_test_log("dir_rewinddir() called."); reset($this->dir); return TRUE; } - + /** * Support for closedir(). * @@ -876,95 +913,58 @@ * @see http://php.net/manual/en/streamwrapper.dir-closedir.php */ public function dir_closedir() { - $this->dir = array(); + $this->_test_log("dir_closedir() called."); + unset($this->dir); return TRUE; } - - /** - * Return the local filesystem path. - * - * @param $uri - * Optional URI, supplied when doing a move or rename. - */ - protected function getLocalPath($uri = NULL) { - if (!isset($uri)) { - $uri = $this->uri; - } - - $path = str_replace('s3://', '', $uri); - $path = trim($path, '/'); - return $path; - } - + /** * Gets the path that the wrapper is responsible for. + * Even though this function isn't specified in DrupalStreamWrapperInterface, + * Drupal's code calls it on all wrappers, so we need to define it. * * @return * String specifying the path. */ public function getDirectoryPath() { + $this->_test_log("getDirectoryPath() called."); return $this->domain; } - + /** - * Flush the object buffers + * Convert a URI into a valid S3 filename. */ - protected function clearBuffer() { + protected function _uri_to_s3_filename($uri) { + $filename = str_replace('s3://', '', $uri); + // Remove both leading and trailing /s. S3 filenames never start with /, + // and a $uri for a folder might be specified with a trailing /, which + // we'd need to remove to be able to retrieve it from the cache. + $path = trim($filename, '/'); + return $filename; + } + + /** + * Flush the object buffers. + */ + protected function _clear_buffer() { $this->position = 0; $this->object_size = 0; $this->buffer = null; $this->buffer_write = false; $this->buffer_length = 0; } - + /** - * Get the S3 connection object + * Get the status of the file with the specified URI. * * @return - * S3 connection object (AmazonS3) - * - * @see http://docs.amazonwebservices.com/AWSSDKforPHP/latest/#i=AmazonS3 - */ - protected function getS3() { - if($this->s3 == null) { - $bucket = variable_get('amazons3_bucket', ''); - - if(!libraries_load('awssdk') && !isset($bucket)) { - drupal_set_message('Unable to load the AWS SDK. Please check you have installed the library correctly and configured your S3 credentials.'. 'error'); - } - else if(!isset($bucket)) { - drupal_set_message('Bucket name not configured.'. 'error'); - } - else { - try { - $this->s3 = new AmazonS3(); - $this->bucket = $bucket; - } - catch(RequestCore_Exception $e){ - drupal_set_message('There was a problem connecting to S3', 'error'); - } - catch(Exception $e) { - drupal_set_message('There was a problem using S3: ' . $e->getMessage(), 'error'); - } - } - } - return $this->s3; - } - - /** - * Get file status - * - * @return - * An array with file status, or FALSE in case of an error - see fstat() - * for a description of this array. + * An array with file status, or FALSE if the file doesn't exist. + * See fstat() for a description of this array. * * @see http://php.net/manual/en/streamwrapper.stream-stat.php */ - protected function _stat($uri = NULL) { - if(!isset($uri)) { - $uri = $this->uri; - } - $metadata = $this->_amazons3_get_object($uri, $this->caching); + protected function _stat($uri) { + $metadata = $this->_amazons3_get_object($uri); if ($metadata) { $stat = array(); $stat[0] = $stat['dev'] = 0; @@ -980,7 +980,7 @@ $stat[10] = $stat['ctime'] = 0; $stat[11] = $stat['blksize'] = 0; $stat[12] = $stat['blocks'] = 0; - + if (!$metadata['dir']) { $stat[4] = $stat['uid'] = $metadata['uid']; $stat[7] = $stat['size'] = $metadata['filesize']; @@ -991,12 +991,11 @@ return $stat; } return FALSE; -} - - -/** - * Determine whether the $uri is a directory - * + } + + /** + * Determine whether the $uri is a directory. + * * @param $uri * A string containing the uri to the resource to check. If none is given * defaults to $this->uri @@ -1004,150 +1003,160 @@ * @return * TRUE if the resource is a directory */ - protected function _amazons3_is_dir($uri = null) { - if($uri == null) { - $uri = $this->uri; - } - if($uri != null) { - $path = $this->getLocalPath($uri); - if (strlen($path) === 0) { - return TRUE; - } - $response = $this->getS3()->list_objects($this->bucket, array( - 'prefix' => $path . '/', - 'max-keys' => 1, - )); - if($response && isset($response->body->Contents->Key)) { - return TRUE; - } + protected function _amazons3_is_dir($uri) { + if ($uri == 's3://' || $uri == 's3:') { + return TRUE; } - return FALSE; - } - - /** - * CACHING FUNCTIONS - */ - + + // Folders only exist in the cache; they are not stored in S3. + $metadata = $this->_amazons3_read_cache($uri); + return $metadata ? $metadata['dir'] : FALSE; + } + + /**************************************************************************** + CACHE FUNCTIONS + ***************************************************************************/ + /** - * Try to fetch an object from the metadata cache, otherwise fetch it's - * info from S3 and populate the cache. + * Try to fetch an object from the metadata cache. If that file isn't in the + * cache, it is considered to be nonexistant. * * @param uri * A string containing the uri of the resource to check. - * @param $cach - * A boolean representing whether to check the cache for file information. * * @return * An array if the $uri exists, otherwise FALSE. */ - protected function _amazons3_get_object($uri, $cache = TRUE) { + protected function _amazons3_get_object($uri) { + // Since this is an internal function, don't log it by default. + //$this->_test_log("_amazons3_get_object($uri) called."); + // For the root directory, just return metadata for a generic folder. if ($uri == 's3://' || $uri == 's3:') { - $metadata = $this->_amazons3_format_response('/', array(), TRUE); + $metadata = _amazons3_format_metadata('/', array(), TRUE); return $metadata; } - else { - $uri = rtrim($uri,'/'); - } - - if ($cache) { - $metadata = $this->_amazons3_get_cache($uri); - if ($metadata) { + + // Trim any trailing '/', in case this is a folder request. + $uri = rtrim($uri, '/'); + + // Check if this URI is in the cache. + $metadata = $this->_amazons3_read_cache($uri); + + // If cache ignore is enabled, query S3 for all file requests. + if (variable_get('amazons3_ignore_cache', FALSE)) { + // Even when ignoring the cache, we still read folders from it, because + // they aren't stored in S3. + if (!empty($metadata['dir'])) { return $metadata; } - } - - $is_dir = $this->_amazons3_is_dir($uri); - $metadata = NULL; - if ($is_dir) { - $metadata = $this->_amazons3_format_response($uri, array(), TRUE); - } - else { - $response = $this->getS3()->get_object_metadata($this->bucket, $this->getLocalPath($uri)); - if ($response) { - $metadata = $this->_amazons3_format_response($uri, $response); + // Query S3. + $s3_metadata = $this->s3->get_object_metadata($this->bucket, $this->_uri_to_s3_filename($uri)); + if ($s3_metadata) { + return _amazons3_format_metadata($uri, $s3_metadata); } + // This URI is neither a cached folder nor in S3: it doesn't exist. + return FALSE; } - if (is_array($metadata)) { - // Save to the cache - db_merge('amazons3_file') - ->key(array('uri' => $metadata['uri'])) - ->fields($metadata) - ->execute(); - return $metadata; - } - return FALSE; + + return $metadata; } - + /** - * Fetch an object from the local metadata cache + * Fetch an object from the file metadata cache table. * * @param uri - * A string containing the uri of the resource to check. + * A string containing the uri of the resource to check. * - * @return - * An array if the $uri is in the cache, otherwise FALSE + * @return + * An array of metadata if the $uri is in the cache, otherwise FALSE. */ - protected function _amazons3_get_cache($uri) { - // Check cache for existing object. - $result = db_query("SELECT * FROM {amazons3_file} WHERE uri = :uri", array( - ':uri' => $uri, - )); - $record = $result->fetchAssoc(); - if ($record) { - return $record; + protected function _amazons3_read_cache($uri) { + // Since this is an internal function, don't log it by default. + //$this->_test_log("_amazons3_read_cache($uri) called."); + $record = db_query("SELECT * FROM {amazons3_file} WHERE uri = :uri", array(':uri' => $uri))->fetchAssoc(); + return $record ? $record : FALSE; + } + + /** + * Write an object's metadata to the cache. Also write its ancestor folders + * to the cache. + * + * @param metadata + * An associative array of file metadata, in this format: + * 'uri' => The full URI of the file, including 's3://'. + * 'filesize' => The size of the file, in bytes. + * 'timestamp' => The file's create/update timestamp. + * 'dir' => A boolean indicating whether the object is a directory. THIS IS DEPRECATED. + * 'mode' => The octal mode of the file. + * 'uid' => The uid of the owner of the S3 object. + * + * @throws + * If an exception occurs from the database call, it will percolate out of this function. + */ + protected function _amazons3_write_cache($metadata) { + // Since this is an internal function, don't log it by default. + //$this->_test_log("_amazons3_write_cache({$metadata['uri']}) called."); + db_merge('amazons3_file') + ->key(array('uri' => $metadata['uri'])) + ->fields($metadata) + ->execute(); + + $dirname = $this->dirname($metadata['uri']); + if ($dirname != 's3://') { + $this->mkdir($dirname, NULL, STREAM_MKDIR_RECURSIVE); } - return FALSE; } - + /** - * Format returned file information from S3 into an array + * Delete an object's metadata from the cache. * - * @param $uri - * A string containing the uri of the resource to check. - * @param $response - * An array containing the collective metadata for the Amazon S3 object - * @param $is_dir - * A boolean indicating whether this object is a directory. + * @param uri + * The URI of the object to be deleted. Also accepts an array of URIs + * to be deleted. * - * @return - * An array containing formatted metadata + * @throws + * If an exception occurs from the database call, it will percolate out of this function. */ - protected function _amazons3_format_response($uri, $response, $is_dir = FALSE) { - $metadata = array('uri' => $uri); - if (isset($response['Size'])) { - $metadata['filesize'] = $response['Size']; - } - if (isset($response['LastModified'])) { - $metadata['timestamp'] = date('U', strtotime((string) $response['LastModified'])); - } - if (isset($response['Owner']['ID'])) { - $metadata['uid'] = (string) $response['Owner']['ID']; - } - if ($is_dir) { - $metadata['dir'] = 1; - $metadata['mode'] = 0040000; // S_IFDIR indicating directory - $metadata['mode'] |= 0777; + protected function _amazons3_delete_cache($uri) { + // Since this is an internal function, don't log it by default. + //$this->_test_log("_amazons3_delete_cache($uri) called."); + $delete_query = db_delete('amazons3_file'); + if (is_array($uri)) { + // Build an OR condition to delete all the URIs in one query. + $or = db_or(); + foreach ($uri as $u) { + $or->condition('uri', $u, '='); + } + $delete_query->condition($or); } else { - $metadata['dir'] = 0; - $metadata['mode'] = 0100000; // S_IFREG indicating file - $metadata['mode'] |= 0777; // everything is writeable + $delete_query->condition('uri', $uri, '='); } - return $metadata; + $delete_query->execute(); } - + /** - * Assert that the constructor has been called, call it if not. + * Call the constructor it it hasn't been has been called yet. * * Due to PHP bug #40459, the constructor of this class isn't always called - * for some of the methods. This private method calls the constructor if - * it hasn't been called before. + * for some of the methods. * * @see https://bugs.php.net/bug.php?id=40459 */ - private function assertConstructorCalled() { - if ($this->domain === NULL) { + protected function _assert_constructor_called() { + if ($this->constructed === FALSE) { $this->__construct(); } } + + /** + * Used for testing. + */ + private static function _test_log($msg) { + global $amazons3_testing; + if ($amazons3_testing) { + print "TEST: $msg\n"; + flush(); + } + } } diff -u README README --- README 2013-09-12 21:14:44.000000000 -0700 +++ README 2013-09-10 13:46:59.000000000 -0700 @@ -1,5 +1,5 @@ Requirements -You will need to set allow_url_fopen to on your PHP settings. This option enables the URL-aware fopen wrappers that enable accessing URL object like files. +You will need to set allow_url_fopen in your PHP settings. This option enables the URL-aware fopen wrappers that enable accessing remote objects like files. Known Issues Some curl libraries, such as the one bundled with MAMP, do not come with authoritative certificate files. http://dev.soup.io/post/56438473/If-youre-using-MAMP-and-doing-something @@ -11,11 +11,13 @@ (For installation of awssdk, you will need to download the Amazon SDK for PHP and place it in sites/all/libraries/awdsdk ) http://aws.amazon.com/sdkforphp/ -- Configure AWS SDK +- Configure AWS SDK (using either the AWS SDK for PHP UI module, or storing the settings in your settings.php file's $conf array). -- Configure your bucket setttings at /admin/config/media/amazon +- Configure your bucket setttings at /admin/config/media/amazons3 + +- Refresh your file metadata cache using the button at the bottom of /admin/config/media/amazons3 You can then: -- Visit admin/config/media/file-system and set the Default download method to S3 +- Visit admin/config/media/file-system and set the Default download method to Amazon Simple Storage Service and/or -- Add a field of type File or Image etc and set the Upload destination to Amazon S3 in the Field Settings tab. \ No newline at end of file +- Add a field of type File or Image etc and set the Upload destination to Amazon Simple Storage Service in the Field Settings tab. diff -u amazons3.api.php amazons3.api.php --- amazons3.api.php 2013-09-12 21:14:44.000000000 -0700 +++ amazons3.api.php 2013-09-10 13:46:59.000000000 -0700 @@ -36,4 +36,3 @@ } return $info; } - Only in amazons3 2.1: amazons3.drush.inc diff -u amazons3.info amazons3.info --- amazons3.info 2013-09-13 00:33:12.000000000 -0700 +++ amazons3.info 2013-09-10 13:46:59.000000000 -0700 @@ -1,17 +1,11 @@ -; $Id$ name = AmazonS3 -description = Provides S3 stream wrapper class +description = Provides a stream wrapper which allows Drupal to use Amazon Simple Storage Service (S3) as a remote file system. package = Amazon S3 core = 7.x -files[] = amazons3.module -files[] = AmazonS3StreamWrapper.inc -dependencies[] = awssdk (>=5.1) -dependencies[] = libraries (2.x) configure = admin/config/media/amazons3 -; Information added by drupal.org packaging script on 2013-09-13 -version = "7.x-1.0-beta7+17-dev" -core = "7.x" -project = "amazons3" -datestamp = "1379032392" +dependencies[] = awssdk (>=5.1) +dependencies[] = libraries (2.x) +files[] = amazons3.module +files[] = AmazonS3StreamWrapper.inc diff -u amazons3.install amazons3.install --- amazons3.install 2013-09-12 21:14:44.000000000 -0700 +++ amazons3.install 2013-09-11 15:31:50.000000000 -0700 @@ -17,11 +17,11 @@ $fopen_allowed = ini_get('allow_url_fopen'); $ok_message = $t('The PHP allow_url_fopen setting is on.'); - $error_message = $t('Amazon S3 module requires that the allow_url_fopen setting be turned on in php.ini.'); + $error_message = $t('The AmazonS3 module requires that the allow_url_fopen setting be turned on in php.ini.'); $requirements['amazons3_allow_url_fopen'] = array( 'severity' => $fopen_allowed ? REQUIREMENT_OK : REQUIREMENT_ERROR, - 'title' => $t('AmazonS3'), + 'title' => 'AmazonS3', 'value' => 'allow_url_fopen', 'description' => $fopen_allowed ? $ok_message : $error_message, ); @@ -36,7 +36,7 @@ variable_del('amazons3_bucket'); variable_del('amazons3_cname'); variable_del('amazons3_domain'); - variable_del('amazons3_cache'); + variable_del('amazons3_ignore_cache'); variable_del('amazons3_torrents'); variable_del('amazons3_presigned_urls'); variable_del('amazons3_saveas'); @@ -50,7 +50,7 @@ 'description' => 'Stores information for uploaded Amazon S3 files.', 'fields' => array( 'uri' => array( - 'description' => 'The URI to access the file (either local or remote).', + 'description' => 'The S3 URI of the file.', 'type' => 'varchar', 'length' => 255, 'not null' => TRUE, @@ -84,7 +84,7 @@ 'default' => 0, ), 'uid' => array( - 'description' => 'The uid of the user who is associated with the file (not Drupal uid).', + 'description' => 'The S3 uid of the user who is associated with the file.', 'type' => 'varchar', 'length' => 255, 'not null' => TRUE, @@ -95,6 +95,7 @@ 'timestamp' => array('timestamp'), ), 'primary key' => array('uri'), + 'collation' => 'utf8_bin' ); return $schema; @@ -168,3 +169,30 @@ db_change_field('amazons3_file', 'uid', 'uid', $spec); } +/** + * Updates AmazonS3 for coredumperror's v2.0 re-write. + */ +function amazons3_update_7200() { + // Caching is now always enabled, unless it gets explicity disabled by the user. + // This disabling is done through a new variable, so the old one is obsolete. + variable_del('amazons3_cache'); + + // Execute a full cache refresh. + $bucket = variable_get('amazons3_bucket', FALSE); + if ($bucket) { + _amazons3_refresh_cache($bucket); + } + else { + drupal_set_message(t('Unable to determine AmazonS3 bucket name for cache refresh. Please set the bucket name and perform a manual cache refresh from the AmazonS3 configuration page.'), 'warning'); + } +} + +/** + * Updates the amazons3_file table to use case sensitive collation. + */ +function amazons3_update_7201() { + // As stated here: http://forums.mysql.com/read.php?103,19380,200971#msg-200971 + // MySQL doesn't directly support case sensitive UTF8 collation. Fortunately, + // 'utf8_bin' collation is good enough for our purposes. + db_query("alter table amazons3_file convert to character set utf8 collate utf8_bin;"); +} diff -u amazons3.module amazons3.module --- amazons3.module 2013-09-12 21:14:44.000000000 -0700 +++ amazons3.module 2013-09-11 16:50:09.000000000 -0700 @@ -28,7 +28,7 @@ $items['admin/config/media/amazons3'] = array( 'title' => 'Amazon S3', - 'description' => 'Configure your S3 credentials', + 'description' => t('Configure Amazons S3 settings.'), 'page callback' => 'drupal_get_form', 'page arguments' => array('amazons3_admin'), 'access arguments' => array('administer amazons3'), @@ -38,7 +38,7 @@ } /** - * Implementation of hook_permission() + * Implementation of hook_permission(). */ function amazons3_permission() { return array( @@ -55,10 +55,12 @@ switch ($path) { case 'admin/config/media/amazons3': if (module_exists('awssdk_ui')) { - return '

' . t('Amazon Web Services authentication can be configured at the AWS SDK configuration page.', array('@awssdk_config' => url('admin/config/media/awssdk'))) . '

'; + return '

' . t('Amazon Web Services authentication can be configured on the AWS SDK configuration page.', + array('@awssdk_config' => url('admin/config/media/awssdk'))) . '

'; } else { - return '

' . t('Enable \'AWS SDK for PHP UI\' module to configure your Amazon Web Services authentication. Configuration can also be defined in the $conf array in settings.php.', array('@awssdk_config' => url('admin/config/media/awssdk'))) . '

'; + return '

' . t('To configure your Amazon Web Services credentials, enable the \'AWS SDK for PHP UI\' module, + or define those settings in the $conf array in settings.php.') . '

'; } } } @@ -70,42 +72,35 @@ $form = array(); $form['amazons3_bucket'] = array( - '#type' => 'textfield', - '#title' => t('Default Bucket Name'), - '#default_value' => variable_get('amazons3_bucket', ''), - '#required' => TRUE, - ); - - $form['amazons3_cache'] = array( - '#type' => 'checkbox', - '#title' => t('Enable database caching'), - '#description' => t('Enable a local file metadata cache, this significantly reduces calls to S3'), - '#default_value' => variable_get('amazons3_cache', 1), + '#type' => 'textfield', + '#title' => t('Default Bucket Name'), + '#default_value' => variable_get('amazons3_bucket', ''), + '#required' => TRUE, ); $form['amazons3_cname'] = array( '#type' => 'checkbox', '#title' => t('Enable CNAME'), - '#description' => t('Serve files from a custom domain by using an appropriately named bucket e.g. "mybucket.mydomain.com"'), + '#description' => t('Serve files from a custom domain by using an appropriately named bucket, e.g. "mybucket.mydomain.com".'), '#default_value' => variable_get('amazons3_cname', 0), ); $form['amazons3_domain'] = array( - '#type' => 'textfield', - '#title' => t('CDN Domain Name'), - '#description' => t('If serving files from CloudFront then the bucket name can differ from the domain name.'), - '#default_value' => variable_get('amazons3_domain', ''), - '#states' => array( - 'visible' => array( - ':input[id=edit-amazons3-cname]' => array('checked' => TRUE), - ) - ), + '#type' => 'textfield', + '#title' => t('CDN Domain Name'), + '#description' => t('If serving files from CloudFront, the bucket name can differ from the domain name.'), + '#default_value' => variable_get('amazons3_domain', ''), + '#states' => array( + 'visible' => array( + ':input[id=edit-amazons3-cname]' => array('checked' => TRUE), + ) + ), ); $form['amazons3_torrents'] = array( '#type' => 'textarea', '#title' => t('Torrents'), - '#description' => t('A list of paths that should be delivered through a torrent url. Enter one value per line e.g. "mydir/*". Paths are relative to the Drupal file directory and use patterns as per preg_match. If no timeout is provided, it defaults to 60 seconds.', array('@preg_match' => 'http://php.net/preg_match')), + '#description' => t('A list of paths that should be delivered through a torrent url. Enter one value per line e.g. "mydir/*". Paths are relative to the Drupal file directory and use patterns as per preg_match.', array('@preg_match' => 'http://php.net/preg_match')), '#default_value' => variable_get('amazons3_torrents', ''), '#rows' => 10, ); @@ -121,76 +116,266 @@ $form['amazons3_saveas'] = array( '#type' => 'textarea', '#title' => t('Force Save As'), - '#description' => t('A list of paths that foce the user to save the file by using Content-disposition header. Prevents autoplay of media. Enter one value per line. e.g. "mydir/*". Paths are relative to the Drupal file directory and use patterns as per preg_match. Files must use a presigned url to use this.', array('@preg_match' => 'http://php.net/preg_match')), + '#description' => t('A list of paths that force the user to save the file, by using the Content-Disposition header. Prevents autoplay of media. Enter one value per line. e.g. "mydir/*". Paths are relative to the Drupal file directory and use patterns as per preg_match. Files must use a presigned url to use this feature.', array('@preg_match' => 'http://php.net/preg_match')), '#default_value' => variable_get('amazons3_saveas', ''), '#rows' => 10, ); - $form['amazons3_clear_cache'] = array( + $form['amazons3_refresh_cache'] = array( '#type' => 'fieldset', - '#title' => t('Clear cache'), + '#description' => t("The file metadata cache keeps track of every file that AmazonS3 writes to (and deletes from) the S3 bucket, so that queries for data about those files (checks for existence, filetype, etc.) don't have to hit S3. This speeds up many operations, most noticeably anything related to images and their derivatives."), + '#title' => t('File Metadata Cache'), ); - $form['amazons3_clear_cache']['clear'] = array( + $form['amazons3_refresh_cache']['refresh'] = array( '#type' => 'submit', - '#value' => t('Clear file metadata cache'), - '#submit' => array('amazons3_clear_cache_submit'), + '#suffix' => '
' . t("This button queries S3 for the metadata of all the files in your site's bucket, and saves it to the database. This may take a while for buckets with many thousands of files.
It should only be necessary to use this button if you've just installed AmazonS3 and you need to cache all the pre-existing files in your bucket, or if you need to restore your metadata cache from scratch for some other reason.") . '
', + '#value' => t('Refresh file metadata cache'), + '#submit' => array('amazons3_refresh_cache_submit'), + ); + // Push the button closer to its own description, rather than the fieldset's description, and push the disable checkbox away from the button description. + $form['amazons3_refresh_cache']['refresh']['#attached']['css'] = array('#edit-refresh {margin-bottom: 0; margin-top: 1em;} div.refresh {margin-bottom: 1em;}' => array('type' => 'inline')); + + $form['amazons3_refresh_cache']['amazons3_ignore_cache'] = array( + '#type' => 'checkbox', + '#title' => t('Ignore the file metadata cache'), + '#description' => t("If you need to debug a problem with S3, you may want to temporarily ignore the file metadata cache. This will make all filesystem reads hit S3 instead of the cache. Please be aware that this will cause an enormous performance loss, and should never be enabled on a production site."), + '#default_value' => variable_get('amazons3_ignore_cache', 0), ); - + return system_settings_form($form); } function amazons3_admin_validate($form, &$form_state) { - $bucket = $form_state['values']['amazons3_bucket']; + _amazons3_validate_config($form_state['values']['amazons3_bucket']); +} - if(!libraries_load('awssdk')) { - form_set_error('amazons3_bucket', t('Unable to load the AWS SDK. Please check you have installed the library correctly and configured your S3 credentials.')); +/** + * Submit callback for the refresh file metadata cache button. + */ +function amazons3_refresh_cache_submit($form, &$form_state) { + _amazons3_refresh_cache($form_state['values']['amazons3_bucket']); +} + +/** + * Checks all the configuration options to ensure that they're valid. + * + * @return + * TRUE if config is good to go, otherwise FALSE. + */ +function _amazons3_validate_config($bucket) { + if (!libraries_load('awssdk')) { + form_set_error('amazons3_bucket', t('Unable to load the AWS SDK. Please ensure that you have installed the library correctly and configured your S3 credentials.')); + return FALSE; } - else if(!class_exists('AmazonS3')) { - form_set_error('amazons3_bucket', t('Cannot load AmazonS3 class. Please check the awssdk is installed correctly')); + else if (!class_exists('AmazonS3')) { + form_set_error('amazons3_bucket', t('Cannot load AmazonS3 class. Please ensure that the awssdk library is installed correctly.')); + return FALSE; } else { try { $s3 = new AmazonS3(); - // test connection + // Test the connection to S3. $user_id = $s3->get_canonical_user_id(); - if(!$user_id['id']) { - form_set_error('amazons3_bucket', t('The S3 access credentials are invalid')); + if (!$user_id['id']) { + form_set_error('amazons3_bucket', t('The S3 access credentials are invalid.')); + return FALSE; } - else if(!$s3->if_bucket_exists($bucket)) { - form_set_error('amazons3_bucket', t('The bucket does not exist')); + else if (!$s3->if_bucket_exists($bucket)) { + form_set_error('amazons3_bucket', t('The bucket "@bucket" does not exist.', array('@bucket' => $bucket))); + return FALSE; } } - catch(RequestCore_Exception $e){ - if(strstr($e->getMessage(), 'SSL certificate problem')) { + catch (RequestCore_Exception $e){ + if (strstr($e->getMessage(), 'SSL certificate problem')) { form_set_error('amazons3_bucket', t('There was a problem with the SSL certificate. Try setting AWS_CERTIFICATE_AUTHORITY to true in "libraries/awssdk/config.inc.php". You may also have a curl library (e.g. the default shipped with MAMP) that does not contain trust certificates for the major authorities.')); + return FALSE; } else { - form_set_error('amazons3_bucket', t('There was a problem connecting to S3')); + form_set_error('amazons3_bucket', t('There was a problem connecting to S3: @error', array('@error' => $e->getMessage()))); + return FALSE; } - } - catch(Exception $e) { - form_set_error('amazons3_bucket', t('There was a problem using S3')); + catch (Exception $e) { + form_set_error('amazons3_bucket', t('There was a problem using S3: @error', array('@error' => $e->getMessage()))); + return FALSE; } } + return TRUE; } -function amazons3_image_style_flush($style) { - // Empty cached data that contains information about the style. - if(isset($style->old_name) && strlen($style->old_name) > 0) { - drupal_rmdir('s3://styles/' . $style->old_name); +/** + * Calls AmazonS3::list_objects() enough times to get all the files in the + * specified bucket (the API returns at most 1000 per call), and stores their + * metadata in the cache table. + * + * Once the file metadata has been created, the the folder metadata will + * also be refreshed. + */ +function _amazons3_refresh_cache($bucket) { + // Don't try to do anything if our configuration settings are invalid. + if (!_amazons3_validate_config($bucket)) { + return; + } + $s3 = new AmazonS3(); + $metadata_fields = array('uri', 'filesize', 'timestamp', 'dir', 'mode', 'uid'); + + // Clear the files out of the metadata table, so we can recreate them from scratch. + // Directories are not erased because if the directory doesn't have any files in it, + // it wouldn't be restored in the last step of this function. + db_delete('amazons3_file') + ->condition('dir', 0, '=') + ->execute(); + + $last_key = NULL; + do { + $args = array(); + if ($last_key) { + $args['marker'] = $last_key; + } + + $response = $s3->list_objects($bucket, $args); + if (!$response->isOK()) { + drupal_set_message(t('Metadata cache refresh aborted. A @code error occurred: @error.', array('@code' => $response->status, '@error' => $response->body->Message)), 'error'); + return; + } + $file_metadata_list = array(); + $folder_metadata_list = array(); + foreach ($response->body->Contents as $object) { + $s3_metadata = _amazons3_s3_object_to_s3_metadata($object); + $uri = "s3://{$s3_metadata['Key']}"; + $is_dir = $uri[strlen($uri)-1] == '/'; + + if ($is_dir) { + // There may be files in the S3 bucket pretending to be folders, by + // having a trailing '/'. Add those to the cache as directories. + $folder_metadata_list[] = _amazons3_format_metadata(rtrim($uri, '/'), array()); + } + else { + $file_metadata_list[] = _amazons3_format_metadata($uri, $s3_metadata); + } + + // Keep track of the last key in the response, so that if we need to get + // another page of responses, we know which filename to use as the 'marker'. + $last_key = $s3_metadata['Key']; + } + + // Re-populate the file metadata table with the current page's file results. + $insert_query = db_insert('amazons3_file') + ->fields($metadata_fields); + foreach ($file_metadata_list as $metadata) { + $insert_query->values($metadata); + } + try { + $insert_query->execute(); + } + catch (PDOException $e) { + if ($e->getCode() == 23000) { + // This shouldn't ever happen!!! + // I originally coded this error correction for the case when there are two files in S3 with the same name, but + // different capitalization. By default, MySQL doesn't allow string which are case-insensitively identical, but + // I found out how to get around that (see amazons3_update_7201()). + // Just in case this does ever happen, though, the best we can do is redo each insert one at a time, catching + // and logging the individual failures. + foreach ($file_metadata_list as $metadata) { + try { + db_insert('amazons3_file') + ->fields($metadata_fields) + ->values($metadata) + ->execute(); + } + catch (PDOException $e) { + drupal_set_message(t("The file @uri has the same name as another file in S3, but with different capitalization. + If you haven't done so already, be sure to run the database update script (drush updb). + If you've already done that, something is very wrong, and you should post a ticket to the AmazonS3 issue queue.", array('@uri' => $metadata['uri'])), 'warning'); + } + } + } + else { + // Other exceptions are unexpected, and should be percolated as normal. + throw $e; + } + } + + // Now add the folders, which we need to use db_merge for because they might + // still exist from a previous cache refresh. + foreach ($folder_metadata_list as $metadata) { + db_merge('amazons3_file') + ->key(array('uri' => $metadata['uri'])) + ->fields($metadata) + ->execute(); + } + } while ($response->body->IsTruncated->to_string() == 'true'); + + // Rebuild the list of directories by looping through all the file URIs to + // to figure out what their parent directories are. + $uris = db_query('SELECT uri FROM {amazons3_file} WHERE dir = 0')->fetchAll(PDO::FETCH_COLUMN, 0); + $folders = array(); + foreach ($uris as $uri) { + // Record each file's parent directory name, unless it's the root directory. + $dirname = drupal_dirname($uri); + if ($dirname && $dirname != 's3://') { + $folders[$dirname] = $dirname; + } } - if(isset($style->name) && strlen($style->name) > 0) { - drupal_rmdir('s3://styles/' . $style->name); + $stream = new AmazonS3StreamWrapper(); + foreach ($folders as $folder) { + $stream->mkdir($folder, NULL, STREAM_MKDIR_RECURSIVE); } + drupal_set_message(t('AmazonS3 cache refreshed.')); } + /** - * Submit callback; clear file metadata cache. + * Convert file metadata returned from S3 into an array appropriate + * for insertion into our file metadata cache. + * + * @param $uri + * A string containing the uri of the resource to check. + * @param $s3_metadata + * An array containing the collective metadata for the Amazon S3 object. + * The caller may send an empty array here to indicate that the returned + * metadata should represent a folder. * + * @return + * An array containing metadata formatted for the file metadata cache. */ -function amazons3_clear_cache_submit($form, &$form_state) { - db_query('TRUNCATE TABLE {amazons3_file}'); - drupal_set_message(t('Cache cleared.')); +function _amazons3_format_metadata($uri, $s3_metadata) { + $metadata = array('uri' => $uri); + + if (empty($s3_metadata)) { + // The caller wants directory metadata, so invent some. + $metadata['dir'] = 1; + $metadata['mode'] = 0040000; // S_IFDIR indicating directory + $metadata['filesize'] = 0; + $metadata['timestamp'] = time(); + $metadata['uid'] = 'AmazonS3'; + } + else { + // The caller sent us some actual metadata, so this must be a file. + if (isset($s3_metadata['Size'])) { + $metadata['filesize'] = $s3_metadata['Size']; + } + if (isset($s3_metadata['LastModified'])) { + $metadata['timestamp'] = date('U', strtotime((string)$s3_metadata['LastModified'])); + } + if (isset($s3_metadata['Owner']['ID'])) { + $metadata['uid'] = (string)$s3_metadata['Owner']['ID']; + } + $metadata['dir'] = 0; + $metadata['mode'] = 0100000; // S_IFREG indicating file + } + $metadata['mode'] |= 0777; // everything is writeable + return $metadata; } +/** + * Converts objects returned by AmazonS3::get_objects() into s3 metadata arrays + * compatible with those returned by AmazonS3::get_object_metadata(); + */ +function _amazons3_s3_object_to_s3_metadata($object) { + // This is a sloppy but effective way to do a deep conversion of an object + // into a multi-dimentional array, found here: + // http://stackoverflow.com/a/2476954/464318 + $s3_metadata = json_decode(json_encode($object), true); + return $s3_metadata; +} Only in amazons3 2.1: amazons3_tests.php