Index: xmlsitemap.install =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/xmlsitemap/xmlsitemap.install,v retrieving revision 1.1.2.3 diff -u -p -r1.1.2.3 xmlsitemap.install --- xmlsitemap.install 11 Dec 2007 22:57:38 -0000 1.1.2.3 +++ xmlsitemap.install 18 May 2008 11:16:07 -0000 @@ -16,6 +16,16 @@ function xmlsitemap_install() { priority float, PRIMARY KEY (path) ) /*!40100 DEFAULT CHARACTER SET utf8 */;"); + db_query("CREATE TABLE {xmlsitemap_links} ( + nid int unsigned NOT NULL default '0', + tid int unsigned NOT NULL default '0', + uid int unsigned NOT NULL default '0', + loc varchar(160) NOT NULL default '', + lastmod int(11) unsigned NOT NULL default '0', + changefreq int(11) unsigned NOT NULL default '0', + priority float NOT NULL default '0.0', + PRIMARY KEY (nid,tid,uid) + ) /*!40100 DEFAULT CHARACTER SET utf8 */;"); break; case 'pgsql': db_query("CREATE TABLE {xmlsitemap_additional} ( @@ -26,6 +36,16 @@ function xmlsitemap_install() { priority real, PRIMARY KEY (path) );"); + db_query("CREATE TABLE {xmlsitemap_links} ( + nid integer unsigned NOT NULL default '0', + tid integer unsigned NOT NULL default '0', + uid integer unsigned NOT NULL default '0', + loc varchar(160) NOT NULL default '', + lastmod integer unsigned NOT NULL default '0', + changefreq integer unsigned NOT NULL default '0', + priority real NOT NULL default '0.0', + PRIMARY KEY (nid,tid,uid) + ) /*!40100 DEFAULT CHARACTER SET utf8 */;"); break; } db_query("DELETE FROM {url_alias} WHERE dst LIKE 'sitemap%.xml'"); @@ -124,3 +144,38 @@ function xmlsitemap_uninstall() { } } +/** + * Implementation of hook_update_N(). + * Create table to collects all links. + */ +function xmlsitemap_update_1() { + $ret = array(); + switch ($GLOBALS['db_type']) { + case 'mysqli': + case 'mysql': + $ret[] = update_sql("CREATE TABLE {xmlsitemap_links} ( + nid int unsigned NOT NULL default '0', + tid int unsigned NOT NULL default '0', + uid int unsigned NOT NULL default '0', + loc varchar(160) NOT NULL default '', + lastmod int(11) unsigned NOT NULL default '0', + changefreq int(11) unsigned NOT NULL default '0', + priority float NOT NULL default '0.0', + PRIMARY KEY (nid,tid,uid) + ) /*!40100 DEFAULT CHARACTER SET utf8 */;"); + break; + case 'pgsql': + $ret[] = update_sql("CREATE TABLE {xmlsitemap_links} ( + nid integer unsigned NOT NULL default '0', + tid integer unsigned NOT NULL default '0', + uid integer unsigned NOT NULL default '0', + loc varchar(160) NOT NULL default '', + lastmod integer unsigned NOT NULL default '0', + changefreq integer unsigned NOT NULL default '0', + priority real NOT NULL default '0.0', + PRIMARY KEY (nid,tid,uid) + ) /*!40100 DEFAULT CHARACTER SET utf8 */;"); + break; + } + return $ret; +} Index: xmlsitemap.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/xmlsitemap/xmlsitemap.module,v retrieving revision 1.1.2.17 diff -u -p -r1.1.2.17 xmlsitemap.module --- xmlsitemap.module 15 May 2008 04:06:27 -0000 1.1.2.17 +++ xmlsitemap.module 18 May 2008 11:16:07 -0000 @@ -376,11 +376,23 @@ function _xmlsitemap_output($chunk = NUL */ function _xmlsitemap_link_count() { static $count; - $count = isset($count) ? $count : count(_xmlsitemap_links()); + if (_xmlsitemap_links()) { + $count = isset($count) ? $count : _xmlsitemap_link_count_run(); + } + else { + $count = 0; + } return $count; } /** + * Count the links in the database table + */ +function _xmlsitemap_link_count_run() { + return db_result(db_query('SELECT COUNT(*) FROM {xmlsitemap_links}')); +} + +/** * Update the cached site map files. * @return TRUE if the update was successful, FALSE otherwise. */ @@ -445,7 +457,7 @@ function _xmlsitemap_output_index($link_ for ($chunk = 0; $chunk < $link_count / $chunk_size; ++$chunk) { $output .= ''. xmlsitemap_url("sitemap$chunk.xml", NULL, NULL, NULL, TRUE) .''; $previous = $chunk * $chunk_size; - $links = array_slice(_xmlsitemap_links(), $previous, $chunk_size); + $links = _xmlsitemap_array_slice(_xmlsitemap_links(), $previous, $chunk_size); $output .= ''. gmdate('Y-m-d\TH:i:s+00:00', array_reduce($links, '_xmlsitemap_chunk_last_change')) .''; $output .= "\n"; } @@ -490,7 +502,7 @@ function _xmlsitemap_output_chunk($chunk $output .= ' xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'."\n"; $chunk_size = variable_get('xmlsitemap_chunk_size', 50000); $previous = $chunk * $chunk_size; - $output .= implode("\n", _xmlsitemap_format(array_slice(_xmlsitemap_links(), $previous, $chunk_size))); + $output .= implode("\n", _xmlsitemap_format(_xmlsitemap_array_slice(_xmlsitemap_links(), $previous, $chunk_size))); $output .= "\n"; return $output; } @@ -527,14 +539,20 @@ function xmlsitemap_file_download($file) function _xmlsitemap_links() { static $links; if (!isset($links)) { - $entries = module_invoke_all('xmlsitemap_links'); - if (!empty($entries)) { - foreach ($entries as $key => $link) { - $lastmod[$key] = $link['#lastmod']; + $links = TRUE; + if (variable_get('xmlsitemap_changed', FALSE) || (_xmlsitemap_link_count_run() == 0)) { + if (variable_set('xmlsitemap_rebuilding_table', FALSE)) { + watchdog('xmlsitemap', t('Skip rebuilding link table - already in process.'), WATCHDOG_WARNING); + } + else { + variable_set('xmlsitemap_rebuilding_table', TRUE); + watchdog('xmlsitemap', t('Rebuilding link table.')); + db_query('DELETE FROM {xmlsitemap_links}'); + module_invoke_all('xmlsitemap_links'); + variable_set('xmlsitemap_changed', FALSE); + variable_set('xmlsitemap_rebuilding_table', FALSE); } - array_multisort($lastmod, $entries); } - $links = $entries; } return $links; } @@ -602,18 +620,46 @@ function xmlsitemap_xmlsitemap_links($ty $links = array(); if (!isset($type)) { global $base_url; - $links[] = array('#loc' => "$base_url/", '#changefreq' => 'always', '#priority' => variable_get('xmlsitemap_front_page_priority', 1)); + $links[] = array('#loc' => "$base_url/", '#changefreq' => 3600, '#priority' => variable_get('xmlsitemap_front_page_priority', 1)); $links = array_merge($links, _xmlsitemap_additional_links()); $links = array_merge($links, module_invoke_all('gsitemap')); $links = array_merge($links, _xmlsitemap_xml_links()); - if (!empty($links)) { - foreach ($links as $key => $link) { - $loc[$key] = $link['#loc']; - } - array_multisort($loc, $links); + _xmlsitemap_insert($links); + } + return array(); +} + +/** + * Insert some $links into the database table + */ +function _xmlsitemap_insert($links) { + $sql = 'INSERT INTO {xmlsitemap_links} (nid,tid,uid,loc,lastmod,changefreq,priority) VALUES '; + $first = TRUE; + foreach ($links as $link) { + $link['nid'] = isset($link['nid']) ? $link['nid'] : 0; + $link['tid'] = isset($link['tid']) ? $link['tid'] : 0; + $link['uid'] = isset($link['uid']) ? $link['uid'] : 0; + $link['#loc'] = isset($link['#loc']) ? $link['#loc'] : ''; + $link['#lastmod'] = isset($link['#lastmod']) ? $link['#lastmod'] : 0; + $link['#changefreq']= isset($link['#changefreq']) ? $link['#changefreq']: 0; + $link['#priority'] = isset($link['#priority']) ? $link['#priority'] : 0.1; + $sql .= '('. $link['nid'] .','. + $link['tid'] .','. + $link['uid'] .','. + $link['#loc'] .','. + $link['#lastmod'] .','. + $link['#changefreq'] .','. + $link['#priority'] .')'; + if ($first) { + $first = FALSE; + } + else { + $sql .= ','; } } - return $links; + if (!$first) { + db_query($sql); + } } /** @@ -651,6 +697,7 @@ function _xmlsitemap_additional_links() */ function _xmlsitemap_xml_links() { $links = array(); + return $links; $xml = module_invoke_all('xmlsitemap_links', 'xml'); $xml = array_merge($xml, module_invoke_all('gsitemap', 'xml')); foreach ($xml as $entries) { @@ -770,6 +817,7 @@ function _xmlsitemap_get_path_alias($pat */ function xmlsitemap_cron() { if (variable_get('xmlsitemap_cron_submit', FALSE) && variable_get('xmlsitemap_changed', FALSE)) { + set_time_limit(0); _xmlsitemap_ping(); } } @@ -820,3 +868,37 @@ function _xmlsitemap_ping() { } } +/** + * Return a number of links from the database for a chunk + */ +function _xmlsitemap_array_slice($status = TRUE, $first, $maxitems) { + $links = array(); + if (!$status) { + return $links; + } + $result = db_query_range('SELECT * FROM {xmlsitemap_links} ORDER BY lastmod', $first, $maxitems); + while ($link = db_fetch_object($result)) { + $nid = NULL; + $tid = NULL; + $uid = NULL; + if ($link->nid) { + $nid = $link->nid; + } + elseif ($link->tid) { + $tid = $link->tid; + } + elseif ($link->uid) { + $uid = $link->uid; + } + $links[] = array( + 'nid' => $nid, + 'tid' => $tid, + 'uid' => $uid, + '#loc' => $link->loc, + '#lastmod' => $link->lastmod, + '#changefreq' => $link->changefreq, + '#priority' => $link->priority, + ); + } + return $links; +} Index: xmlsitemap_node/xmlsitemap_node.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/xmlsitemap/xmlsitemap_node/xmlsitemap_node.module,v retrieving revision 1.1.2.22 diff -u -p -r1.1.2.22 xmlsitemap_node.module --- xmlsitemap_node/xmlsitemap_node.module 15 May 2008 04:06:28 -0000 1.1.2.22 +++ xmlsitemap_node/xmlsitemap_node.module 18 May 2008 11:16:07 -0000 @@ -14,15 +14,9 @@ function xmlsitemap_node_xmlsitemap_link $links = _xmlsitemap_node_links(_xmlsitemap_node_excludes()); $links = array_merge($links, module_invoke_all('xmlsitemap_links', 'node', _xmlsitemap_node_excludes())); $links = array_merge($links, module_invoke_all('gsitemap', 'node', _xmlsitemap_node_excludes())); - if (!empty($links)) { - foreach ($links as $key => $link) { - $nid[$key] = $link['nid']; - $loc[$key] = $link['#loc']; - } - array_multisort($nid, $loc, $links); - } + _xmlsitemap_insert($links); } - return $links; + return array(); } /** @@ -68,14 +62,29 @@ function _xmlsitemap_node_links($exclude AND (n.type NOT IN ('". implode("', '", $excludes) ."') AND xn.priority_override IS NULL OR xn.priority_override >= 0) AND n.nid <> %d"; $result = db_query(db_rewrite_sql($sql), _xmlsitemap_node_frontpage()); + $i = 0; + $chunk_size = variable_get('xmlsitemap_chunk_size', 50000); while ($node = db_fetch_object($result)) { - $links[] = array( - 'nid' => $node->nid, - '#loc' => xmlsitemap_url('node/'. $node->nid, $node->alias, NULL, NULL, TRUE), - '#lastmod' => variable_get('xmlsitemap_node_count_comments', TRUE) ? max($node->changed, $node->last_comment_timestamp) : $node->changed, - '#changefreq' => xmlsitemap_node_frequency($node), - '#priority' => xmlsitemap_node_priority($node), - ); + if ($i == 0) { + $sql = 'INSERT INTO {xmlsitemap_links} (nid,loc,lastmod,changefreq,priority) VALUES '; + } + else { + $sql .= ','; + } + $nid = $node->nid; + $loc = xmlsitemap_url('node/'. $node->nid, $node->alias, NULL, NULL, TRUE); + $lastmod = variable_get('xmlsitemap_node_count_comments', TRUE) ? max($node->changed, $node->last_comment_timestamp) : $node->changed; + $changefreq = xmlsitemap_node_frequency($node); + $priority = xmlsitemap_node_priority($node); + $sql .= "($nid,'$loc',$lastmod,$changefreq,$priority)"; + $i += 1; + if ($i >= $chunk_size) { + $i = 0; + db_query($sql); + } + } + if ($i > 0) { + db_query($sql); } return $links; } Index: xmlsitemap_term/xmlsitemap_term.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/xmlsitemap/xmlsitemap_term/xmlsitemap_term.module,v retrieving revision 1.1.2.13 diff -u -p -r1.1.2.13 xmlsitemap_term.module --- xmlsitemap_term/xmlsitemap_term.module 15 May 2008 04:06:28 -0000 1.1.2.13 +++ xmlsitemap_term/xmlsitemap_term.module 18 May 2008 11:16:07 -0000 @@ -14,15 +14,9 @@ function xmlsitemap_term_xmlsitemap_link $links = _xmlsitemap_term_links(_xmlsitemap_term_excludes()); $links = array_merge($links, module_invoke_all('xmlsitemap_links', 'term', _xmlsitemap_term_excludes())); $links = array_merge($links, module_invoke_all('gsitemap', 'term', _xmlsitemap_term_excludes())); - if (!empty($links)) { - foreach ($links as $key => $link) { - $tid[$key] = $link['tid']; - $loc[$key] = $link['#loc']; - } - array_multisort($tid, $loc, $links); - } + _xmlsitemap_insert($links); } - return $links; + return array(); } /** @@ -60,7 +54,15 @@ function _xmlsitemap_term_links($exclude WHERE (t.vid NOT IN (". (empty($excludes) ? 0 : implode(', ', $excludes)) .") AND xt.priority_override IS NULL OR xt.priority_override >= 0) AND t.tid <> %d ", 't', 'tid'), _xmlsitemap_term_frontpage()); + $i = 0; + $chunk_size = variable_get('xmlsitemap_chunk_size', 50000); while ($term = db_fetch_object($result)) { + if ($i == 0) { + $sql = ' INSERT INTO {xmlsitemap_links} (tid,loc,lastmod,changefreq,priority) VALUES '; + } + else { + $sql .= ','; + } if ($term->module == 'forum') { $url = xmlsitemap_url("forum/$term->tid", $term->alias, NULL, NULL, TRUE); } @@ -76,13 +78,21 @@ function _xmlsitemap_term_links($exclude } $age = time() - $term->last_changed; $interval = empty($term->previously_changed) ? 0 : $term->last_changed - $term->previously_changed; - $links[] = array( - 'tid' => $term->tid, - '#loc' => $url, - '#lastmod' => $term->last_changed, - '#changefreq' => max($age, $interval), - '#priority' => _xmlsitemap_term_priority($term), - ); + + $tid = $term->tid; + $loc = $url; + $lastmod = $term->last_changed; + $changefreq = max($age, $interval); + $priority = _xmlsitemap_term_priority($term); + $sql .= "($tid,'$loc',$lastmod,$changefreq,$priority)"; + $i += 1; + if ($i >= $chunk_size) { + $i = 0; + db_query($sql); + } + } + if ($i > 0) { + db_query($sql); } return $links; } Index: xmlsitemap_user/xmlsitemap_user.module =================================================================== RCS file: /cvs/drupal-contrib/contributions/modules/xmlsitemap/xmlsitemap_user/xmlsitemap_user.module,v retrieving revision 1.1.2.15 diff -u -p -r1.1.2.15 xmlsitemap_user.module --- xmlsitemap_user/xmlsitemap_user.module 15 May 2008 04:06:28 -0000 1.1.2.15 +++ xmlsitemap_user/xmlsitemap_user.module 18 May 2008 11:16:07 -0000 @@ -14,15 +14,9 @@ function xmlsitemap_user_xmlsitemap_link $links = _xmlsitemap_user_links(); $links = array_merge($links, module_invoke_all('xmlsitemap_links', 'user')); $links = array_merge($links, module_invoke_all('gsitemap', 'user')); - if (!empty($links)) { - foreach ($links as $key => $link) { - $uid[$key] = $link['uid']; - $loc[$key] = $link['#loc']; - } - array_multisort($uid, $loc, $links); - } + _xmlsitemap_insert($links); } - return $links; + return array(); } /** @@ -43,16 +37,32 @@ function _xmlsitemap_user_links() { GROUP BY u.uid, xu.last_changed, xu.previously_changed, xu.priority_override, alias HAVING COUNT(xu.priority_override) > 0 OR (COUNT(xur.rid) = 0 AND %f <> -1 OR MIN(xur.priority) <> -1) ", _xmlsitemap_user_frontpage(), variable_get('xmlsitemap_user_default_priority', 0.5)); + $i = 0; + $chunk_size = variable_get('xmlsitemap_chunk_size', 50000); while ($user = db_fetch_object($result)) { + if ($i == 0) { + $sql = 'INSERT INTO {xmlsitemap_links} (uid,loc,lastmod,changefreq,priority) VALUES '; + } + else { + $sql .= ','; + } $age = time() - $user->last_changed; $interval = empty($user->previously_changed) ? 0 : $user->last_changed - $user->previously_changed; - $links[] = array( - 'uid' => $user->uid, - '#loc' => xmlsitemap_url("user/$user->uid", $user->alias, NULL, NULL, TRUE), - '#lastmod' => $user->last_changed, - '#changefreq' => max($age, $interval), - '#priority' => _xmlsitemap_user_priority($user), - ); + + $uid = $user->uid; + $loc = xmlsitemap_url("user/$user->uid", $user->alias, NULL, NULL, TRUE); + $lastmod = $user->last_changed; + $changefreq = max($age, $interval); + $priority = _xmlsitemap_user_priority($user); + $sql .= "($uid,'$loc',$lastmod,$changefreq,$priority)"; + $i += 1; + if ($i >= $chunk_size) { + $i = 0; + db_query($sql); + } + } + if ($i > 0) { + db_query($sql); } return $links; }