Index: similarity.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/similarity/similarity.inc,v
retrieving revision 1.1.2.3
diff -u -p -r1.1.2.3 similarity.inc
--- similarity.inc	13 Mar 2009 21:14:23 -0000	1.1.2.3
+++ similarity.inc	23 Apr 2009 16:36:35 -0000
@@ -122,10 +122,16 @@ abstract class similarity {
       'min_sim' => array(
         '#type' => 'select',
         '#title' => t('Minimum similarity score'),
-        '#options' => drupal_map_assoc(range(0, 100, 5)),
+        '#options' => drupal_map_assoc(range(50, 100, 5)),
         '#description' => t('Only store similarities greater then this number. This means that if a node is similar to another node to the 50%, it will only be stored if this value is 50 or greater'),
         '#default_value' => isset($this->min_sim) ? $this->min_sim : 50,
       ),
+      'time_limit' => array(
+        '#type' => 'select',
+        '#title' => t('Time Limit'),
+        '#description' => t('The number of minutes allowed for processing of similarities.'),
+        '#options' => drupal_map_assoc(array(60, 60 * 2, 60 * 3, 60 * 4, 60 * 5, 60 * 6, 60 * 7, 60 * 8, 60 * 9, 60 * 10), 'format_interval')
+      ),
     );
   }
   
@@ -185,7 +191,7 @@ abstract class similarity {
       // create the objects table
       $ret = array();
       $schema = $this->schema();
-      db_create_table(&$ret, $this->machine_name . '_similarity', $schema);
+      db_create_table($ret, $this->machine_name . '_similarity', $schema);
     }
   }
     
 
@@ -217,55 +233,7 @@ abstract class similarity {
  * Class similarity_node is used as an abstract class for all node
  * similarities
  */
-abstract class similarity_node extends similarity {
-  private $pending_saves = array();
-  
-  /**
-   * save the similarity away
-   */
-  protected function save_similarity($nid1, $nid2, $similarity) {
-    static $registered = FALSE;
-    if (!$registered) {
-      register_shutdown_function(array($this, 'write_similarities'));
-      $registered = TRUE;
-    }
-    if ($nid1 > $nid2) {
-      $key1 = $nid1;
-      $key2 = $nid2;
-    }
-    else {
-      $key1 = $nid2;
-      $key2 = $nid1;
-    }
-    // put it in the pending saves only if it has a close enough similarity
-    if (!isset($this->pending_saves[$key1][$key2])) {
-      $this->pending_saves[$key1][$key2] = $similarity;
-    }
-  }
-  
-  /**
-   * Save the similarities away
-   */
-  public function write_similarities() {
-    static $written = FALSE;
-    if ($written) return;
-    
-    $value_string = '';
-    $delete_string = '';
-    foreach($this->pending_saves as $nid1 => $pair) {
-      foreach($pair as $nid2 => $sim) {
-        $value_string .= "($nid1, $nid2, $sim), ";
-      }
-    }
-    
-    if (!empty($value_string)) {
-      // write the big insert string
-      db_query("INSERT INTO {" . $this->machine_name . "_similarity} VALUES " . substr($value_string, 0, -2) . " ON DUPLICATE KEY UPDATE score = VALUES(score)");
-    }
-    $written = TRUE;
-    $this->cleanup();
-  }
-  
+abstract class similarity_node extends similarity {  
   /**
    * provide a table schema
    */
@@ -286,8 +254,17 @@ abstract class similarity_node extends s
           'type' => 'float',
           'not null' => TRUE,
         ),
+        'created' => array(
+          'descritpion' => t('Used as a timestamp from when the entry was last created'),
+          'type' => 'int',
+          'not null' => TRUE,
+        ),
       ),
       'primary key' => array('nid1', 'nid2'),
+      'indexes' => array(
+        'id_1' => array('nid1'),
+        'id_2' => array('nid2'),
+      ),
     );
     
     return $schema;
@@ -305,8 +282,8 @@ abstract class similarity_node extends s
     $form['node_limit'] = array(
       '#type' => 'select',
       '#title' => t('Node Throttle'),
-      '#options' => drupal_map_assoc(range(50, 500, 50)),
-      '#description' => t('Select the number of node similarities to calculate per cron run'),
+      '#options' => drupal_map_assoc(range(50, 400, 25)),
+      '#description' => t('Select the number of node similarities to calculate per cron run. The higher the throttle the longer your cron process will take. If you notice your cron process taking too long, lower this number and consider changing the node types'),
       '#default_value' => isset($this->node_limit) ? $this->node_limit : 100,
     );
     $form['types'] = array(
@@ -369,6 +346,29 @@ abstract class similarity_node extends s
 
     return $data;
   }
+  
+  /**
+   * return a type string for the SQL
+   */
+  protected function node_type_string() {
+    $type_string = '';
+    
+    // yuck, can't enforce checkboxes as required. Thus...
+    if (empty($this->types)) {
+      return $type_string;
+    }
+    
+    foreach($this->types as $node_type) {
+      if (empty($type_string)) {
+        $type_string = "n.type = '%s'";
+      }
+      else {
+        $type_string .= " OR n.type = '%s'";
+      }
+    }
+    $type_string = "(" . $type_string . ")";
+    return $type_string;
+  }
 }
 
 // util functions
Index: classes/similarity_cosine_vote.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/similarity/classes/Attic/similarity_cosine_vote.inc,v
retrieving revision 1.1.2.3
diff -u -p -r1.1.2.3 similarity_cosine_vote.inc
--- classes/similarity_cosine_vote.inc	13 Mar 2009 21:14:23 -0000	1.1.2.3
+++ classes/similarity_cosine_vote.inc	23 Apr 2009 16:36:35 -0000
@@ -46,52 +46,60 @@ class similarity_cosine_vote extends sim
     // TODO: implement with better limits
     foreach($this->types as $node_type) {
       if (empty($type_string)) {
-        $type_string = "type = '%s'";
+        $type_string = "n.type = '%s'";
       }
       else {
-        $type_string .= " OR type = '%s'";
+        $type_string .= " OR n.type = '%s'";
       }
     }
     $type_string = "(" . $type_string . ")";
-    $db_nids_query = "SELECT nid FROM {node} WHERE " . $type_string;
-    
+    $db_nids_query = "SELECT n.nid, SQRT(SUM(POW(v.value / 100, 2))) as mag FROM {node} n INNER JOIN {votingapi_vote} vote ON vote.content_id = n.nid AND vote.type = 'node' WHERE n.nid > %d AND $type_string GROUP BY n.nid ORDER BY v.content_id LIMIT 0, %d";
     
     $last_nid = variable_get($this->machine_name . '_cron_last_nid', 0);
     $limit = $this->node_limit; // very basic right now
     $args = array($last_nid);
-    if (!empty($this->vote_tag)) {
+    
+    // TODO: tag handling
+    /*if (!empty($this->vote_tag)) {
       $args[] = $this->vote_tag;
-    }
-    $args = array_merge($args, $this->types);
+    }*/
     
-    // build the query
-    $sql = "SELECT v.content_id as nid, v.value / 100 as value, v.uid";
-    $sql .= " FROM {votingapi_vote} v";
-    $sql .= " WHERE v.content_id > %d";
-    if (!empty($this->vote_tag)) {
-      $sql .= " AND v.tag = '%s'";
-    }
-    $sql .= " AND v.content_id IN ($db_nids_query) AND v.content_type = 'node'";
-    $sql .= " ORDER BY nid ASC";
-    $result = db_query_range($sql, $args, 0, $limit);
-    
-    // build up a cache so we can cosine it
-    while ($vote_obj = db_fetch_object($result)) {
-      $this->vote_cache[$vote_obj->nid][$vote_obj->uid] = $vote_obj->value;
-    }
+    // merge all the args into the query
+    $args = array_merge($args, $this->types, $limit);
     
-    // collect the votes and cosine it!
-    $nids = array_keys($this->vote_cache);
-    for ($i = 0; $i < count($nids); $i++) {
-      for ($j = 0; $j < count($nids); $j++) {
-        if ($i != $j) {
-          $this->save_similarity($nids[$i], $nids[$j], similarity_cosine($this->vote_cache[$nids[$i]], $this->vote_cache[$nids[$j]]));
-        }
-      }
-      $last_nid = $nids[$i];
+    // create the meta data temp table. This table serves
+    // as a limit and for the init magintude calc for the processed nodes
+    db_query_temporary($db_nids_query, $args, 'meta_data');
+    db_query("ALTER TABLE {meta_data} ADD PRIMARY KEY (nid)");
+    
+    // create the temp table for the rest of the magintudes
+    db_query_temporary("SELECT v.content_id as nid, SQRT(SUM(POW(v.value / 100, 2))) as mag FROM {votingapi_vote} v WHERE v.type = 'node' GROUP BY v.content_id", 'mag_table');
+    db_query("ALTER TABLE {mag_table} ADD PRIMARY KEY(nid)");
+    
+    // this is to remove the stale data. It removes all the data clearing the way for the inseration
+    // With the indexes on both nid1 and nid2 doing two queries signifcantly faster then an OR
+    // Investigate a UNION? In MySQL 5.0 OR OR can be translated to UNION by query parser when it would speed it up...
+    db_query("DELETE s FROM {" . $this->machine_name . "_similarity} s INNER JOIN {meta_data} m ON m.nid = s.nid1");
+    db_query("DELETE s FROM {" . $this->machine_name . "_similarity} s INNER JOIN {meta_data} m ON m.nid = s.nid2");
+    
+    // It is actually faster to go nid by nid as then the meta data table will
+    // use the PRIMARY KEY. Strange though.. consider reworking
+    $nid_result = db_query("SELECT nid FROM {meta_data}");
+    while ($nid_object = db_fetch_object($nid_result)) {
+      db_query("INSERT INTO {". $this->machine_name . "_similarity} " .
+        // get the largest of the two nids and put it as nid1
+        "SELECT IF (v1.content_id > v2.content_id, v1.content_id, v2.content_id) as nid1, IF (v1.content_id > v2.content_id, v2.content_id, v1.content_id) as nid2, ".
+        // calculate the score here and put the $created_time in the SELECT fields as a const
+        "(SUM((v1.value / 100) * (v2.value / 100)) / (md.mag * m2.mag)) as sim, $created_time " .
+        // JOIN two vote tables on the same content_id but not the same uids.
+        "FROM {meta_data} md INNER JOIN {votingapi_vote} v1 ON v1.content_id = md.nid AND v1.type = 'node' INNER JOIN {votingapi_vote} v2 ON v2.content_id = v1.content_id AND v2.type = 'node' AND v2.content_id <> v1.content_id " .
+        "INNER JOIN {mag_table} m2 ON m2.nid = v2.content_id " .
+        // use the primary key for the first table and group the result set
+        "WHERE md.nid = %d AND s1.type = 'node' GROUP BY v1.content_id, v2.content_id HAVING sim > %f " .
+        // ON DUPLICATE KEY update the score value with the new score
+        "ON DUPLICATE KEY UPDATE score=VALUES(score)", $nid_object->nid, ($this->min_sim / 100));
     }
     
-    $this->write_similarities();
     // check if it had cycled around
     if ($last_nid == db_result(db_query("SELECT MAX(content_id) FROM {votingapi_vote}"))) {
       $last_nid = 0;
Index: classes/similarity_search_similarity.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/similarity/classes/Attic/similarity_search_similarity.inc,v
retrieving revision 1.1.2.3
diff -u -p -r1.1.2.3 similarity_search_similarity.inc
--- classes/similarity_search_similarity.inc	13 Mar 2009 21:14:23 -0000	1.1.2.3
+++ classes/similarity_search_similarity.inc	23 Apr 2009 16:36:35 -0000
@@ -11,83 +11,74 @@
  * node similarity based on the search index
  */
 class similarity_search_similarity extends similarity_node {
-  
-  // keyed array of nid -> words
-  // acts as a static cache
-  private $word_cache = array();
-  
-  
   /**
    * Implementation of the calculate function
    * cosine similarity of the search index
    */
   public function calculate() {
-    // TODO: shutdown functions and globals
-    
     // get the possible nids based on limits
     // TODO: implement with better limits
-    $possible_nids = array();
+    $created_time = time();
+    
+    // yuck, can't enforce checkboxes as required. Thus...
+    if (empty($this->types)) {
+      return;
+    }
     foreach($this->types as $node_type) {
       if (empty($type_string)) {
-        $type_string = "type = '%s'";
+        $type_string = "n.type = '%s'";
       }
       else {
-        $type_string .= " OR type = '%s'";
+        $type_string .= " OR n.type = '%s'";
       }
     }
     $type_string = "(" . $type_string . ")";
-    $db_nids = db_query("SELECT nid FROM {node} WHERE " . $type_string, $this->types);
-    while($nid_obj = db_fetch_object($db_nids)) {
-      $possible_nids[] = $nid_obj->nid;
-    }
     
     // emulate node_update_index()
     $last = variable_get($this->machine_name . '_cron_last', 0);
-    // only delete when we have run this before
-    $delete_first = ($last != 0);
-    $last_nid = variable_get($this->machine_name . '_cron_last_nid', 0);
+    
     $limit = $this->node_limit; // very basic right now
-    $args = array($last, $last_nid, $last, $last);
+    $args = array($last, $last, $last);
     $args = array_merge($args, $this->types);
-    $result = db_query_range('SELECT GREATEST(IF(c.last_comment_timestamp IS NULL, 0, c.last_comment_timestamp), n.changed) as last_change, n.nid, n.created FROM {node} n LEFT JOIN {node_comment_statistics} c ON n.nid = c.nid WHERE n.status = 1 AND ((GREATEST(n.changed, c.last_comment_timestamp) = %d AND n.nid > %d) OR (n.changed > %d OR c.last_comment_timestamp > %d)) AND ' . $type_string . ' ORDER BY GREATEST(n.changed, c.last_comment_timestamp) ASC, n.nid ASC', $args, 0, $limit);
+    $args[] = $limit;
+    $temp_query = "SELECT GREATEST(IF(c.last_comment_timestamp IS NULL, 0, c.last_comment_timestamp), n.changed) as last_change, n.nid, n.created, SQRT(SUM(POW(s.score,2))) as mag FROM {node} n LEFT JOIN {node_comment_statistics} c ON n.nid = c.nid INNER JOIN {search_index} s ON s.type = 'node' AND s.sid = n.nid WHERE n.status = 1 AND ((GREATEST(n.changed, c.last_comment_timestamp) = %d) OR (n.changed > %d OR c.last_comment_timestamp > %d)) AND " . $type_string . " GROUP BY n.nid ORDER BY GREATEST(n.changed, c.last_comment_timestamp) ASC, n.nid ASC";
     
-    $value_string = '';
-    $sim_cache = array();
-    // cosine of the search index for the node against all other nodes
-    // TODO: check limit on each passing changed_obj
-    while ($changed_obj = db_fetch_object($result)) {
-      $current_words = $this->get_words($changed_obj->nid);
-      $last_changed = $changed_obj->last_change;
-      $last_nid = $changed_obj->nid;
-      
-      // check against the possible nids for similarities
-      foreach($possible_nids as $other_nid) {
-        if ($other_nid != $changed_obj->nid) {
-          $others_words = $this->get_words($other_nid);
-          $this->save_similarity($changed_obj->nid, $other_nid, similarity_cosine($current_words, $others_words));
-        }
-      }
-    }
-    $this->write_similarities();
-    if ($last_changed > 0 ) {
-      variable_set($this->machine_name . '_cron_last', $last_changed);
-      variable_set($this->machine_name . '_cron_last_nid', $last_nid);
-    }
-  }
-  
-  /**
-   * fetches the words for a given nid
-   * prevents hitting the db a crazy amount, only once per nid
-   */
-  protected function get_words($nid) {    
-    if (!isset($this->word_cache[$nid])) {
-      // now we have to query for the words
-      $db_words = db_query("SELECT word, score FROM {search_index} WHERE sid = %d AND type = 'node'", $nid);
-      while($word = db_fetch_object($db_words)) {
-        $this->word_cache[$nid][$word->word] = $word->score;
-      }
-    }
+    // create the meta data table for the basic information
+    db_query_temporary($temp_query . ' LIMIT 0, %d', $args, 'meta_data');
+    db_query("ALTER TABLE {meta_data} ADD PRIMARY KEY(nid)");
     
-    return $this->word_cache[$nid];
+    // create the table that holds the magnitude information for each nid
+    db_query_temporary("SELECT sid as nid, SQRT(SUM(POW(score,2))) as mag FROM {search_index} WHERE type = 'node' GROUP BY sid", "local_mag");
+    
+    // add in primary keys so its faster
+    db_query("ALTER TABLE local_mag ADD PRIMARY KEY(nid)");
+    
+    // this is to remove the stale data. It removes all the data clearing the way for the inseration
+    // With the indexes on both nid1 and nid2 doing two queries signifcantly faster then an OR
+    // Investigate a UNION? In MySQL 5.0 OR OR can be translated to UNION by query parser when it would speed it up...
+    db_query("DELETE s FROM {" . $this->machine_name . "_similarity} s INNER JOIN {meta_data} m ON m.nid = s.nid1");
+    db_query("DELETE s FROM {" . $this->machine_name . "_similarity} s INNER JOIN {meta_data} m ON m.nid = s.nid2");
+    
+    // It is actually faster to go nid by nid as then the meta data table will
+    // use the PRIMARY KEY. Strange though.. consider reworking
+    $nid_result = db_query("SELECT nid FROM {meta_data}");
+    while ($nid_object = db_fetch_object($nid_result)) {
+      // the big nasty insert query
+      db_query("INSERT INTO {". $this->machine_name . "_similarity} " .
+        // get the largest of the two nids and put it as nid1
+        "SELECT IF (s1.sid > s2.sid, s1.sid, s2.sid) as nid1, IF (s1.sid > s2.sid, s2.sid, s1.sid) as nid2, ".
+        // calculate the score here and put the $created_time in the SELECT fields as a const
+        "(SUM(s1.score * s2.score) / (md.mag * m2.mag)) as sim, $created_time " .
+        // JOIN two search tables on the same words but not the same ids.
+        "FROM {meta_data} md INNER JOIN {search_index} s1 ON s1.sid = md.nid AND s1.type = 'node' INNER JOIN {search_index} s2 ON s2.word = s1.word AND s1.type = 'node' AND s2.sid <> s1.sid " .
+        "INNER JOIN {local_mag} m2 ON m2.nid = s2.sid " .
+        // use the primary key for the first table and group the result set
+        "WHERE md.nid = %d AND s1.type = 'node' GROUP BY s1.sid, s2.sid HAVING sim > %f " .
+        // ON DUPLICATE KEY update the score value with the new score
+        "ON DUPLICATE KEY UPDATE score=VALUES(score)", $nid_object->nid, ($this->min_sim / 100));
+    }
+    // save the last changed away
+    $changed = db_result(db_query("SELECT MAX(last_change) FROM {meta_data}"));
+    variable_set($this->machine_name . '_cron_last', $changed);
   }
-}
\ No newline at end of file
+}
Index: classes/similarity_taxonomy.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/similarity/classes/Attic/similarity_taxonomy.inc,v
retrieving revision 1.1.2.3
diff -u -p -r1.1.2.3 similarity_taxonomy.inc
--- classes/similarity_taxonomy.inc	13 Mar 2009 21:14:23 -0000	1.1.2.3
+++ classes/similarity_taxonomy.inc	23 Apr 2009 16:36:35 -0000
@@ -17,26 +17,13 @@ class similarity_taxonomy extends simila
   function calculate() {
     // get the possible nids based on limits
     // TODO: implement with better limits
-    $possible_nids = array();
-    $type_string = '';
+    $created_time = time();
     $args = array();
-    $sim_cache = array();
-    foreach($this->types as $node_type) {
-      if (empty($type_string)) {
-        $type_string = "n.type = '%s'";
-      }
-      else {
-        $type_string .= " OR n.type = '%s'";
-      }
-    }
-    $type_string = "(" . $type_string . ")";
-    $db_nids = db_query("SELECT n.nid, n.vid FROM {node} n WHERE " . $type_string, $this->types);
-    while($nid_obj = db_fetch_object($db_nids)) {
-      $terms = db_query("SELECT d.name FROM {term_data} d JOIN {term_node} tn ON tn.tid = d.tid WHERE tn.nid = %d", $nid_obj->nid);
-      while($term_data_row = db_fetch_object($terms)) {
-        // TODO: can we use term_data->weight?
-        $possible_nids[$nid_obj->nid][strtolower($term_data_row->name)] = 1;
-      }
+    $type_string = $this->node_type_string();
+    
+    // can't enforce checkboxes....
+    if (empty($type_string)) {
+      return;
     }
     
     // set limits
@@ -46,27 +33,34 @@ class similarity_taxonomy extends simila
 
     $args[] = $last;
     $args = array_merge($args, $this->types);
-    $result = db_query_range('SELECT n.nid, n.changed FROM {node} n WHERE n.changed > %d AND ' . $type_string . ' ORDER BY changed ASC', $args, 0, $limit);
-
+    
+    db_query_temporary("SELECT n.nid, n.changed, SQRT(COUNT(t.tid)) as mag FROM {node} n INNER JOIN {term_node} t ON t.nid = n.nid WHERE n.changed > %d AND " . $type_string . " ORDER BY changed ASC LIMIT 0, $limit", $args, "meta_data");
+    db_query("ALTER TABLE {meta_data} ADD PRIMARY KEY(nid)");
     $last_changed = 0;
-   
+    
+    db_query_temporary("SELECT nid, SQRT(COUNT(tid)) as mag FROM {term_node} GROUP BY nid", 'local_mag');
+    
+    // add in the primary key. makes the rest of the queries much faster
+    db_query("ALTER TABLE {local_mag} ADD PRIMARY KEY(nid)");
+    
+    // Clear out stale records
+    db_query("DELETE s FROM {" . $this->machine_name ."_similarity} s INNER JOIN {meta_data} m ON m.nid = s.nid1");
+    db_query("DELETE s FROM {" . $this->machine_name ."_similarity} s INNER JOIN {meta_data} m ON m.nid = s.nid2");
+    
+    $result = db_query("SELECT nid, changed FROM {meta_data}");
     while ($changed_obj = db_fetch_object($result)) {
-      $current_terms = $possible_nids[$changed_obj->nid];
-      $last_changed = $changed_obj->changed;
-      
-      // process the terms
-      if (!empty($current_terms)) {
-        foreach($possible_nids as $other_nid => $other_terms) {
-          if ($other_nid != $changed_obj->nid && !empty($other_terms)) {
-            $this->save_similarity($changed_obj->nid, $other_nid, similarity_cosine($current_terms, $other_terms));
-          }
-        }
-      }
-      // then insert
-      $this->write_similarities();
-      if ($last_changed > 0 ) {
-        variable_set($this->machine_name . '_cron_last', $last_changed);
+      // the big insert query
+      db_query("INSERT INTO {". $this->machine_name . "_similarity} ".
+               "SELECT IF(t1.nid > t2.nid, t1.nid, t2.nid) as nid1, IF(t1.nid > t2.nid, t2.nid, t1.nid) as nid2, COUNT(t2.nid) / (md.mag * m2.mag) as sim " .
+               "FROM {meta_data} md INNER JOIN {term_node} t1 ON t1.nid = md.nid INNER JOIN {term_node} t2 ON t2.tid = t1.tid AND t2.nid <> t1.nid WHERE t1.nid = %d GROUP BY nid1, nid2 HAVING sim > %f " .
+               "ON DUPLICATE KEY UPDATE score=VALUES(score)", $changed_obj->nid, ($this->min_sim / 100));
+
+      if ($changed_obj->changed > $last_changed) {
+        $last_changed = $changed_obj->changed;
       }
     }
+    
+    // save the last changed away for next calculation
+    variable_set($this->machine_name . '_cron_last', $last_changed);
   }
-}
\ No newline at end of file
+}