Crawling a website which requires credentials to login

/* This script is for building cache. It just crawls the required nodes
*/

include_once './includes/bootstrap.inc';
include_once './includes/common.inc';
drupal_bootstrap(DRUPAL_BOOTSTRAP_FULL);
$i = 0;
$j = 0;
$base_url='http://my.websiteaddress.com/';

if (user_access('administer nodes')) {
//opening handler for file which will have all urls
$fh_csvfilename=fopen('/tmp/node.txt','w');

//SQL to get only required nodes.
//This SQL is for my environment. Just change it as per your requirement.
//
$sqltext="SELECT CONCAT(c.nid,',',"."'".$base_url."', 'node/', c.nid) url FROM node_counter c, node n where c.totalcount > 10 and c.nid = n.nid";
$sql=db_query($sqltext);

//Fetch all records of SQL statement
while ($n = db_fetch_object($sql)) {
$j++;
$url=$n->url;
fwrite($fh_csvfilename,$url."\n");
}
fclose($fh_csvfilename);
}

// call download_data_from_url function which will read all URLs from /tmp/node.txt and
// run curl multihandle to get data for all URLs.
download_data_from_url('/tmp/node.txt');

// printing number of nodes processed
print $j .' nodes.';
print "\n";

function download_data_from_url($csvfilenamefull) {
$allurls=file($csvfilenamefull);
$counturls=count($allurls);
$crl = curl_init();
$login_url = "http://my.websiteaddress.com/user/login";
$cookie = "/tmp/cookie.txt";
// Remove the cookie, because we can't reuse old cookie files.
unlink($cookie);

curl_setopt($crl, CURLOPT_URL, $login_url);
curl_setopt($crl, CURLOPT_COOKIEFILE, $cookie);
curl_setopt($crl, CURLOPT_COOKIEJAR, $cookie);
curl_setopt($crl, CURLOPT_FOLLOWLOCATION, 1);
curl_setopt($crl, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($crl, CURLOPT_POST, 1);

// This array will hold the field names and values.
$postdata=array(
"name"=>"myusername",
"pass"=>"mypassword",
"form_id"=>"user_login",
"op"=>"Log in"
);
// Tell curl we're going to send $postdata as the POST data
curl_setopt ($crl, CURLOPT_POSTFIELDS, $postdata);
$result=curl_exec($crl);
$headers = curl_getinfo($crl);
if ($headers['url'] == $url) {
die("Cannot login.");
}
else {
echo "We are in.";
echo "\n";
}

//node data will be saved in this location.
$save_to="/tmp/sk/";

$k=0;

foreach ($allurls as $i => $url1)
{
$array_url=explode(",",$url1);
$filename=$array_url[0].'.html';
$url=$array_url[1];
$g=$save_to.$filename;
$fp=fopen ($g, "w");

curl_setopt ($crl, CURLOPT_FILE, $fp);
curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT,60);
curl_setopt ($crl, CURLOPT_URL, $url);
$result=curl_exec($crl);
$headers = curl_getinfo($crl);
fclose ($fp);
$k++;
echo $k.".".$url;
}
curl_close($crl);
}