[mirrors/Programs.git] / php / crawler / crawler.php

#!/usr/bin/php
<?php
//HSp33der Web Crawler 5.6
//<-Harvie 2oo7/8
/*
 * Description:
 * This script crawls the web and printing found URLs to STDOUT.
 *
 * Installation:
 * - Debian:
 *   # apt-get install php5-cli php5-curl curl sort
 *   $ chmod +x crawler.php
 *   $ ./crawler.php (or $ php crawler.php)
 * - Windows:
 *   c:/>path/to/php5/php.exe crawler.php
 *   (you can add php.exe to PATH or associate .php files with it)
 *
 * Notes:
 * - You can use EtherApe (or similar sniffer) to get graphical
 * illustration of connecions (spider with your machine in center).
 * - There are few bugs in PHP5,
 * so this crawler may freeze after few minutes,
 * but you can use $use_curlbin option (need some extra files).
 * Check this for more info: http://bugs.php.net/bug.php?id=43098
 *
 * Tips & Tricks:
 * - Make found URLs 100% unique:
 * cat urls.txt | sort -u > uurls.txt && mv -f uurls.txt urls.txt
 * - Show filesize and total urls count:
 * du -h urls.txt && cat urls.txt | wc -l
 * - Open another output pipe:
 * tail -f urls.txt
 * - Use some plugin (filter) to extract emails, etc...
 * tail -f urls.txt | ./example_mail_extractor.php
 *
 * Special thx2:
 * - PHP, cURL (technology)
 * - root.cz & php.net (help)
 * - Linus Torvalds (because I am crawling from Linux)
 */

///SETTINGS//////////////////
//Basic
$seed = 'http://blog.harvie.cz/'; //Start crawling from this page
$file = 'urls.txt'; //File to save crawled URLs (use sort to make URLs 100% exclusive)
$delete_url_db = false; //Delete file with saved URLs before crawling
$restore = true; //Use last url from $file instead of seed
//Advanced
$max_size = 10000; //How many bytes download from each page?
$max_urls = 30; //Size of URL buffer (prebuffered URLs to crawl, if full, no more URLs will be added to queue)
$buffer_increase = true; //Increase buffer at buffer underrun?
$random_url = false; //Select random urls from page? (This may override URL filter)
$history_max = 1000; //How many last URLs keep in history? (historied URLs will not be crawled again...)
//Curl binary plugin
$use_curlbin = true; //Use interface to cURL executable instead of file_get_contents() (More speed, less crashes)
$curlpath = 'curl'; //Path to cURL executable binary (you can download it from http://curl.haxx.se/)
//Sort binary plugin
$use_sortbin = true; //Use sort to make URLs unique?
$sortpath = 'sort'; //Path to SORT executable binary (get it from your OS or UNXUtils for Windows)
$sort_each = $history_max; //Sort file after X new URLs
//System/PHP
$socket_timeout = 1; //How long you want wait for a webserver? (seconds)
$time_limit = 0; //How long run? (seconds; 0==infinite)
$memory_limit = '128M'; //Maximum memory to eat (if exhausted, crawler will stop)
//Debug
$debug = true; //Use debuging mode? (Print errors and statistics)

//Filter
$eregi_url_blacklist = array( //Most important thing when crawling ;D
    '(W3\.org|W3C)', //Hell knows...
    '(shop|xxx|porn|lesbian|hot)', //Commercial sites
    '(google|209.85.135.104|yahoo.com|amazon.com|youtube.com)', //Big sites
    '(seznam.cz|centrum.cz|atlas.cz|zoznam.sk|quick.cz)', //Big local sites
    '.\.(css|ico|gif|jpg|png|bmp|cgi|js|vbs)', //Misc. webpage content
    '.\.(avi|mpg|mpeg|mov|wmv|wm|mp3|ogg|wma)', //Multimedia files
    '.\.(pdf|swf|flv|cfm)', //Other text files
    '.\.(exe|zip|rar|gz|bz|bz2|tar)' //Big and binary files
);


///FUNCTIONS/////////////////
function shell_sort_file($file, $tmpfile='.tmp') {
    $tmpfile = $file.$tmpfile;
    if($GLOBALS['debug']) echo("-SORTing $file (using tempfile $tmpfile)\n");
    @unlink($tmpfile);
    system($GLOBALS['sortpath']." -u $file > $tmpfile");
    if(is_file($tmpfile) && (filesize($tmpfile) > 3)) {
        unlink($file);
        rename($tmpfile, $file);
    } else {
        if($GLOBALS['debug']) echo("-SORTing failed!!!\n");
    }
     if($GLOBALS['debug']) echo("-SORTing done!\n");    
}

function shell_curl_get($url, $timeout=0, $maxdata=0) { //I made this cURL interface cause file_get_contents() absolutely sux
    $curl = 'curl';
    if(isset($GLOBALS['curlpath'])) $curl = $GLOBALS['curlpath'];
    $args = "--get --location --connect-timeout $timeout --max-time $timeout --compressed --stderr curl.debug";
    $url = escapeshellcmd(trim($url));
    $cmd = "$curl --url \"$url\" $args";
    if($GLOBALS['debug']) echo("-Ex: $cmd\n");
    $fp = popen($cmd, 'r');
    stream_set_timeout($fp, $timeout);
    stream_set_blocking($fp, 0);
    //if($timeout>0) sleep($timeout);
    $data = '';
    while(!feof($fp)) {
        $in = fgets($fp, 100);
        $data .= $in;
        if($maxdata>0 && strlen($data)>=$maxdata) break;
        if($in == '') usleep(1000000/8);
    }
    @fclose($fp);
    return $data;
}

function check_crawl_url($url) { //Use this function to determine if url is wanted
    foreach($GLOBALS['eregi_url_blacklist'] as $black_url) {
        if(eregi($black_url, $url)) return(0);
    }
    if(in_array($url, $GLOBALS['urls'])) return(0);
    $file_url = explode('?', $url, 2); $file_url = $file_url[0];
    if(in_array($file_url, $GLOBALS['history'])) return(0);
    return(1);
}


///CODE//////////////////////
//Init
if($restore && is_file($file)) { //Load last url as seed
    $urls[1]=$seed; //original seed as backup (if no urls found at restored page)
    if($debug) echo("!Restoring: ");
    $rest = fopen($file, 'r');
    while(!feof($rest)) {
        $seed = $new;
        $new = fgets($rest);
    }
    fclose($rest);
    $seed = trim($seed);
    if($debug) echo($seed."\n");
    $urls[0]=$seed;
    $seed=$urls[1];
}

if($delete_url_db) @unlink($file);
ini_set('default_socket_timeout',$socket_timeout);
set_time_limit($time_limit);
ini_set('memory_limit', $memory_limit);

//Loop
if($debug) { $u = 0; $c = 0; $t = time(); }
$history[0] = '';
$sort_next = $sort_each;
$sorted_db = 0;
$fp = fopen($file, 'a+');
while(true) { //MAIN L00P
    if(sizeof($urls) <= 0) {
        $urls='';
        $urls[0]=$seed;
        if($buffer_increase) $max_urls++; //Increase size of buffer by one
        if($debug) echo("!Buffer underrun! !Buffer size is: $max_urls!\n");
    }
    
    //Get URL from buffer, download few bytes and write down URLs to array:
    $url = array_shift($urls);
    if($debug) echo("-Parsing: $url\n");
    if(!$use_curlbin) { $page = @file_get_contents($url, false, null, 0, $max_size); }
    else { $page = shell_curl_get($url, $socket_timeout, $max_size); }
    preg_match_all('(http:\/\/[_a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,4}\/{1}[-_~&=\ ?\.a-z0-9\/]*)',htmlspecialchars_decode($page), $new_urls);
    $new_urls = $new_urls[0];
    
    foreach($new_urls as $new_url) { //Process grabed URLs
        if($debug) $c++;///Debug
        if(check_crawl_url($new_url)) {
            echo($new_url."\n");    
            fwrite($fp, $new_url."\n");

            if($use_sortbin) { //Handle sorting (unique URLs)
                $sort_next--;
                if($sort_next<=0) {
                  fclose($fp);
                    shell_sort_file($file);
                    if($debug) {
                        $sorted_db = 0;
                        $fp = fopen($file, 'r');
                        while(!feof($fp)) {
                            fgets($fp);
                            $sorted_db++;
                        }
                        fclose($fp);
                    }
                    $fp = fopen($file, 'a+');
                    $sort_next = $sort_each;
                }
            }

            $file_url = explode('?', $new_url, 2); $file_url = $file_url[0];
            array_push($history, $file_url);
            while(sizeof($history) > $history_max) @array_shift($history);
            if($random_url) { if(sizeof($urls) < $max_urls) array_push($urls, $new_urls[rand(0,sizeof($new_urls)-1)]); }
                else { if(sizeof($urls) < $max_urls) array_push($urls, $new_url); }
            if($debug) { ///Debug Block
                $u++;
                $sorted_db++;
                $uspeed = round($u/(time()-$t+1), 2);
                $cspeed = round($c/(time()-$t+1), 2);
                $time = round((time()-$t)/60, 1);
                $buffered = sizeof($urls);
                $historied = sizeof($history);
                $memory = round(memory_get_usage()/1000000, 2);
                echo("+$u ($sorted_db sorted) URLs; $c Downloaded; UpTime: $time mins; Buffered: $buffered URLs; History: $historied URLs; To sort: $sort_next URLs; Speed: $uspeed URLs/s, $cspeed Downloads/s; Memory used: $memory/$memory_limit\n");
            }
        }
    }
    $new_urls = ''; //Memory cleanup
}

##EOF
Commit	Line	Data
	1	#!/usr/bin/php
	2	<?php
	3	//HSp33der Web Crawler 5.6
	4	//<-Harvie 2oo7/8
	5	/*
	6	* Description:
	7	* This script crawls the web and printing found URLs to STDOUT.
	8	*
	9	* Installation:
	10	* - Debian:
	11	* # apt-get install php5-cli php5-curl curl sort
	12	* $ chmod +x crawler.php
	13	* $ ./crawler.php (or $ php crawler.php)
	14	* - Windows:
	15	* c:/>path/to/php5/php.exe crawler.php
	16	* (you can add php.exe to PATH or associate .php files with it)
	17	*
	18	* Notes:
	19	* - You can use EtherApe (or similar sniffer) to get graphical
	20	* illustration of connecions (spider with your machine in center).
	21	* - There are few bugs in PHP5,
	22	* so this crawler may freeze after few minutes,
	23	* but you can use $use_curlbin option (need some extra files).
	24	* Check this for more info: http://bugs.php.net/bug.php?id=43098
	25	*
	26	* Tips & Tricks:
	27	* - Make found URLs 100% unique:
	28	* cat urls.txt \| sort -u > uurls.txt && mv -f uurls.txt urls.txt
	29	* - Show filesize and total urls count:
	30	* du -h urls.txt && cat urls.txt \| wc -l
	31	* - Open another output pipe:
	32	* tail -f urls.txt
	33	* - Use some plugin (filter) to extract emails, etc...
	34	* tail -f urls.txt \| ./example_mail_extractor.php
	35	*
	36	* Special thx2:
	37	* - PHP, cURL (technology)
	38	* - root.cz & php.net (help)
	39	* - Linus Torvalds (because I am crawling from Linux)
	40	*/
	41
	42	///SETTINGS//////////////////
	43	//Basic
	44	$seed = 'http://blog.harvie.cz/'; //Start crawling from this page
	45	$file = 'urls.txt'; //File to save crawled URLs (use sort to make URLs 100% exclusive)
	46	$delete_url_db = false; //Delete file with saved URLs before crawling
	47	$restore = true; //Use last url from $file instead of seed
	48	//Advanced
	49	$max_size = 10000; //How many bytes download from each page?
	50	$max_urls = 30; //Size of URL buffer (prebuffered URLs to crawl, if full, no more URLs will be added to queue)
	51	$buffer_increase = true; //Increase buffer at buffer underrun?
	52	$random_url = false; //Select random urls from page? (This may override URL filter)
	53	$history_max = 1000; //How many last URLs keep in history? (historied URLs will not be crawled again...)
	54	//Curl binary plugin
	55	$use_curlbin = true; //Use interface to cURL executable instead of file_get_contents() (More speed, less crashes)
	56	$curlpath = 'curl'; //Path to cURL executable binary (you can download it from http://curl.haxx.se/)
	57	//Sort binary plugin
	58	$use_sortbin = true; //Use sort to make URLs unique?
	59	$sortpath = 'sort'; //Path to SORT executable binary (get it from your OS or UNXUtils for Windows)
	60	$sort_each = $history_max; //Sort file after X new URLs
	61	//System/PHP
	62	$socket_timeout = 1; //How long you want wait for a webserver? (seconds)
	63	$time_limit = 0; //How long run? (seconds; 0==infinite)
	64	$memory_limit = '128M'; //Maximum memory to eat (if exhausted, crawler will stop)
	65	//Debug
	66	$debug = true; //Use debuging mode? (Print errors and statistics)
	67
	68	//Filter
	69	$eregi_url_blacklist = array( //Most important thing when crawling ;D
	70	'(W3\.org\|W3C)', //Hell knows...
	71	'(shop\|xxx\|porn\|lesbian\|hot)', //Commercial sites
	72	'(google\|209.85.135.104\|yahoo.com\|amazon.com\|youtube.com)', //Big sites
	73	'(seznam.cz\|centrum.cz\|atlas.cz\|zoznam.sk\|quick.cz)', //Big local sites
	74	'.\.(css\|ico\|gif\|jpg\|png\|bmp\|cgi\|js\|vbs)', //Misc. webpage content
	75	'.\.(avi\|mpg\|mpeg\|mov\|wmv\|wm\|mp3\|ogg\|wma)', //Multimedia files
	76	'.\.(pdf\|swf\|flv\|cfm)', //Other text files
	77	'.\.(exe\|zip\|rar\|gz\|bz\|bz2\|tar)' //Big and binary files
	78	);
	79
	80
	81	///FUNCTIONS/////////////////
	82	function shell_sort_file($file, $tmpfile='.tmp') {
	83	$tmpfile = $file.$tmpfile;
	84	if($GLOBALS['debug']) echo("-SORTing $file (using tempfile $tmpfile)\n");
	85	@unlink($tmpfile);
	86	system($GLOBALS['sortpath']." -u $file > $tmpfile");
	87	if(is_file($tmpfile) && (filesize($tmpfile) > 3)) {
	88	unlink($file);
	89	rename($tmpfile, $file);
	90	} else {
	91	if($GLOBALS['debug']) echo("-SORTing failed!!!\n");
	92	}
	93	if($GLOBALS['debug']) echo("-SORTing done!\n");
	94	}
	95
	96	function shell_curl_get($url, $timeout=0, $maxdata=0) { //I made this cURL interface cause file_get_contents() absolutely sux
	97	$curl = 'curl';
	98	if(isset($GLOBALS['curlpath'])) $curl = $GLOBALS['curlpath'];
	99	$args = "--get --location --connect-timeout $timeout --max-time $timeout --compressed --stderr curl.debug";
	100	$url = escapeshellcmd(trim($url));
	101	$cmd = "$curl --url \"$url\" $args";
	102	if($GLOBALS['debug']) echo("-Ex: $cmd\n");
	103	$fp = popen($cmd, 'r');
	104	stream_set_timeout($fp, $timeout);
	105	stream_set_blocking($fp, 0);
	106	//if($timeout>0) sleep($timeout);
	107	$data = '';
	108	while(!feof($fp)) {
	109	$in = fgets($fp, 100);
	110	$data .= $in;
	111	if($maxdata>0 && strlen($data)>=$maxdata) break;
	112	if($in == '') usleep(1000000/8);
	113	}
	114	@fclose($fp);
	115	return $data;
	116	}
	117
	118	function check_crawl_url($url) { //Use this function to determine if url is wanted
	119	foreach($GLOBALS['eregi_url_blacklist'] as $black_url) {
	120	if(eregi($black_url, $url)) return(0);
	121	}
	122	if(in_array($url, $GLOBALS['urls'])) return(0);
	123	$file_url = explode('?', $url, 2); $file_url = $file_url[0];
	124	if(in_array($file_url, $GLOBALS['history'])) return(0);
	125	return(1);
	126	}
	127
	128
	129	///CODE//////////////////////
	130	//Init
	131	if($restore && is_file($file)) { //Load last url as seed
	132	$urls[1]=$seed; //original seed as backup (if no urls found at restored page)
	133	if($debug) echo("!Restoring: ");
	134	$rest = fopen($file, 'r');
	135	while(!feof($rest)) {
	136	$seed = $new;
	137	$new = fgets($rest);
	138	}
	139	fclose($rest);
	140	$seed = trim($seed);
	141	if($debug) echo($seed."\n");
	142	$urls[0]=$seed;
	143	$seed=$urls[1];
	144	}
	145
	146	if($delete_url_db) @unlink($file);
	147	ini_set('default_socket_timeout',$socket_timeout);
	148	set_time_limit($time_limit);
	149	ini_set('memory_limit', $memory_limit);
	150
	151	//Loop
	152	if($debug) { $u = 0; $c = 0; $t = time(); }
	153	$history[0] = '';
	154	$sort_next = $sort_each;
	155	$sorted_db = 0;
	156	$fp = fopen($file, 'a+');
	157	while(true) { //MAIN L00P
	158	if(sizeof($urls) <= 0) {
	159	$urls='';
	160	$urls[0]=$seed;
	161	if($buffer_increase) $max_urls++; //Increase size of buffer by one
	162	if($debug) echo("!Buffer underrun! !Buffer size is: $max_urls!\n");
	163	}
	164
	165	//Get URL from buffer, download few bytes and write down URLs to array:
	166	$url = array_shift($urls);
	167	if($debug) echo("-Parsing: $url\n");
	168	if(!$use_curlbin) { $page = @file_get_contents($url, false, null, 0, $max_size); }
	169	else { $page = shell_curl_get($url, $socket_timeout, $max_size); }
	170	preg_match_all('(http:\/\/[_a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,4}\/{1}[-_~&=\ ?\.a-z0-9\/]*)',htmlspecialchars_decode($page), $new_urls);
	171	$new_urls = $new_urls[0];
	172
	173	foreach($new_urls as $new_url) { //Process grabed URLs
	174	if($debug) $c++;///Debug
	175	if(check_crawl_url($new_url)) {
	176	echo($new_url."\n");
	177	fwrite($fp, $new_url."\n");
	178
	179	if($use_sortbin) { //Handle sorting (unique URLs)
	180	$sort_next--;
	181	if($sort_next<=0) {
	182	fclose($fp);
	183	shell_sort_file($file);
	184	if($debug) {
	185	$sorted_db = 0;
	186	$fp = fopen($file, 'r');
	187	while(!feof($fp)) {
	188	fgets($fp);
	189	$sorted_db++;
	190	}
	191	fclose($fp);
	192	}
	193	$fp = fopen($file, 'a+');
	194	$sort_next = $sort_each;
	195	}
	196	}
	197
	198	$file_url = explode('?', $new_url, 2); $file_url = $file_url[0];
	199	array_push($history, $file_url);
	200	while(sizeof($history) > $history_max) @array_shift($history);
	201	if($random_url) { if(sizeof($urls) < $max_urls) array_push($urls, $new_urls[rand(0,sizeof($new_urls)-1)]); }
	202	else { if(sizeof($urls) < $max_urls) array_push($urls, $new_url); }
	203	if($debug) { ///Debug Block
	204	$u++;
	205	$sorted_db++;
	206	$uspeed = round($u/(time()-$t+1), 2);
	207	$cspeed = round($c/(time()-$t+1), 2);
	208	$time = round((time()-$t)/60, 1);
	209	$buffered = sizeof($urls);
	210	$historied = sizeof($history);
	211	$memory = round(memory_get_usage()/1000000, 2);
	212	echo("+$u ($sorted_db sorted) URLs; $c Downloaded; UpTime: $time mins; Buffered: $buffered URLs; History: $historied URLs; To sort: $sort_next URLs; Speed: $uspeed URLs/s, $cspeed Downloads/s; Memory used: $memory/$memory_limit\n");
	213	}
	214	}
	215	}
	216	$new_urls = ''; //Memory cleanup
	217	}
	218
	219	##EOF