6 use LWP
::Simple
qw($ua get);
12 my $urldb = './urls.txt';
14 print "This is libwww-perl-$LWP::VERSION\n";
16 my $urlqueue = Thread::Queue->new();
17 #open(my $fp, "<$urldb"); while(<$fp>) { $urlqueue->enqueue($_); } close($fp);
18 $urlqueue->enqueue("http://root.cz/");
19 $urlqueue->enqueue("http://blog.harvie.cz/");
24 @$i = grep(!$h{$_}++, @$i);
29 #die(Dumper($urlqueue));
35 for(my $i=0; $i<$threads; $i++) {
36 my $thr = threads->create(sub {
37 print("[*] Worker #$i running!\n");
39 while (my $url = $urlqueue->dequeue()) {
40 #$urlqueue->enqueue($url);
41 print "#$i:\tGET $url\n";
42 $_ = LWP::Simple::get($url);
45 @urls = /(http:\/\/[_a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,4}\/{1}[-_~&=\ ?\.a-z0-9\/]*)/g; #urls
46 #@urls = /(http:\/\/[^\/'" ]*)/g; #domains
47 if($urlqueue->pending() < 1000) {
48 #print("#$i:\tENQ: @urls\n");
49 $urlqueue->enqueue(uniq(@urls));
50 #while(uniq(@urls)) { $urlqueue->enqueue($_); }
55 print("[*] Worker #$i stopped!\n");
60 print '###: '.$urlqueue->pending()."\n";
This page took 0.395854 seconds and 4 git commands to generate.