7 use LWP
::Simple
qw($ua get);
12 my $urldb = './urls.txt';
14 print "This is libwww-perl-$LWP::VERSION\n";
16 my @urlbuffer :shared;
17 #open(my $fp, "<$urldb"); @urlbuffer = <$fp>; close($fp); chop(@urlbuffer);
18 push(@urlbuffer, 'http://root.cz/');
19 push(@urlbuffer, 'http://blog.harvie.cz/');
24 @$i = grep(!$h{$_}++, @$i);
32 for(my $i=0; $i<$threads; $i++) {
33 my $thr = threads->create(sub {
34 print("[*] Worker #$i running!\n");
37 lock(@urlbuffer); shift(@urlbuffer);
38 if(length($_) < 3) { sleep(1); next; }
40 print "#$i:\tGET $_\n";
41 $_ = LWP::Simple::get($_);
45 @urls = /(http:\/\/[_a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,4}\/{1}[-_~&=\ ?\.a-z0-9\/]*)/g; #urls
46 #@urls = /(http:\/\/[^\/'" ]*)/g; #domains
47 if($i && @urlbuffer < 1000) {
48 #print("#$i:\tENQ: @urls\n");
50 push(@urlbuffer, uniq(@urls));
51 #while(uniq(@urls)) { push(@urlbuffer, $_); }
56 print("[*] Worker #$i stopped!\n");
61 print '###: '.@urlbuffer."\n";
62 lock(@urlbuffer); uniq(@urlbuffer);
63 print '###: '.@urlbuffer."\n";
65 open(my $fp, ">$urldb");
66 print $fp join("\n", @urlbuffer);
This page took 0.388548 seconds and 4 git commands to generate.