perl/crawler/berkeley.pl

   1 #!/usr/bin/env perl
   2 use strict;
   3 use warnings;
   4 use threads; # ('exit' => 'threads_only');
   5 use BerkeleyDB; #man 3pm BerkeleyDB
   6
   7
   8
   9 our $dbenv = BerkeleyDB::Env->new(
  10         -Home => './',
  11         -Flags =>  DB_CREATE|DB_INIT_CDB|DB_INIT_MPOOL
  12 ) or die("cannot open environment: $BerkeleyDB::Error\n");
  13
  14 our $dbq   = BerkeleyDB::Queue->new(
  15         -Filename => 'urls.queue' ,
  16         -Flags      => DB_CREATE,
  17         -Len        => 32,
  18         -Env        => $dbenv
  19 ) or die("cannot open queue: $BerkeleyDB::Error\n");
  20
  21 for(my $i=0;$i<10;$i++) { $dbq->db_put($i, 'toto'); }
  22
  23
  24 my $threads = 2;
  25 for(my $i=0; $i<$threads; $i++) {
  26         my $thr = threads->create(sub {
  27                 print("[*] Worker #$i running!\n");
  28                 my $key=0;
  29                 my $val='';
  30                 my $cursor = $dbq->db_cursor() ;
  31                 while ($cursor->c_get($key, $val, DB_NEXT) == 0) {
  32                         #print "$key -> $val\n"
  33                         sleep(1);
  34                         if($val) { print("[$i] $key = $val\n"); }
  35                 }
  36                 #$dbq->db_get($key, $val, DB_CONSUME);
  37                 print("[*] Worker #$i stopped!\n");
  38         }) -> detach();
  39 }
  40
  41 sleep(3);
  42
  43