bash/bashcrawl/crawl.bash

   1 #!/bin/bash
   2
   3 CURL_PATH="curl";
   4 CURL_TIMEOUT=1;
   5 #CURL_ARGS=--get --location --connect-timeout "$CURL_TIMEOUT" --max-time "$CURL_TIMEOUT" --compressed --stderr curl.debug;
   6
   7 #        $cmd = "$curl --url \"$url\" $args";
   8
   9
  10 grab_urls() {
  11         if [[ $( "$CURL_PATH" "--url" "$1" --get --location --connect-timeout "$CURL_TIMEOUT" --max-time "$CURL_TIMEOUT" --compressed --stderr curl.debug) =~ \(http:\/\/[_a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,4}\/{1}[-_~\&=\ ?\.a-z0-9\/]*\) ]]; then
  12                 for i in $( seq 0 $[ ${#BASH_REMATCH}-1 ] ); do
  13                         # echo $i;
  14                 if [ "${BASH_REMATCH[i]}" != "" ]; then
  15                         echo "${BASH_REMATCH[i]}";
  16                 fi;
  17         done | sort -u;
  18         fi;
  19 }
  20
  21 grab_urls http://www.abclinuxu.cz/