CodeSnippets: internet code

Never been to CodeSnippets before?

Snippets is a public source code repository. Easily build up your personal collection of code snippets, categorize them with tags / keywords, and share them with the world (or not, you can keep them private!)

Older Snippets »

8 total

Batch download code snippets

to shell osx mac curl bash unix code web download internet batch ed getopts snippets trap by jvs on Nov 12, 2008

Batch download snippets from http://codesnippets.joyent.com and convert them to text files using man textutil (available on Mac OS X 10.4 or later).

Note: Old snippet versions will be automatically replaced by the downloaded snippets without a backup!

Author: jv
License: The MIT License, Copyright (c) 2008 jv

Usage:

# usage: bds [-p num] [-t tag] [-u user] tag
bds vim
bds -p 1280
bds -u jvs
bds -t plistbuddy
bds -t tar
bds -t ipfw -u jvs

#!/opt/local/bin/bash

# "batch download snippets" from http://codesnippets.joyent.com and
# convert them to text files using man textutil (available on Mac OS X 10.4 or later).
#
# Note: Old snippet versions will be automatically replaced by the downloaded snippets without a backup!
#      An alternative to man textutil is html2text, http://www.mbayer.de/html2text/ (which is available via MacPorts).
#
# Author: jv
# License: The MIT License, http://www.opensource.org/licenses/mit-license.php
# Copyright (c) 2008 jv
#
# cat /usr/local/bin/bds
#
# usage: bds [-p num] [-t tag] [-u user] tag


declare BaseURL='http://codesnippets.joyent.com'
declare download_dir="${HOME}/Desktop/Snippets"

# make sure there is no trailing slash
BaseURL="${BaseURL%/}"
download_dir="${download_dir%/}"

declare BasePostURL="${BaseURL}/posts/show"
declare BaseTagURL="${BaseURL}/tag"
declare BaseUserURL="${BaseURL}/user"

# make sure there is no trailing slash
BasePostURL="${BasePostURL%/}"
BaseTagURL="${BaseTagURL%/}"
BaseUserURL="${BaseUserURL%/}"


# man textutil
declare InputEncoding='utf-8'
declare OutputEncoding='utf-8'

export IFS=$' \t\n'


# function to download a single post specified by a post number: bds -p num
# cf. snippet, http://codesnippets.joyent.com/posts/show/1282

function snippet() {

   declare NL OPWD file outputfile postnum title url

   if [[ "${1//[[:digit:]]/}" != "" ]]; then echo "Argument error. No positive integer: ${1}"; return 1; fi

   postnum="${1}"
   url="${BasePostURL}/${postnum}"
   download_dir="${download_dir}/single-downloads"
   /bin/mkdir -p "${download_dir}"
   OPWD="${PWD}"
   cd "${download_dir}"
   /usr/bin/curl -L -O -s --max-time 25 "${url}" || exit 1    # download snippet web page
   file="${download_dir}/${url##*/}"
   trap '/bin/rm -f "${file}"; exit 0' 0 1 2 13 15

   # get title of downloaded web page
   #title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
   #         /usr/bin/sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

   title="$(/usr/bin/egrep -m 1 -io '<title>.*</title>' "${file}" | /usr/bin/sed -E -e 's/^<title>[[:space:]]*|[[:space:]]*<\/title>$//g' \
             -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title


   title="${title//CodeSnippets:/}"
   title="${title//\//:}"
   title="${title// /_}"
   title="${title//[[:cntrl:]]/}"
   title="${title%"${title##*[!_]}"}"   # remove trailing underscores

   if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
      /bin/rm "${file}"
      return 1
   fi

   outputfile="${download_dir}/${postnum}_${title}.txt"
   #outputfile="${download_dir}/${title}.txt"  # without post number prefix
   #outputfile="${outputfile//__/_}"  # uniq underscores

   printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "saved as" "${outputfile}"

   /usr/bin/textutil -output "${outputfile}" -convert txt -inputencoding "${InputEncoding}" -encoding "${OutputEncoding}" "${file}"
   /bin/rm "${file}"

   # escape backslashes
   # man bash 2>/dev/null | less -p 'Each command in a pipeline'
   #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
   outputfile="${outputfile//\\/\\\\}"

   NL=$'\\\n'

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

# additional ed commands
# delete line numbers
# ,g|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}|s|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}\(.*\)$|\1|
# delete range of lines
# 4,11d


   cd "${OPWD}"
   return 0

}



#----------------------------------------- end of function snippet



declare pflag tflag uflag
declare cnt count dir_name file no_posts_check NL OPWD outputfile postnum tagsite title url urls website 

if [[ $# -eq 0 ]]; then 
   printf "%s\n%s\n" 'No arguments given!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1
fi


while getopts ":p:t:u:" option
do
  case $option in
    p) pflag="$OPTARG" ;;
    t) tflag="$OPTARG" ;;
    u) uflag="$OPTARG" ;;
    [?]) printf "%s\n%s\n" 'Argument error!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2; exit 1;;
    *) ;;
  esac
done

shift $(($OPTIND - 1))


if [[ $# -eq 1 ]]; then

   dir_name="${1}"
   tagsite="${BaseTagURL}/${1}"

elif [[ $# -gt 1 ]]; then

   printf "%s\n%s\n" 'Too many arguments!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1

elif [[ -n "${pflag}" ]]; then
   snippet "${pflag}"
   exit 0

elif [[ -n "${tflag}" ]] && [[ -n "${uflag}" ]]; then

   dir_name="${tflag}-${uflag}"
   tagsite="${BaseUserURL}/${uflag}/tag/${tflag}"

elif [[ -n "${tflag}" ]]; then

   dir_name="${tflag}"
   tagsite="${BaseTagURL}/${tflag}"

elif [[ -n "${uflag}" ]]; then

   dir_name="${uflag}"
   tagsite="${BaseUserURL}/${uflag}"

else

   printf "%s\n%s\n" 'Argument error!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1

fi


tagsite="${tagsite%/}"

#echo $dir_name
#echo $tagsite

count=1
cnt=0
curl_max_time=20
website=''
no_posts_check=''
NL=$'\\\n'
download_dir="${download_dir}/${dir_name//\//:}"
download_dir="${download_dir%/}"
/bin/mkdir -p "${download_dir}"
OPWD="${PWD}"
cd "${download_dir}"

# print download directory
printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "download directory" "${download_dir}"


while [[ -z "${no_posts_check}" ]]; do

   # download website of the form: 
   # http://somewebsite.com/tag/bash/1,
   # http://somewebsite.com/user/name/1 or 
   # http://somewebsite.com/user/name/tag/bash/1

   website="$(/usr/bin/curl -L -s --max-time $curl_max_time "${tagsite}/${count}" )"

   if [[ $? -ne 0 ]]; then 
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "curl_max_time ${curl_max_time}" "${tagsite}/${count}"
      exit 1
   fi

   #if [[ -n "$(printf "%s" "${website}" | /usr/bin/egrep -o 'Application error \(Apache\)')" ]]; then 
      #no_posts_check='Application error (Apache)'
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   #fi

   if [[ -n "$(printf "%s" "${website}" | /usr/bin/egrep -o '>No posts<')" ]]; then 
      no_posts_check='>No posts<'
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   fi

: <<-'COMMENT'

   # works for Bash 3.0 or later
   if [[ "${website}" =~ '>No posts<' ]]; then 
      no_posts_check="${BASH_REMATCH[0]}"
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   fi

COMMENT


   if [[ -z "${no_posts_check}" ]]; then

      # extract relevant post URLs
      #urls=( $(printf "%s\n" "${website}" | /usr/bin/sed -E -n -e "s|^.* href=\"(/posts/show/[[:digit:]]+)\".*$|${BaseURL}\1|p;g") )
      urls=( $(printf "%s\n" "${website}" | /usr/bin/egrep -o 'href="/posts/show/[[:digit:]]+"' | /usr/bin/sed -E -n -e "s|href=\"(/posts/show/[[:digit:]]+)\"|${BaseURL}\1|p;g") )

      for ((i=0; i < "${#urls[@]}"; i++)); do

         url="${urls[${i}]}"

         postnum="${url##*/}"
         file="${download_dir}/${postnum}"
         trap '/bin/rm -f "${file}"; exit 0' 0 1 2 13 15

         /usr/bin/curl -L -O -s --max-time $curl_max_time "${url}"

         if [[ $? -ne 0 ]]; then 
            printf "\e[0K\e[31m%s\e[0m:  %s\n" "curl_max_time ${curl_max_time}" "${url}"
            continue
         fi
 

         # get title of downloaded web page
         #title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
         #    /usr/bin/sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

         title="$(/usr/bin/egrep -m 1 -io '<title>.*</title>' "${file}" | /usr/bin/sed -E -e 's/^<title>[[:space:]]*|[[:space:]]*<\/title>$//g' \
                -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title


         title="${title//CodeSnippets:/}"
         title="${title//\//:}"
         title="${title// /_}"
         title="${title//[[:cntrl:]]/}"
         title="${title%"${title##*[!_]}"}"   # remove trailing underscores

         #printf "%s\n" "${title}"

         if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
            printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
            /bin/rm "${file}"
            continue
         fi

         outputfile="${download_dir}/${postnum}_${title}.txt"
         #outputfile="${download_dir}/${title}.txt"  # without post number prefix
         #outputfile="${outputfile//__/_}"  # uniq underscores

         let cnt++
         printf "\e[0K\e[1;32m%-6s\e[0m  %s\n" "${cnt}" "${outputfile##*/}"

         /usr/bin/textutil -output "${outputfile}" -convert txt -inputencoding "${InputEncoding}" -encoding "${OutputEncoding}" "${file}"

         /bin/rm "${file}"


         # escape backslashes
         # man bash 2>/dev/null | less -p 'Each command in a pipeline'
         #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
         outputfile="${outputfile//\\/\\\\}"

# edit $outputfile in-place with man ed
# first delete lines at the beginning & end,
# then remove the string 'See related posts' and add some newlines with $NL,
# then convert the line 'to...by...on' to line 'Author:...', line 'Date:...', line 'URL:...' and line 'Tags:...'
# and finally the last two ed commands insert two further newlines with $NL

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

# additional ed commands
# delete line numbers
# ,g|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}|s|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}\(.*\)$|\1|
# delete range of lines
# 4,11d


      done  # for

      let count++

   fi

done   # while


   cd "${OPWD}"


exit 0

Ping a port with hping3

to shell osx mac bash ping unix network web port ipfw internet hping hping3 by jvs on Oct 14, 2008

# cf. http://trac.macports.org/wiki/InstallingMacPorts
/opt/local/bin/port info hping3
/usr/bin/sudo /opt/local/bin/port install hping3

hping3 --help


# Terminal window 1
alias sudo=/usr/bin/sudo
alias tcpdump=/usr/sbin/tcpdump
#sudo tcpdump -s0 -xX -i lo0 port 4678 and host localhost
sudo tcpdump -s0 -vvv -i lo0 port 4678 and host localhost


# Terminal window 2
alias sudo=/usr/bin/sudo
alias hping3=/opt/local/sbin/hping3
sudo hping3 -I lo0 -s 4678 -c 1 localhost -p 4678
sudo hping3 -S -I lo0 -d 995 -s 4678 -c 1 localhost -p 4678
sudo hping3 -SA -I lo0 -d 995 -w 200 -s 4678 -c 1 localhost -p 4678
sudo hping3 -SA -M 3000 -I lo0 -d 995 -w 65 -s 4678 -c 1 localhost -p 4678
sudo hping3 -DV -SA -I lo0 -s 4678 -a 192.168.1.100 -c 1 localhost -p 4678


#----------------------------------------


# some corresponding ipfw rules for testing purposes 
# (just place them at the beginning of your ipfw rule set)

# not me to me & me to not me
/sbin/ipfw -q add count log all from not me to me 4678 in
/sbin/ipfw -q add count log all from me to not me 4678 out

# any to me & me to any
/sbin/ipfw -q add count log all from any to me 4678 in
/sbin/ipfw -q add count log all from me to any 4678 out
    
# any to any
/sbin/ipfw -q add count log all from any to any 4678

Analyze internet traffic volume with dynamic ipfw rules

to awk shell osx mac bash unix dynamic web ipfw internet traffic volume rule by jvs on Oct 11, 2008

# cf. Example ipfw ruleset, http://codesnippets.joyent.com/posts/show/1267
# cf. also http://codesnippets.joyent.com/posts/show/1729

man ipfw 2>/dev/null | less -p "If the ruleset"
man ipfw 2>/dev/null | less -p "These dynamic rules"
man ipfw 2>/dev/null | less -p "All rules"

man ipfw 2>/dev/null | less -p "STATEFUL FIREWALL"     # press [n]
man ipfw 2>/dev/null | less -p "SYSCTL VARIABLES"
man ipfw 2>/dev/null | less -p "EXAMPLES"
man ipfw 2>/dev/null | less -p "DYNAMIC RULES"

/usr/bin/sudo /sbin/ipfw -d -e -t list
/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/sed -E -n -e '1,/^## Dynamic rules/p'
/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/sed -E -n -e '/^## Dynamic rules/,$p'
/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/awk '/^## Dynamic rules/,/^$/ {print $0}'
/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/awk '/<->/ {print $0}'
/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/awk '{ if ( $0 ~ /<->/ ) {print $0}}'

/usr/bin/sudo /sbin/ipfw -d -e -t list | grep RULENUM
/usr/bin/sudo /sbin/ipfw -d -e -t list | grep IPADDR

/usr/sbin/sysctl -a | egrep 'tcp'
/usr/sbin/sysctl -a | egrep 'net.inet'
/usr/sbin/sysctl -a | egrep '\.fw'
/usr/sbin/sysctl -a | egrep 'ip.fw'
/usr/sbin/sysctl -a | egrep 'li[fv]e'
/usr/sbin/sysctl -a | egrep 'ip.fw.+life'

/usr/sbin/sysctl -n net.inet.tcp.always_keepalive
/usr/sbin/sysctl -n net.inet.ip.fw.dyn_keepalive
/usr/sbin/sysctl -n net.inet.ip.fw.dyn_buckets
/usr/sbin/sysctl -n net.inet.ip.fw.dyn_count
/usr/sbin/sysctl -n net.inet.ip.fw.dyn_max


# list all dynamic ipfw rules

function ipfwtraffic() {

   declare args argsregex bytes megabytes

   if [[ $# -eq 0 ]]; then

      /usr/bin/sudo /sbin/ipfw -d -e -t list | \
         /usr/bin/awk '/<->/ {printf "%-10s %-10s %-20s %-10s %-20s %-10s %-10s\n", $3, $6, $7, $8, $10, $11, $1}' | \
         /usr/bin/sort -bu | while IFS=" " read  bytes proto ipnum1 port1 ipnum2 port2 rulenum; do

      # byte
      #bytes=$(printf "%s\n" "${bytes}" | /usr/bin/awk '{ total = total + $1 } END { print total }')
      #printf "\e[1mbytes\e[m: %-17s %-10s %-40s \e[1mrule\e[m: %-15s \e[1mports\e[m: %-15s\n" \
           #"${bytes}" "${proto}" "${ipnum1}  ::  ${ipnum2}" "${rulenum}" "${port1}  ${port2}"

      # mega byte
      megabytes=$(printf "%s\n" "${bytes}" | /usr/bin/awk '{ total = (total + $1) / (1024*1024.0) } END { print total }')
      printf "\e[1mmbytes\e[m: %-20.6f %-10s %-40s \e[1mrule\e[m: %-15s \e[1mports\e[m: %-15s\n" \
           "${megabytes}" "${proto}" "${ipnum1}  ::  ${ipnum2}" "${rulenum}" "${port1}  ${port2}"

      done | /usr/bin/sort -rn -k 2,2

   else

      args="${@}"
      if [[ "${args}" != "${args//[^. [:digit:]]/}" ]]; then 
         printf "%s\n" 'Found at least one invalid rule number or IP address!'
         return 1
      fi

      if [[ "${args//[ [:digit:]]/}" == '' ]]; then 
         argsregex="^0*(${args// /|})"    #  ipfw rule numbers 
      else
         argsregex="(${args// /|})"    #  IP addresses
      fi
    
      #echo $argsregex

      /usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
         /usr/bin/awk '/<->/ {printf "%-10s %-10s %-20s %-10s %-20s %-10s %-10s\n", $3, $6, $7, $8, $10, $11, $1}' | \
         /usr/bin/sort -bu | while IFS=" " read  bytes proto ipnum1 port1 ipnum2 port2 rulenum; do

      # byte
      #bytes=$(printf "%s\n" "${bytes}" | /usr/bin/awk '{ total = total + $1 } END { print total }')
      #printf "\e[1mbytes\e[m: %-17s %-10s %-40s \e[1mrule\e[m: %-15s \e[1mports\e[m: %-15s\n" \
           #"${bytes}" "${proto}" "${ipnum1}  ::  ${ipnum2}" "${rulenum}" "${port1}  ${port2}"

      # mega byte
      megabytes=$(printf "%s\n" "${bytes}" | /usr/bin/awk '{ total = (total + $1) / (1024*1024.0) } END { print total }')
      printf "\e[1mmbytes\e[m: %-20.6f %-10s %-40s \e[1mrule\e[m: %-15s \e[1mports\e[m: %-15s\n" \
           "${megabytes}" "${proto}" "${ipnum1}  ::  ${ipnum2}" "${rulenum}" "${port1}  ${port2}"

      done | /usr/bin/sort -rn -k 2,2

   fi

   return 0
}



# usage:
# ipfwtraffic
# ipfwtraffic [rulenum1] [rulenum2] [rulenum3] ...
# ipfwtraffic [ipaddr1] [ipaddr2] [ipaddr3] ...


ipfwtraffic
ipfwtraffic  9600 10600 11000
ipfwtraffic xx.xxx.xx.xxx xx.xxx.xx.xx

ipfwtraffic | grep 'xx.xxx.xx.xx'



#------------------------------------------------------------------------------



# summarize pairs of IP addresses

function ipfwdynstats() {

   declare args argsregex dynrules ipaddr_pairs

   OIFS=${IFS}
   IFS=$'\n'

   if [[ $# -eq 0 ]]; then

      ipaddr_pairs=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/awk '/<->/ {print $7, $10}' | /usr/bin/sort -bu))

      dynrules=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/awk '/<->/ {print $7, $8, $10, $11, $6, $2, $3, $1}' | /usr/bin/sort -bu))

   else

      args="${@}"

      if [[ "${args}" != "${args//[^. [:digit:]]/}" ]]; then 
         printf "%s\n" 'Found at least one invalid rule number or IP address!'
         return 1
      fi

      if [[ "${args//[ [:digit:]]/}" == '' ]]; then 
         argsregex="^0*(${args// /|})"    #  ipfw rule numbers 
      else
         argsregex="(${args// /|})"    #  IP addresses
      fi
    

      ipaddr_pairs=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
           /usr/bin/awk '/<->/ {print $7, $10}' | /usr/bin/sort -bu))

      dynrules=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
           /usr/bin/awk '/<->/ {print $7, $8, $10, $11, $6, $2, $3, $1}' | /usr/bin/sort -bu))


   fi


for ((i=0; i < "${#ipaddr_pairs[@]}"; i++)); do 

   # byte
   #bytesum=$(printf "%s\n" "${dynrules[@]}" | \
        #/usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $7 }' | \
        #/usr/bin/awk '{ total = total + $1 } END { print total }')

   # mega byte
   bytesum=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $7 }' | \
        /usr/bin/awk 'BEGIN { total=0 }; { total = (total + $1) / (1024*1024.0) } END { print total }')


   proto=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $5 }' | \
        /usr/bin/sort -bu)


   rule=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $NF }' | \
        /usr/bin/sort -bu)


   ports=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $2,$4 }' | \
        /usr/bin/sort -bu)


   # byte
   #printf "\e[1mbytes\e[m: %-20s %-10s %-40s \e[1mrules\e[m: %-25s \e[1mports\e[m: %-30s\n" \
        #"${bytesum}" "${proto//[[:cntrl:]]/, }" "${ipaddr_pairs[${i}]% *}  ::  ${ipaddr_pairs[${i}]#* }" \
        #"${rule//[[:cntrl:]]/, }" "${ports//[[:cntrl:]]/, }"


   # mega byte
   printf "\e[1mmbytes\e[m: %-20.6f %-10s %-40s \e[1mrules\e[m: %-25s \e[1mports\e[m: %-30s\n" \
        "${bytesum}" "${proto//[[:cntrl:]]/, }" "${ipaddr_pairs[${i}]% *}  ::  ${ipaddr_pairs[${i}]#* }" \
        "${rule//[[:cntrl:]]/, }" "${ports//[[:cntrl:]]/, }"

done | /usr/bin/sort -rn -k 2,2

   export IFS=${OIFS}
   return 0

}


# usage:
# ipfwdynstats
# ipfwdynstats [rulenum1] [rulenum2] [rulenum3] ...
# ipfwdynstats [ipaddr1] [ipaddr2] [ipaddr3] ...


ipfwdynstats
ipfwdynstats  5200 12700
ipfwdynstats xx.xxx.xx.xxx xxx.xxx.xx.xxx



#------------------------------------------------------------------------------



# list port-specific internet traffic

function porttraffic() {

   declare args argsregex dynrules ipaddr_pairs

   OIFS=${IFS}
   IFS=$'\n'

   if [[ $# -eq 0 ]]; then
   
      printf "%s\n" 'No port number given!'
      return 1
   
   elif [[ $# -eq 1 ]]; then

      if [[ "${1//[[:digit:]]/}" != '' ]]; then 
         printf "%s\n" 'Invalid port number!'
         return 1
      fi

      ipaddr_pairs=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | \
         /usr/bin/awk '/<->/ && ( $8 == "'"${1}"'" || $11 == "'"${1}"'" ) {print $7, $10 }' | /usr/bin/sort -bu))

      dynrules=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | \
         /usr/bin/awk '/<->/ && ( $8 == "'"${1}"'" || $11 == "'"${1}"'" ) {print $7, $8, $10, $11, $6, $2, $3, $1 }' | \
         /usr/bin/sort -bu))

   else

      args="${@:2}"   # all arguments starting with the second

      if [[ "${args}" != "${args//[^. [:digit:]]/}" ]]; then 
         printf "%s\n" 'Found at least one invalid rule number or IP address!'
         return 1
      fi

      if [[ "${args//[ [:digit:]]/}" == '' ]]; then 
         argsregex="^0*(${args// /|})"    #  ipfw rule numbers 
      else
         argsregex="(${args// /|})"    #  IP addresses
      fi
    
      #echo $argsregex

      ipaddr_pairs=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
         /usr/bin/awk '/<->/ && ( $8 == "'"${1}"'" || $11 == "'"${1}"'" )  {print $7, $10 }' | /usr/bin/sort -bu))

      dynrules=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
         /usr/bin/awk '/<->/ && ( $8 == "'"${1}"'" || $11 == "'"${1}"'" )  {print $7, $8, $10, $11, $6, $2, $3, $1 }' | \
         /usr/bin/sort -bu))

   fi


for ((i=0; i < "${#ipaddr_pairs[@]}"; i++)); do 

   # byte
   #bytesum=$(printf "%s\n" "${dynrules[@]}" | \
        #/usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $7 }' | \
        #/usr/bin/awk '{ total = total + $1 } END { print total }')

   # mega byte
   bytesum=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $7 }' | \
        /usr/bin/awk 'BEGIN { total=0 }; { total = (total + $1) / (1024*1024.0) } END { print total }')


   proto=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $5 }' | \
        /usr/bin/sort -bu)


   rule=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $NF }' | \
        /usr/bin/sort -bu)


   ports=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $2,$4 }' | \
        /usr/bin/sort -bu)


   # byte
   #printf "\e[1mbytes\e[m: %-20s %-10s %-40s \e[1mrules\e[m: %-25s \e[1mports\e[m: %-30s\n" \
        #"${bytesum}" "${proto//[[:cntrl:]]/, }" "${ipaddr_pairs[${i}]% *}  ::  ${ipaddr_pairs[${i}]#* }" \
        #"${rule//[[:cntrl:]]/, }" "${ports//[[:cntrl:]]/, }"


   # mega byte
   printf "\e[1mmbytes\e[m: %-20.6f %-10s %-40s \e[1mrules\e[m: %-25s \e[1mports\e[m: %-30s\n" \
        "${bytesum}" "${proto//[[:cntrl:]]/, }" "${ipaddr_pairs[${i}]% *}  ::  ${ipaddr_pairs[${i}]#* }" \
        "${rule//[[:cntrl:]]/, }" "${ports//[[:cntrl:]]/, }"

done | /usr/bin/sort -rn -k 2,2

   export IFS=${OIFS}
   return 0

}


# usage:
# porttraffic [portnum]
# porttraffic [portnum] [rulenum1] [rulenum2] [rulenum3] ...
# porttraffic [portnum] [ipaddr1] [ipaddr2] [ipaddr3] ...


porttraffic 80
porttraffic 80 5200 12700 7100
porttraffic 80 xx.xxx.xx.xxx xxx.xxx.xx.xxx



#------------------------------------------------------------------------------



# list rule-specific internet traffic

function ruletraffic() {

   declare args argsregex dynrules ipaddr_pairs

   OIFS=${IFS}
   IFS=$'\n'

   if [[ $# -eq 0 ]]; then
   
      printf "%s\n" 'No rule number given!'
      return 1
   
   elif [[ $# -eq 1 ]]; then

      if [[ "${1//[[:digit:]]/}" != '' ]]; then 
         printf "%s\n" 'Invalid rule number!'
         return 1
      fi

      ipaddr_pairs=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "^0*${1}" | \
         /usr/bin/awk '/<->/ {print $7, $10}' | /usr/bin/sort -bu))

      dynrules=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "^0*${1}" | \
         /usr/bin/awk '/<->/ {print $7, $8, $10, $11, $6, $2, $3, $1}' | /usr/bin/sort -bu))

      #echo "rulenum: ${1}"
      #echo "ipaddr_pairs: ${ipaddr_pairs[@]/%/$'\n'}"
      #echo "dynrules: ${dynrules[@]/%/$'\n'}"

   else

      args="${@}"

      if [[ "${args//[ [:digit:]]/}" != '' ]]; then 
         printf "%s\n" 'Found at least one invalid rule number!'
         return 1
      fi

      argsregex="^0*(${args// /|})"    #  ipfw rule numbers 

      ipaddr_pairs=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
         /usr/bin/awk '/<->/ {print $7, $10}' | /usr/bin/sort -bu))

      dynrules=($(/usr/bin/sudo /sbin/ipfw -d -e -t list | /usr/bin/egrep "${argsregex}" | \
         /usr/bin/awk '/<->/ {print $7, $8, $10, $11, $6, $2, $3, $1}' | /usr/bin/sort -bu))

      #echo "rulenums: ${@/%/$'\n'}"
      #echo "argsregex: ${argsregex}"
      #echo "ipaddr_pairs: ${ipaddr_pairs[@]/%/$'\n'}"
      #echo "dynrules: ${dynrules[@]/%/$'\n'}"

   fi


for ((i=0; i < "${#ipaddr_pairs[@]}"; i++)); do 

   # byte
   #bytesum=$(printf "%s\n" "${dynrules[@]}" | \
        #/usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $7 }' | \
        #/usr/bin/awk '{ total = total + $1 } END { print total }')

   # mega byte
   bytesum=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $7 }' | \
        /usr/bin/awk 'BEGIN { total=0 }; { total = (total + $1) / (1024*1024.0) } END { print total }')


   proto=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $5 }' | \
        /usr/bin/sort -bu)


   rule=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $NF }' | \
        /usr/bin/sort -bu)


   ports=$(printf "%s\n" "${dynrules[@]}" | \
        /usr/bin/awk '$1 == "'"${ipaddr_pairs[${i}]% *}"'" && $3 == "'"${ipaddr_pairs[${i}]#* }"'" { print $2,$4 }' | \
        /usr/bin/sort -bu)


   #echo "field 1: ${ipaddr_pairs[${i}]% *}"
   #echo "field 2: ${ipaddr_pairs[${i}]#* }"
   #echo "bytesum: ${bytesum}"
   #echo "proto: ${proto//[[:cntrl:]]/, }"
   #echo "rule: ${rule//[[:cntrl:]]/, }"
   #echo "ports: ${ports//[[:cntrl:]]/, }"


   # byte
   #printf "\e[1mbytes\e[m: %-20s %-10s %-40s \e[1mrules\e[m: %-25s \e[1mports\e[m: %-30s\n" \
        #"${bytesum}" "${proto//[[:cntrl:]]/, }" "${ipaddr_pairs[${i}]% *}  ::  ${ipaddr_pairs[${i}]#* }" \
        #"${rule//[[:cntrl:]]/, }" "${ports//[[:cntrl:]]/, }"


   # mega byte
   printf "\e[1mmbytes\e[m: %-20.6f %-10s %-40s \e[1mrules\e[m: %-25s \e[1mports\e[m: %-30s\n" \
        "${bytesum}" "${proto//[[:cntrl:]]/, }" "${ipaddr_pairs[${i}]% *}  ::  ${ipaddr_pairs[${i}]#* }" \
        "${rule//[[:cntrl:]]/, }" "${ports//[[:cntrl:]]/, }"

done | /usr/bin/sort -rn -k 2,2

   export IFS=${OIFS}
   return 0
}


# usage:
# ruletraffic [rulenum]
# ruletraffic [rulenum1] [rulenum2] [rulenum3] ...

ruletraffic 5200
ruletraffic 5200 12700 7100

deny_rules="$(/usr/bin/sudo /sbin/ipfw show | /usr/bin/awk '/deny/ {printf $1" "}')"
echo $deny_rules

ruletraffic $deny_rules


#-------------------------------


/usr/sbin/netstat -n -ibd
/usr/bin/sudo /usr/sbin/tcpdump -vvv -ni en0 not ip

Further information:

- IPFWstats
- ipfwsnmp
- ipfwsnmp (patched)
- ipfw counters (field 3)
- Multi Router Traffic Grapher
- cricket

Block internet traffic to specified web sites on the fly

to shell osx mac bash ip unix web internet access route traffic website localhost ipaddress block blackhole by jvs on Oct 09, 2008

man 8 route
/sbin/route -n get default
/sbin/route -n get default | grep -w gateway
/sbin/route -n get default | grep interface | awk '{print $NF}'

/usr/bin/dig +short www.web_site.com                                 # get IPNUM
/usr/bin/sudo /sbin/route -n add -host IPNUM 127.0.0.1 -blackhole    # block IPNUM
/usr/sbin/netstat -rn | grep IPNUM                                   # show routing table
/usr/bin/sudo /sbin/route delete IPNUM 127.0.0.1                     # undo blocking


function blocksite() {
   declare ipaddr ipnum
   if [[ "${1//localhost/}" == '' ]] || [[ "${1//127.0.0.1/}" == '' ]]; then 
      printf "%s\n" 'Argument "localhost" is not permitted!'
      return

CodeSnippets

Related Tags

Never been to CodeSnippets before?

On This Page:

Batch download code snippets

Ping a port with hping3

Analyze internet traffic volume with dynamic ipfw rules

Block internet traffic to specified web sites on the fly

Joyent

CodeSnippets

Related Tags

Never been to CodeSnippets before?

On This Page:

Batch download code snippets

Ping a port with hping3

Analyze internet traffic volume with dynamic ipfw rules

Block internet traffic to specified web sites on the fly