#!/bin/bash

# This script lives at http://www.elifulkerson.com

# We are, of course, IE6 under Windows XP
USERAGENT="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)"

# Adult version
# DOGPILE="lynx --dump http://www.dogpile.com/info.dogpl/searchspy/inc/data.xml"

# Safe version
DOGPILE="http://www.dogpile.com/info.dogpl/searchspy/inc/data.xml?filter=1";

# default output file is 'dogpile.out'
OUTFILE="dogpile.out"

# Check to make sure that we have our necessary bits
if which lynx > /dev/null; then
    echo -n
else
    echo Error, 'lynx' is required.
    exit 1;
fi

if which sed > /dev/null; then
    echo -n
else
    echo Error, 'sed' is required.
    exit 1;
fi

if which date > /dev/null; then
    echo -n
else
    echo Error, 'date' is required.
    exit 1;
fi

echo Retriever:  Digging for bones.  Output appends to $OUTFILE.mm_dd_yy

while [ 1 -eq 1 ]; do
    lynx --dump -useragent=$USERAGENT $DOGPILE 2> /dev/null | sed -e :a -e 's/<[^>]*>/\n/g;/</N;//ba'|  sed '/^$/d' >> $OUTFILE.`date +%m_%d_%y`
    sleep 3;
done;