#!/bin/bash
user=""
pass=""

tmp_dir=tmp_dir_$$
mkdir $tmp_dir

page_size=20

echo "connecting to duolingo..." 1>&2
result=`curl -s -d "login=$user&password;=$pass" --dump-header $tmp_dir/headers http://duolingo.com/login | grep "\"response\": \"OK\""` 
if [[ -z "$result" ]]; then
    echo "connection error" 1>&2
    exit -1
fi
echo "getting word count..." 1>&2
curl -s -L -b $tmp_dir/headers http://duolingo.com/words > $tmp_dir/words1.json
count=$(cat $tmp_dir/words1.json | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["vocab_count"]')
echo "downloading $count words..." 1>&2
pages=$(((count/$page_size)+1))
tmp_file=unsorted_list.txt
for x in `seq 1 1 $pages`;do
    if [ "$x" -gt "1" ]; then
        curl -s -L -b $tmp_dir/headers http://duolingo.com/words?page=$x > $tmp_dir/words$x.json
    fi
    limit=`grep -roh \"pos\" $tmp_dir/words$x.json  | wc -l`
    for y in `seq 0 1 $(($limit-1))`;do
        pos=`cat $tmp_dir/words$x.json | python -c "import json,sys;obj=json.load(sys.stdin);print obj['vocab'][$y]['pos'].encode('utf-8')"` 2>/dev/null
        surface=`cat $tmp_dir/words$x.json | python -c "import json,sys;obj=json.load(sys.stdin);print obj['vocab'][$y]['surface_form'].encode('utf-8')"` 2>/dev/null
        echo "$pos $surface" >> $tmp_dir/$tmp_file
    done
done;
cat $tmp_dir/$tmp_file | sort
rm -r $tmp_dir