igor@38: #!/bin/bash
igor@38: 
igor@38: show_usage()
igor@38: {
igor@38: cat <<HELP > /dev/stderr
igor@38: 
igor@38: USAGE: 
igor@38: 
igor@38:     new-words [ -l lang ] [ -s ] [ ARG ] 
igor@38: 
igor@38: SWITCHES: 
igor@38: 
igor@38:     -h          print this screen
igor@38:     -c          show compressed wordlist: one word per group
igor@38:     -k          put higher words that are similar to the known words (only for English)
igor@38:     -l lang     override language settings
igor@38:     -n          non-interactive mode (don't run vi)
igor@38:     -N          turn off known words filtering
igor@38:     -a          don't add marks (and don't save marks added by user)
igor@38:     -p pages    work with specified pages only (pages = start-stop/total )
igor@38:     -s          show the text statistics (percentage of known words and so on) and exit
igor@38:     -S          show your vocabulary statistics (number of words and word groups)
igor@38:     -t tag      tag known words with tag
igor@38:     -T          show list of active tags
igor@38:     -m tag      merge the words tagged with "tag" into the main vocabulary
igor@38:     -M          merge the words tagged with any tag into the main vocabulary 
igor@38:     -r tag      remove subvocabulary for the "tag"
igor@38:     -2 -3       find 2 and 3 words' sequences
igor@38: 
igor@38: The language of the text can be specified also
igor@38: by name of the program new-words (correspondent link must be created before).
igor@38: For example, these calls are equivalent:
igor@38: 
igor@38:     de-words URL
igor@38:     new-words -l de URL
igor@38: 
igor@38: HELP
igor@38: }
igor@38: 
igor@38: if [ "$1" = "-h" ]
igor@38: then
igor@38:     show_usage
igor@38:     exit 0
igor@38: fi
igor@38: 
igor@38: NEW_WORDS_PY=/home/igor/hg/new-words/new-words.py
igor@38: WORK_DIR=~/.new-words/
igor@38: TEMP1=`mktemp /tmp/new-words-temp1.XXXXXXXXXX`
igor@38: TEMP2=`mktemp /tmp/new-words-temp2.XXXXXXXXXX`
igor@38: export ORIGINAL_TEXT=`mktemp /tmp/new-words-orig.XXXXXXXXXX`
igor@38: editor=${EDITOR:-vim}
igor@38: 
igor@38: # language detection
igor@38: 
igor@38: LANGUAGE=en
igor@38: my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `"
igor@38: for arg
igor@38: do
igor@38:     if echo "$arg" | grep -q http://...wikipedia.org/wiki/
igor@38:     then
igor@38:     LANGUAGE="`echo $arg | sed s@http://@@ | sed s@.wikipedia.*@@`"
igor@38:     fi
igor@38: done
igor@38: [ "${my_name}" = "new" ] || LANGUAGE="$my_name"
igor@38: 
igor@38: #----------------------------------------------------
igor@38: # command line options processing
igor@38: 
igor@38: STAT_ONLY=NO
igor@38: NEED_TO_USE_VOCABULARY_WHEN_SORT=NO
igor@38: DONT_ADD_MARKS=NO
igor@38: NON_INTERACTIVE_MODE=NO
igor@38: PART_TO_PROCESS=''
igor@38: GROUP_WORDS_BY_THREE=NO
igor@38: GROUP_WORDS_BY_TWO=NO
igor@38: TAG_NAME=''
igor@38: MERGE_THIS_TAGS=''
igor@38: TAGS_LIST_ONLY=NO
igor@38: MERGE_TAGGED_WORDS=NO
igor@38: MERGE_ALL_TAGGED=NO
igor@38: DONT_ADD_MARKLINES=NO
igor@38: FILTER_WORDS=YES
igor@38: SHOW_VOC_STAT=NO
igor@38: COMPRESSED_WORDLIST=NO
igor@40: while getopts cl:sSkanNp:t:Tm:Mr:23 opt
igor@38: do
igor@38:     case "$opt" in
igor@38:       c)  COMPRESSED_WORDLIST=YES;;
igor@38:       s)  STAT_ONLY=YES;;
igor@38:       S)  SHOW_VOC_STAT=YES;;
igor@38:       k)  NEED_TO_USE_VOCABULARY_WHEN_SORT=YES;;
igor@38:       l)  LANGUAGE="$OPTARG";;
igor@38:       a)  DONT_ADD_MARKS=YES;;
igor@38:       n)  NON_INTERACTIVE_MODE=YES;;
igor@38:       N)  FILTER_WORDS=NO;;
igor@38:       p)  PART_TO_PROCESS="$OPTARG";;
igor@38:       t)  TAG_NAME="$OPTARG";;
igor@38:       T)  TAGS_LIST_ONLY="YES";;
igor@38:       m)  DONT_ADD_MARKLINES="YES"; MERGE_TAGGED_WORDS="YES"; MERGE_THIS_TAGS="$TAG_NAME $OPTARG";;
igor@38:       M)  DONT_ADD_MARKLINES="YES"; MERGE_ALL_TAGGED="YES";;
igor@38:       r)  REMOVE_TAG="YES"; TAG_NAME="$TAG_NAME $OPTARG";;
igor@38:       2)  GROUP_WORDS_BY_TWO=YES;;
igor@38:       3)  GROUP_WORDS_BY_THREE=YES;;
igor@38:       \?)       # unknown flag
igor@38:           show_usage
igor@38:           exit 1;;
igor@38:     esac
igor@38: done
igor@38: shift `expr $OPTIND - 1`
igor@38: 
igor@38: if [ "$1" = "-l" ]
igor@38: then
igor@38:     LANGUAGE="$2"
igor@38:     shift 2
igor@38: fi
igor@38: 
igor@38: VOCABULARY=${LANGUAGE}.txt
igor@38: NOTES_FILE=notes-${LANGUAGE}.txt
igor@38: 
igor@38: if [ "${SHOW_VOC_STAT}" = "YES" ]
igor@38: then
igor@38:   $0 -l "${LANGUAGE}" -N -n ${WORK_DIR}/${VOCABULARY} | head -1 | awk '{print $5}' | tr -d "<>"
igor@38:   exit 0
igor@38: fi
igor@38: 
igor@38: text_from_url()
igor@38: {
igor@38: lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@'
igor@38: }
igor@38: 
igor@38: add_marks()
igor@38: {
igor@40:     $NEW_WORDS_PY -l "$LANGUAGE" -f add_notes "$1" 
igor@39: }
igor@40: remove_marks()  
igor@39: {
igor@40:     $NEW_WORDS_PY -l "$LANGUAGE" -f remove_notes "$1"
igor@40: }
igor@40: get_words_group_words_add_stat()
igor@38: {
igor@44:     STAT_ONLY="$STAT_ONLY" \
igor@44:     GROUP_WORDS_BY_TWO="$GROUP_WORDS_BY_TWO" \
igor@44:     GROUP_WORDS_BY_THREE="$GROUP_WORDS_BY_THREE" \
igor@40:     $NEW_WORDS_PY -l "$LANGUAGE" -f get_words_group_words_add_stat "$1"
igor@38: }
igor@38: 
igor@38: part()
igor@38: {
igor@38:     PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-part-XXXXXXXX`
igor@38:     cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
igor@38: #!/usr/bin/perl
igor@38: 
igor@38: my @lines=<STDIN>;
igor@38: my $lines=$#lines;
igor@38: my $interval=$ARGV[0];
igor@38: if (not $interval) {
igor@38:     print @lines;
igor@38: }
igor@38: else {
igor@38:     my ($start,$stop,$total);
igor@38:     if ($interval =~ m@(.*)/(.*)@) {
igor@38:         $start = $1;
igor@38:         $total = $2;
igor@38:     }
igor@38:     else {
igor@38:         $start=$interval;
igor@38:         $total=0;
igor@38:     }
igor@38:     if ($start =~ m@(.*)-(.*)@) {
igor@38:         $start = $1;
igor@38:         $stop = $2;
igor@38:     }
igor@38:     if ($start =~ m@(.*)\+(.*)@) {
igor@38:         $start = $1;
igor@38:         $stop = $start+$2;
igor@38:     }
igor@38: 
igor@38:     $start=int($lines/$total*$start);
igor@38:     $stop=int($lines/$total*$stop);
igor@38: 
igor@38:     for($i=$start;$i<$stop;$i++){
igor@38:         print $lines[$i];
igor@38:     }
igor@38: }
igor@38: PERL_SCRIPT
igor@38:     perl $PERL_SCRIPT_TEMP_NAME "$1"
igor@38:     rm $PERL_SCRIPT_TEMP_NAME
igor@38: }
igor@38: 
igor@38: if [ "$TAGS_LIST_ONLY" = "YES" ] 
igor@38: then
igor@38:     cd "${WORK_DIR}"
igor@38:     echo ${LANGUAGE}_*.txt | tr ' ' '\n'  | grep -v '*' | sed 's/[^_]*_//;s/.txt$//'
igor@38:     exit 0
igor@38: fi
igor@38: 
igor@38: tag_file_name()
igor@38: {
igor@38:     echo "${LANGUAGE}_${1}.txt"
igor@38: }
igor@38: 
igor@38: if [ "$REMOVE_TAG" = "YES" ]
igor@38: then
igor@38:     cd "${WORK_DIR}"
igor@38:     for i in $TAG_NAME 
igor@38:     do
igor@38:         echo "$TAGNAME" | grep -q '[/*?]' && continue
igor@38:         f="`tag_file_name $i`"
igor@38:         if [ -e "$f" ] 
igor@38:         then
igor@38:             rm -f "$f" && echo Tag "'$i'" removed
igor@38:         else
igor@38:             echo Unknown tag "'$i'"
igor@38:         fi
igor@38:     done
igor@38:     exit 0
igor@38: fi
igor@38: 
igor@38: mkdir -p $WORK_DIR
igor@38: oldpwd="$PWD"
igor@38: cd $WORK_DIR
igor@38: if [ "$MERGE_TAGGED_WORDS" = "YES" ]
igor@38: then
igor@38:     VOC_FILES=''
igor@38:     for i in $MERGE_THIS_TAGS
igor@38:     do
igor@38:         f=`tag_file_name $i`
igor@38:         [ -e "$f" ] && VOC_FILES="${VOC_FILES} $f"
igor@38:     done
igor@38:     if [ -z "$VOC_FILES" ]
igor@38:     then 
igor@38:         echo Unknown tags: $MERGE_THIS_TAGS > /dev/stderr
igor@38:     else
igor@38:         cat $VOC_FILES
igor@38:     fi
igor@38: elif [ "$MERGE_ALL_TAGGED" = "YES" ]
igor@38: then
igor@38:     cat ${LANGUAGE}_*.txt
igor@38: elif echo "$1" | grep -q http: 
igor@38: then 
igor@38:     text_from_url "$1"
igor@38: elif [ "$#" != 0 ]
igor@38: then
igor@38:     if echo $1 | grep -q ^/
igor@38:     then
igor@38:         cat "$1"
igor@38:     else
igor@38:         cat "$oldpwd/$1"
igor@38:     fi
igor@38: else 
igor@38:     cat
igor@38: fi \
igor@38:    | part $PART_TO_PROCESS \
igor@38:    | tee $ORIGINAL_TEXT \
igor@44:    | \
igor@44:     get_words_group_words_add_stat \
igor@38:    | tee "$TEMP1" > "$TEMP2"
igor@38: 
igor@38: if [ "$STAT_ONLY" = "YES" ]
igor@38: then
igor@38:     cat "$TEMP1"
igor@38: elif [ "$NON_INTERACTIVE_MODE" = "YES" ]
igor@38: then
igor@38:     cat "$TEMP1"
igor@38: else
igor@38:     if [ `wc -l "$TEMP2" | awk '{print $1}'` != 0 ] 
igor@38:     then
igor@38:         [ "$DONT_ADD_MARKS" = "YES" ] || add_marks "$TEMP2"
igor@38:         if [ "$editor" = vim ]
igor@38:         then
igor@38:             vim -c 'set keywordprg='"$LANGUAGE" -c 'set iskeyword=@,48-57,/,.,-,_,+,,,#,$,%,~,=,48-255' "$TEMP2" < /dev/tty > /dev/tty
igor@38:         else
igor@38:             $editor "$TEMP2"
igor@38:         fi
igor@38:         remove_marks "$TEMP2"
igor@38: 
igor@38:         vocabulary="$VOCABULARY"
igor@38:         [ -n "$TAG_NAME" ] && vocabulary="`tag_file_name $TAG_NAME`"
igor@38:         diff "$TEMP1" "$TEMP2" | awk '{print $3}' | sort -u >> "$vocabulary"
igor@38:     fi
igor@38: fi
igor@38: 
igor@38: rm -f "$TEMP1" "$TEMP2" "${TEMP1}-full" "$ORIGINAL_TEXT"