igor@0: #!/bin/sh igor@0: igor@0: cat < /dev/null igor@0: igor@0: Поддержка нескольких языков: igor@0: igor@0: new-words -l lang URL igor@0: igor@0: Например, для немецких текстов: igor@0: igor@0: new-words -l de URL igor@0: igor@0: Или, предварительно создав соответствующую ссылку: igor@0: igor@0: de-words URL igor@0: igor@0: HELP igor@0: igor@0: WORK_DIR=~/.new-words/ igor@0: TEMP1=`mktemp /tmp/news-words-XXXXXXXXXX` igor@0: TEMP2=`mktemp /tmp/news-words-XXXXXXXXXX` igor@0: editor=${EDITOR:-vim} igor@0: igor@2: LANGUAGE=en igor@2: my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `" igor@2: [ "${my_name}" = "new" ] || LANGUAGE="$my_name" igor@0: if [ "$1" = "-l" ] igor@0: then igor@0: LANGUAGE="$2" igor@0: VOCABULARY="$LANGUAGE".txt igor@0: shift 2 igor@0: fi igor@2: VOCABULARY=${LANGUAGE}.txt igor@2: NOTES_FILE=notes-${LANGUAGE}.txt igor@0: igor@0: get_words() igor@0: { igor@1: tr ' ' '\n' | sed 's/--/ /g' \ igor@0: | tr -d '*\r,.-:#@()+=—<>$;"?!|·[]^%&'"'" \ igor@0: | tr ' ' '\n' | grep_v_english_perl \ igor@1: | grep -x '[[:alnum:]]*' \ igor@0: | sort | uniq -c | awk '{if ($2!="") print;}' | sort -rn igor@0: } igor@0: igor@0: grep_v_english() igor@0: { igor@0: [ -e "$VOCABULARY" ] || touch "$VOCABULARY" igor@0: eval $(cat $VOCABULARY | tr -d "'" | xargs -n10 echo | tr ' ' '|' | sed 's/^/egrep -xv "RRRRRRR|/' | sed 's/$/"/' | tr '\n' '|')cat igor@0: } igor@0: igor@0: grep_v_english_perl() igor@0: { igor@0: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@0: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@0: open(VOC, $ENV{VOCABULARY}) igor@0: or die "Can't open VOCABULARY"; igor@0: while (){ igor@0: chomp; igor@0: s/'//g; igor@0: $voc{$_}="1"; igor@0: } igor@0: while(<>) { igor@0: chomp; igor@0: if (not defined($voc{$_})) { print "$_\n"; } igor@0: } igor@0: PERL_SCRIPT igor@0: [ -e "$VOCABULARY" ] || touch "$VOCABULARY" igor@0: export VOCABULARY igor@0: perl $PERL_SCRIPT_TEMP_NAME igor@0: rm $PERL_SCRIPT_TEMP_NAME igor@0: } igor@0: igor@0: text_from_url() igor@0: { igor@0: lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@' igor@0: } igor@0: igor@2: add_marks() igor@2: { igor@2: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@2: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@2: $file = $ARGV[0]; igor@2: our $dict; igor@2: if (open(NOTES, $ENV{NOTES_FILE})) { igor@2: while() { igor@2: chomp; igor@2: s/^\s+//; igor@2: my ($a,$b)=split /\s+/,$_,2; igor@2: $dict{$a}=$b; igor@2: } igor@2: } igor@2: if (open(F, $file)) { igor@2: @lines=; igor@2: close(F); igor@2: igor@2: if (open(F, ">$file")) { igor@2: for (@lines) { igor@2: m/\s+\S+\s+(\S+)/; igor@2: $name=$1; igor@2: if (defined($dict{$name})) { igor@2: chomp; igor@2: $mark=$dict{$name}; igor@2: $space=" "x(30-length($_)); igor@2: print F "$_$space$mark\n"; igor@2: } igor@2: else { igor@2: print F "$_"; igor@2: } igor@2: } igor@2: close(F); igor@2: } igor@2: } igor@2: PERL_SCRIPT igor@2: [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE" igor@2: export NOTES_FILE igor@2: perl $PERL_SCRIPT_TEMP_NAME "$1" igor@2: rm $PERL_SCRIPT_TEMP_NAME igor@2: } igor@2: igor@2: remove_marks() igor@2: { igor@2: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@2: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@2: $file = $ARGV[0]; igor@2: our %dict; igor@2: if (open(F, $file)) { igor@2: @lines=; igor@2: close(F); igor@2: igor@2: if (open(F, ">$file")) { igor@2: for (@lines) { igor@2: chomp; igor@2: if (m/(\s+)(\S+)(\s+)(\S+)(\s+)(.*)/) { igor@2: my $name=$4; igor@2: my $comment=$6; igor@2: $dict{$name}=$comment; igor@2: print F "$1$2$3$4\n"; igor@2: } igor@2: else { igor@2: print F "$_\n"; igor@2: } igor@2: } igor@2: } igor@2: } igor@2: if (open(NOTES, $ENV{NOTES_FILE})) { igor@2: @lines=; igor@2: close(NOTES); igor@2: igor@2: if (open(NOTES, ">".$ENV{NOTES_FILE})) { igor@2: for (@lines) { igor@2: chomp; igor@2: s/^\s+//; igor@2: my ($a,$b)=split /\s+/,$_,2; igor@2: if (not defined($dict{$a}) || ($dict{$a} eq $b)) { igor@2: print NOTES "$_\n"; igor@2: if (defined($dict{$a})) { unset($dict{$a}); } igor@2: } igor@2: } igor@2: for (keys %dict) { igor@2: $mark=$dict{$_}; igor@2: $space=" "x(30-length($_)); igor@2: print NOTES "$_$space$mark\n"; igor@2: } igor@2: } igor@2: } igor@2: PERL_SCRIPT igor@2: [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE" igor@2: export NOTES_FILE igor@2: perl $PERL_SCRIPT_TEMP_NAME "$1" igor@2: rm $PERL_SCRIPT_TEMP_NAME igor@2: } igor@2: igor@0: mkdir -p $WORK_DIR igor@0: cd $WORK_DIR igor@0: if echo "$1" | grep -q http: igor@0: then igor@0: text_from_url "$1" | get_words | tee "$TEMP1" > "$TEMP2" igor@0: elif [ "$#" != 0 ] igor@0: then igor@0: cat "$1" | get_words | tee "$TEMP1" > "$TEMP2" igor@0: else igor@0: get_words | tee "$TEMP1" > "$TEMP2" igor@0: fi igor@0: igor@2: add_marks "$TEMP2" igor@0: if [ "$editor" = vim ] igor@0: then igor@0: vim -c 'set keywordprg='"$LANGUAGE" -c 'set iskeyword=@,48-57,/,.,-,_,+,,,#,$,%,~,=' "$TEMP2" < /dev/tty > /dev/tty igor@0: else igor@0: echo 2 igor@0: $editor "$TEMP2" igor@0: fi igor@2: remove_marks "$TEMP2" igor@2: igor@0: diff "$TEMP1" "$TEMP2" | awk '{print $3}' | sort -u >> "$VOCABULARY" igor@0: rm -f "$TEMP1" "$TEMP2"