igor@0: #!/bin/sh igor@0: cat < /dev/null igor@0: igor@0: Поддержка нескольких языков: igor@0: igor@0: new-words -l lang URL igor@0: igor@0: Например, для немецких текстов: igor@0: igor@0: new-words -l de URL igor@0: igor@0: Или, предварительно создав соответствующую ссылку: igor@0: igor@0: de-words URL igor@0: igor@0: HELP igor@0: igor@0: WORK_DIR=~/.new-words/ igor@0: TEMP1=`mktemp /tmp/news-words-XXXXXXXXXX` igor@0: TEMP2=`mktemp /tmp/news-words-XXXXXXXXXX` igor@5: export ORIGINAL_TEXT=`mktemp /tmp/news-words-XXXXXXXXXX-orig` igor@0: editor=${EDITOR:-vim} igor@0: igor@2: LANGUAGE=en igor@2: my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `" igor@3: if echo "$1" | grep -q http://...wikipedia.org/wiki/ igor@3: then igor@3: LANGUAGE="`echo $1 | sed s@http://@@ | sed s@.wikipedia.*@@`" igor@3: fi igor@2: [ "${my_name}" = "new" ] || LANGUAGE="$my_name" igor@0: if [ "$1" = "-l" ] igor@0: then igor@0: LANGUAGE="$2" igor@0: VOCABULARY="$LANGUAGE".txt igor@0: shift 2 igor@0: fi igor@2: VOCABULARY=${LANGUAGE}.txt igor@2: NOTES_FILE=notes-${LANGUAGE}.txt igor@0: igor@0: get_words() igor@0: { igor@1: tr ' ' '\n' | sed 's/--/ /g' \ igor@3: | sed "s/'/__APOSTROPHE__/g" \ igor@3: | tr '—·-' '-----' \ igor@3: | tr '*\r,.:#@()+=—<>$;"?!|·[]^%&' ' ' \ igor@3: | tr ' ' '\n' \ igor@3: | grep -x '[[:alpha:]'"'"'-]*' \ igor@3: | tee "$1" \ igor@3: | grep_v_english_perl \ igor@0: | sort | uniq -c | awk '{if ($2!="") print;}' | sort -rn igor@0: } igor@0: igor@3: add_stat() igor@3: { igor@3: before="$1" igor@3: after=${before}2 igor@3: cat > "$after" igor@3: total="`wc -w $1 | awk '{print $1}'`" igor@3: total_unknown="`cat $after|awk '{s=s+$1}END{print s}'`" igor@3: total_known="`echo $total-$total_unknown|bc`" igor@3: percentage="`echo '100*('$total-$total_unknown')'/$total | bc -l | sed 's/\\.\(.\).*/.\1/'`" igor@3: echo "# $LANGUAGE, $percentage, <$total_known/$total>" igor@14: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@14: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@14: my $total=shift(@ARGV); igor@14: my $total_known=shift(@ARGV); igor@14: my $s=0; igor@14: my $mark_line=int($total_known*100/$total/5)*5+5; igor@14: while(<>) igor@14: { igor@14: print; igor@14: /^\s*([0-9]*)\s*/; igor@14: $s+=$1; igor@14: if (int(($total_known+$s)*100/$total)>=$mark_line) { igor@14: print "# $mark_line\n"; igor@14: if ($mark_line>=90) { $mark_line+=1; } else { $mark_line +=5; }; igor@14: } igor@14: } igor@14: PERL_SCRIPT igor@14: perl $PERL_SCRIPT_TEMP_NAME "$total" "$total_known" "$after" igor@14: rm $PERL_SCRIPT_TEMP_NAME igor@3: rm $after igor@3: } igor@3: igor@0: grep_v_english() igor@0: { igor@0: [ -e "$VOCABULARY" ] || touch "$VOCABULARY" igor@0: eval $(cat $VOCABULARY | tr -d "'" | xargs -n10 echo | tr ' ' '|' | sed 's/^/egrep -xv "RRRRRRR|/' | sed 's/$/"/' | tr '\n' '|')cat igor@0: } igor@0: igor@0: grep_v_english_perl() igor@0: { igor@0: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@0: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@0: open(VOC, $ENV{VOCABULARY}) igor@0: or die "Can't open VOCABULARY"; igor@0: while (){ igor@0: chomp; igor@3: #s/'//g; igor@0: $voc{$_}="1"; igor@0: } igor@0: while(<>) { igor@0: chomp; igor@0: if (not defined($voc{$_})) { print "$_\n"; } igor@0: } igor@0: PERL_SCRIPT igor@0: [ -e "$VOCABULARY" ] || touch "$VOCABULARY" igor@0: export VOCABULARY igor@0: perl $PERL_SCRIPT_TEMP_NAME igor@0: rm $PERL_SCRIPT_TEMP_NAME igor@0: } igor@0: igor@11: group_words() igor@11: { igor@11: if [ "$LANGUAGE" != "en" ] igor@11: then igor@11: cat igor@11: return igor@11: fi igor@11: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-group-words-XXXXXXXX` igor@11: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@11: #!/usr/bin/perl igor@11: igor@12: eval { igor@12: # http://stackoverflow.com/questions/251694/how-can-i-check-if-i-have-a-perl-module-before-using-it igor@12: require String::Similarity; igor@12: String::Similarity->import(); igor@12: }; igor@12: unless($@) igor@12: { igor@12: our $HAVE_String_Similarity=1; igor@12: } igor@12: igor@12: sub similar($$){ igor@12: my $a=shift; igor@12: my $b=shift; igor@12: if ($HAVE_String_Similarity) { igor@12: return $Similarity{"$a $b"}; igor@12: } igor@12: else { igor@12: return 0; igor@12: } igor@12: } igor@12: igor@11: sub normalize($) igor@11: { igor@11: $_=lc(shift); igor@12: igor@11: s/s$//; igor@11: s/ed$//; igor@12: s/en$//; igor@12: s/er$//; igor@12: s/est$//; igor@11: s/ing$//; igor@12: igor@12: s/ism$//; s/ist$//; s/ful$//; s/able$//; s/ably$//; igor@12: s/ation$//; s/ness$//; s/ally$//; igor@12: s/ment$//; s/ify$//; s/ity$//; s/fy$//; s/ly$//; igor@12: s/ise$//; s/ize$//; igor@12: igor@12: s/e$//; igor@11: return $_; igor@11: } igor@11: igor@12: igor@11: sub compare($$) igor@11: { igor@11: my $a=shift; igor@11: my $b=shift; igor@11: $a =~ s/^\s*//; igor@11: $b =~ s/^\s*//; igor@11: my ($a1, $a2)= split /\s+/,$a,2; igor@11: my ($b1, $b2)= split /\s+/,$b,2; igor@11: igor@11: my $cmp = $group_weight{normalize($a2)} <=> $group_weight{normalize($b2)}; igor@12: igor@11: if ($cmp) { igor@11: return $cmp; igor@11: } igor@11: else { igor@11: if (normalize($a2) ne normalize($b2)) { igor@11: return normalize($a2) cmp normalize($b2); igor@11: } igor@11: else { igor@11: return $a1 <=> $b1; igor@11: } igor@11: } igor@11: } igor@11: igor@11: @lines=<>; igor@11: for $L (@lines) { igor@11: chomp($L); igor@11: $l=$L; igor@11: $l =~ s/^\s*//; igor@11: my ($a, $b)=split(/\s+/,$l,2); igor@11: $group_weight{normalize($b)}+=$a; igor@11: } igor@11: @lines2 = sort { compare($b,$a) } @lines; igor@11: for $l (@lines2) { igor@11: print "$l\n"; igor@11: } igor@11: PERL_SCRIPT igor@11: perl $PERL_SCRIPT_TEMP_NAME igor@11: rm $PERL_SCRIPT_TEMP_NAME igor@11: } igor@11: igor@0: text_from_url() igor@0: { igor@0: lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@' igor@0: } igor@0: igor@2: add_marks() igor@2: { igor@2: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@2: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@2: $file = $ARGV[0]; igor@2: our $dict; igor@2: if (open(NOTES, $ENV{NOTES_FILE})) { igor@2: while() { igor@2: chomp; igor@2: s/^\s+//; igor@2: my ($a,$b)=split /\s+/,$_,2; igor@2: $dict{$a}=$b; igor@2: } igor@2: } igor@2: if (open(F, $file)) { igor@2: @lines=; igor@2: close(F); igor@2: igor@2: if (open(F, ">$file")) { igor@2: for (@lines) { igor@2: m/\s+\S+\s+(\S+)/; igor@2: $name=$1; igor@3: if (not /^#/ and defined($dict{$name})) { igor@2: chomp; igor@2: $mark=$dict{$name}; igor@2: $space=" "x(30-length($_)); igor@2: print F "$_$space$mark\n"; igor@2: } igor@2: else { igor@2: print F "$_"; igor@2: } igor@2: } igor@2: close(F); igor@2: } igor@2: } igor@2: PERL_SCRIPT igor@2: [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE" igor@2: export NOTES_FILE igor@2: perl $PERL_SCRIPT_TEMP_NAME "$1" igor@2: rm $PERL_SCRIPT_TEMP_NAME igor@2: } igor@2: igor@2: remove_marks() igor@2: { igor@2: PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` igor@2: cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME igor@2: $file = $ARGV[0]; igor@2: our %dict; igor@2: if (open(F, $file)) { igor@2: @lines=; igor@2: close(F); igor@2: igor@2: if (open(F, ">$file")) { igor@2: for (@lines) { igor@2: chomp; igor@3: if (not /^#/ and m/(\s+)(\S+)(\s+)(\S+)(\s+)(.*)/) { igor@2: my $name=$4; igor@2: my $comment=$6; igor@2: $dict{$name}=$comment; igor@2: print F "$1$2$3$4\n"; igor@2: } igor@2: else { igor@2: print F "$_\n"; igor@2: } igor@2: } igor@2: } igor@2: } igor@2: if (open(NOTES, $ENV{NOTES_FILE})) { igor@2: @lines=; igor@2: close(NOTES); igor@2: igor@2: if (open(NOTES, ">".$ENV{NOTES_FILE})) { igor@2: for (@lines) { igor@2: chomp; igor@2: s/^\s+//; igor@2: my ($a,$b)=split /\s+/,$_,2; igor@2: if (not defined($dict{$a}) || ($dict{$a} eq $b)) { igor@2: print NOTES "$_\n"; igor@2: if (defined($dict{$a})) { unset($dict{$a}); } igor@2: } igor@2: } igor@2: for (keys %dict) { igor@2: $mark=$dict{$_}; igor@2: $space=" "x(30-length($_)); igor@2: print NOTES "$_$space$mark\n"; igor@2: } igor@2: } igor@2: } igor@2: PERL_SCRIPT igor@2: [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE" igor@2: export NOTES_FILE igor@2: perl $PERL_SCRIPT_TEMP_NAME "$1" igor@2: rm $PERL_SCRIPT_TEMP_NAME igor@2: } igor@2: igor@0: mkdir -p $WORK_DIR igor@8: oldpwd="$PWD" igor@0: cd $WORK_DIR igor@0: if echo "$1" | grep -q http: igor@0: then igor@11: text_from_url "$1" igor@0: elif [ "$#" != 0 ] igor@0: then igor@13: if echo $1 | grep -q ^/ igor@13: then igor@13: cat "$1" igor@13: else igor@13: cat "$oldpwd/$1" igor@13: fi igor@0: else igor@11: cat igor@11: fi \ igor@11: | tee $ORIGINAL_TEXT \ igor@11: | get_words ${TEMP1}-full \ igor@11: | group_words \ igor@11: | add_stat ${TEMP1}-full \ igor@11: | tee "$TEMP1" > "$TEMP2" igor@0: igor@2: add_marks "$TEMP2" igor@0: if [ "$editor" = vim ] igor@0: then igor@0: vim -c 'set keywordprg='"$LANGUAGE" -c 'set iskeyword=@,48-57,/,.,-,_,+,,,#,$,%,~,=' "$TEMP2" < /dev/tty > /dev/tty igor@0: else igor@0: echo 2 igor@0: $editor "$TEMP2" igor@0: fi igor@2: remove_marks "$TEMP2" igor@2: igor@0: diff "$TEMP1" "$TEMP2" | awk '{print $3}' | sort -u >> "$VOCABULARY" igor@5: rm -f "$TEMP1" "$TEMP2" "${TEMP1}-full" "$ORIGINAL_TEXT"