1.1 --- a/new-words.sh	Wed Mar 31 22:38:03 2010 +0300
     1.2 +++ b/new-words.sh	Fri Apr 02 19:46:44 2010 +0300
     1.3 @@ -92,6 +92,67 @@
     1.4      rm $PERL_SCRIPT_TEMP_NAME
     1.5  }
     1.6  
     1.7 +group_words()
     1.8 +{
     1.9 +    if [ "$LANGUAGE" != "en" ]
    1.10 +    then
    1.11 +        cat 
    1.12 +        return
    1.13 +    fi
    1.14 +    PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-group-words-XXXXXXXX`
    1.15 +    cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
    1.16 +#!/usr/bin/perl
    1.17 +
    1.18 +sub normalize($)
    1.19 +{
    1.20 +    $_=lc(shift);
    1.21 +    s///;
    1.22 +    s/s$//;
    1.23 +    s/ed$//;
    1.24 +    s/ing$//;
    1.25 +    return $_;
    1.26 +}
    1.27 +
    1.28 +sub compare($$)
    1.29 +{
    1.30 +    my $a=shift;
    1.31 +    my $b=shift;
    1.32 +    $a =~ s/^\s*//;
    1.33 +    $b =~ s/^\s*//;
    1.34 +    my ($a1, $a2)= split /\s+/,$a,2;
    1.35 +    my ($b1, $b2)= split /\s+/,$b,2;
    1.36 +
    1.37 +    my $cmp = $group_weight{normalize($a2)} <=> $group_weight{normalize($b2)};
    1.38 +    if ($cmp) {
    1.39 +        return $cmp;
    1.40 +    }
    1.41 +    else {
    1.42 +        if (normalize($a2) ne normalize($b2)) {
    1.43 +            return normalize($a2) cmp normalize($b2);
    1.44 +        }
    1.45 +        else {
    1.46 +            return $a1 <=> $b1;
    1.47 +        }
    1.48 +    }
    1.49 +}
    1.50 +
    1.51 +@lines=<>;
    1.52 +for $L (@lines) {
    1.53 +    chomp($L);
    1.54 +    $l=$L;
    1.55 +    $l =~ s/^\s*//;
    1.56 +    my ($a, $b)=split(/\s+/,$l,2);
    1.57 +    $group_weight{normalize($b)}+=$a;
    1.58 +}
    1.59 +@lines2 = sort { compare($b,$a) } @lines;
    1.60 +for $l (@lines2) {
    1.61 +    print "$l\n";
    1.62 +}
    1.63 +PERL_SCRIPT
    1.64 +    perl $PERL_SCRIPT_TEMP_NAME
    1.65 +    rm $PERL_SCRIPT_TEMP_NAME
    1.66 +}
    1.67 +
    1.68  text_from_url()
    1.69  {
    1.70  lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@'
    1.71 @@ -197,16 +258,18 @@
    1.72  cd $WORK_DIR
    1.73  if echo "$1" | grep -q http: 
    1.74  then 
    1.75 -    text_from_url "$1" | tee $ORIGINAL_TEXT \
    1.76 -    | get_words ${TEMP1}-full | add_stat ${TEMP1}-full| tee "$TEMP1" > "$TEMP2"
    1.77 +    text_from_url "$1"
    1.78  elif [ "$#" != 0 ]
    1.79  then
    1.80 -    cat "$oldpwd/$1" | tee $ORIGINAL_TEXT \
    1.81 -    | get_words ${TEMP1}-full | add_stat ${TEMP1}-full | tee "$TEMP1" > "$TEMP2"
    1.82 +    cat "$oldpwd/$1"
    1.83  else 
    1.84 -    tee $ORIGINAL_TEXT \
    1.85 -    | get_words ${TEMP1}-full | add_stat ${TEMP1}-full | tee "$TEMP1" > "$TEMP2"
    1.86 -fi
    1.87 +    cat
    1.88 +fi \
    1.89 +   | tee $ORIGINAL_TEXT \
    1.90 +   | get_words ${TEMP1}-full \
    1.91 +   | group_words \
    1.92 +   | add_stat ${TEMP1}-full \
    1.93 +   | tee "$TEMP1" > "$TEMP2"
    1.94  
    1.95  add_marks "$TEMP2"
    1.96  if [ "$editor" = vim ]