new-words
annotate new-words.sh @ 43:d532e7b52ab2
-s key support in new-words.py
Now new-words-py.sh -s works in the same way as new-words.sh.
(WPS and UWPS fields are not calculated correctly yet).
Now new-words-py.sh -s works in the same way as new-words.sh.
(WPS and UWPS fields are not calculated correctly yet).
author | Igor Chubin <igor@chub.in> |
---|---|
date | Fri Jan 28 12:40:58 2011 +0200 (2011-01-28) |
parents | 3827cce83602 |
children | 4e931db74618 |
rev | line source |
---|---|
igor@26 | 1 #!/bin/bash |
igor@20 | 2 |
igor@20 | 3 show_usage() |
igor@20 | 4 { |
igor@20 | 5 cat <<HELP > /dev/stderr |
igor@0 | 6 |
igor@16 | 7 USAGE: |
igor@16 | 8 |
igor@16 | 9 new-words [ -l lang ] [ -s ] [ ARG ] |
igor@16 | 10 |
igor@16 | 11 SWITCHES: |
igor@16 | 12 |
igor@20 | 13 -h print this screen |
igor@36 | 14 -c show compressed wordlist: one word per group |
igor@20 | 15 -k put higher words that are similar to the known words (only for English) |
igor@16 | 16 -l lang override language settings |
igor@21 | 17 -n non-interactive mode (don't run vi) |
igor@33 | 18 -N turn off known words filtering |
igor@27 | 19 -a don't add marks (and don't save marks added by user) |
igor@25 | 20 -p pages work with specified pages only (pages = start-stop/total ) |
igor@20 | 21 -s show the text statistics (percentage of known words and so on) and exit |
igor@33 | 22 -S show your vocabulary statistics (number of words and word groups) |
igor@27 | 23 -t tag tag known words with tag |
igor@27 | 24 -T show list of active tags |
igor@27 | 25 -m tag merge the words tagged with "tag" into the main vocabulary |
igor@27 | 26 -M merge the words tagged with any tag into the main vocabulary |
igor@27 | 27 -r tag remove subvocabulary for the "tag" |
igor@26 | 28 -2 -3 find 2 and 3 words' sequences |
igor@16 | 29 |
igor@20 | 30 The language of the text can be specified also |
igor@20 | 31 by name of the program new-words (correspondent link must be created before). |
igor@20 | 32 For example, these calls are equivalent: |
igor@0 | 33 |
igor@20 | 34 de-words URL |
igor@0 | 35 new-words -l de URL |
igor@0 | 36 |
igor@20 | 37 HELP |
igor@20 | 38 } |
igor@0 | 39 |
igor@20 | 40 if [ "$1" = "-h" ] |
igor@20 | 41 then |
igor@20 | 42 show_usage |
igor@20 | 43 exit 0 |
igor@20 | 44 fi |
igor@0 | 45 |
igor@0 | 46 WORK_DIR=~/.new-words/ |
igor@28 | 47 TEMP1=`mktemp /tmp/new-words-temp1.XXXXXXXXXX` |
igor@28 | 48 TEMP2=`mktemp /tmp/new-words-temp2.XXXXXXXXXX` |
igor@28 | 49 export ORIGINAL_TEXT=`mktemp /tmp/new-words-orig.XXXXXXXXXX` |
igor@0 | 50 editor=${EDITOR:-vim} |
igor@0 | 51 |
igor@24 | 52 # language detection |
igor@21 | 53 |
igor@2 | 54 LANGUAGE=en |
igor@2 | 55 my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `" |
igor@22 | 56 for arg |
igor@22 | 57 do |
igor@22 | 58 if echo "$arg" | grep -q http://...wikipedia.org/wiki/ |
igor@22 | 59 then |
igor@22 | 60 LANGUAGE="`echo $arg | sed s@http://@@ | sed s@.wikipedia.*@@`" |
igor@22 | 61 fi |
igor@22 | 62 done |
igor@2 | 63 [ "${my_name}" = "new" ] || LANGUAGE="$my_name" |
igor@24 | 64 |
igor@24 | 65 #---------------------------------------------------- |
igor@24 | 66 # command line options processing |
igor@24 | 67 |
igor@24 | 68 STAT_ONLY=NO |
igor@24 | 69 NEED_TO_USE_VOCABULARY_WHEN_SORT=NO |
igor@24 | 70 DONT_ADD_MARKS=NO |
igor@24 | 71 NON_INTERACTIVE_MODE=NO |
igor@25 | 72 PART_TO_PROCESS='' |
igor@26 | 73 GROUP_WORDS_BY_THREE=NO |
igor@26 | 74 GROUP_WORDS_BY_TWO=NO |
igor@27 | 75 TAG_NAME='' |
igor@27 | 76 MERGE_THIS_TAGS='' |
igor@27 | 77 TAGS_LIST_ONLY=NO |
igor@27 | 78 MERGE_TAGGED_WORDS=NO |
igor@27 | 79 MERGE_ALL_TAGGED=NO |
igor@27 | 80 DONT_ADD_MARKLINES=NO |
igor@33 | 81 FILTER_WORDS=YES |
igor@33 | 82 SHOW_VOC_STAT=NO |
igor@36 | 83 COMPRESSED_WORDLIST=NO |
igor@36 | 84 while getopts cl:sSkanNp:t:Tm:Mr:23 opt |
igor@24 | 85 do |
igor@24 | 86 case "$opt" in |
igor@36 | 87 c) COMPRESSED_WORDLIST=YES;; |
igor@24 | 88 s) STAT_ONLY=YES;; |
igor@33 | 89 S) SHOW_VOC_STAT=YES;; |
igor@24 | 90 k) NEED_TO_USE_VOCABULARY_WHEN_SORT=YES;; |
igor@24 | 91 l) LANGUAGE="$OPTARG";; |
igor@27 | 92 a) DONT_ADD_MARKS=YES;; |
igor@24 | 93 n) NON_INTERACTIVE_MODE=YES;; |
igor@33 | 94 N) FILTER_WORDS=NO;; |
igor@25 | 95 p) PART_TO_PROCESS="$OPTARG";; |
igor@27 | 96 t) TAG_NAME="$OPTARG";; |
igor@27 | 97 T) TAGS_LIST_ONLY="YES";; |
igor@27 | 98 m) DONT_ADD_MARKLINES="YES"; MERGE_TAGGED_WORDS="YES"; MERGE_THIS_TAGS="$TAG_NAME $OPTARG";; |
igor@27 | 99 M) DONT_ADD_MARKLINES="YES"; MERGE_ALL_TAGGED="YES";; |
igor@27 | 100 r) REMOVE_TAG="YES"; TAG_NAME="$TAG_NAME $OPTARG";; |
igor@26 | 101 2) GROUP_WORDS_BY_TWO=YES;; |
igor@26 | 102 3) GROUP_WORDS_BY_THREE=YES;; |
igor@24 | 103 \?) # unknown flag |
igor@24 | 104 show_usage |
igor@24 | 105 exit 1;; |
igor@24 | 106 esac |
igor@24 | 107 done |
igor@24 | 108 shift `expr $OPTIND - 1` |
igor@24 | 109 |
igor@0 | 110 if [ "$1" = "-l" ] |
igor@0 | 111 then |
igor@0 | 112 LANGUAGE="$2" |
igor@0 | 113 shift 2 |
igor@0 | 114 fi |
igor@24 | 115 |
igor@2 | 116 VOCABULARY=${LANGUAGE}.txt |
igor@2 | 117 NOTES_FILE=notes-${LANGUAGE}.txt |
igor@0 | 118 |
igor@33 | 119 if [ "${SHOW_VOC_STAT}" = "YES" ] |
igor@33 | 120 then |
igor@33 | 121 $0 -l "${LANGUAGE}" -N -n ${WORK_DIR}/${VOCABULARY} | head -1 | awk '{print $5}' | tr -d "<>" |
igor@33 | 122 exit 0 |
igor@33 | 123 fi |
igor@33 | 124 |
igor@21 | 125 #---------------------------------------------------- |
igor@21 | 126 |
igor@0 | 127 get_words() |
igor@0 | 128 { |
igor@33 | 129 export FILTER_WORDS |
igor@1 | 130 tr ' ' '\n' | sed 's/--/ /g' \ |
igor@3 | 131 | sed "s/'/__APOSTROPHE__/g" \ |
igor@31 | 132 | perl -MEncode -Mutf8 -n -e '$_ = decode( "utf8", $_);y/*\r,.:#@()+=—<>$;"?!|·[]^%&/ /; binmode STDOUT, ":utf8"; print if /^[[:alpha:] '"'"'_-]*$/'\ |
igor@26 | 133 | sed "s/__APOSTROPHE__/'/g" \ |
igor@3 | 134 | tr ' ' '\n' \ |
igor@3 | 135 | tee "$1" \ |
igor@3 | 136 | grep_v_english_perl \ |
igor@0 | 137 | sort | uniq -c | awk '{if ($2!="") print;}' | sort -rn |
igor@0 | 138 } |
igor@0 | 139 |
igor@3 | 140 add_stat() |
igor@3 | 141 { |
igor@27 | 142 if [ "$DONT_ADD_MARKLINES" = "YES" ] |
igor@27 | 143 then |
igor@27 | 144 cat |
igor@27 | 145 return |
igor@27 | 146 fi |
igor@3 | 147 before="$1" |
igor@3 | 148 after=${before}2 |
igor@3 | 149 cat > "$after" |
igor@3 | 150 total="`wc -w $1 | awk '{print $1}'`" |
igor@3 | 151 total_unknown="`cat $after|awk '{s=s+$1}END{print s}'`" |
igor@3 | 152 total_known="`echo $total-$total_unknown|bc`" |
igor@3 | 153 percentage="`echo '100*('$total-$total_unknown')'/$total | bc -l | sed 's/\\.\(.\).*/.\1/'`" |
igor@18 | 154 #sentences="`cat $after | perl -e 'local $/; $_=<>; s@http://[a-zA-Z&_.:/0-9%?=,\#+()\[\]~-]*@@g; s@\n@@g; s@(Mr|Mrs)\.@\1POINT@g; @sentences=split /\\./;print $#sentences;'`" |
igor@18 | 155 sentences="`cat $ORIGINAL_TEXT | perl -e 'local $/; $_=<>; s/[^.]//msg; print length($_);'`" |
igor@18 | 156 |
igor@18 | 157 |
igor@16 | 158 if [ "$STAT_ONLY" = "YES" ] |
igor@16 | 159 then |
igor@18 | 160 echo "LANG KNOWN% UNKNOWN% KNOWN TOTAL WPS UWPS*10" |
igor@18 | 161 echo "$LANGUAGE $percentage `echo \(100-$percentage\) | bc -l` $total_known $total `echo $total/$sentences|bc` `echo 10*$total_unknown/$sentences|bc` " |
igor@17 | 162 rm $after |
igor@16 | 163 return 0 |
igor@16 | 164 else |
igor@34 | 165 groups="`echo $(grep '# groups' $after | awk '{print $3}')`" |
igor@34 | 166 words="`echo $(grep -v '^#' $after | wc -l)`" |
igor@33 | 167 echo "# $LANGUAGE, $percentage, <$total_known/$total>, <$groups/$words>" |
igor@16 | 168 fi |
igor@16 | 169 |
igor@14 | 170 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` |
igor@14 | 171 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME |
igor@14 | 172 my $total=shift(@ARGV); |
igor@14 | 173 my $total_known=shift(@ARGV); |
igor@14 | 174 my $s=0; |
igor@16 | 175 my $mark_line=int($total_known*100/$total/5)*5; |
igor@19 | 176 if ($mark_line>=90) { |
igor@19 | 177 $mark_line=int($total_known*100/$total)+1; |
igor@19 | 178 } else { $mark_line +=5; }; |
igor@14 | 179 while(<>) |
igor@14 | 180 { |
igor@33 | 181 next if /^#\s*groups\s*/; |
igor@14 | 182 print; |
igor@14 | 183 /^\s*([0-9]*)\s*/; |
igor@14 | 184 $s+=$1; |
igor@19 | 185 if (($total_known+$s)*100/$total>=$mark_line) { |
igor@14 | 186 print "# $mark_line\n"; |
igor@14 | 187 if ($mark_line>=90) { $mark_line+=1; } else { $mark_line +=5; }; |
igor@14 | 188 } |
igor@14 | 189 } |
igor@14 | 190 PERL_SCRIPT |
igor@14 | 191 perl $PERL_SCRIPT_TEMP_NAME "$total" "$total_known" "$after" |
igor@14 | 192 rm $PERL_SCRIPT_TEMP_NAME |
igor@3 | 193 rm $after |
igor@3 | 194 } |
igor@3 | 195 |
igor@26 | 196 two_and_three_words() |
igor@26 | 197 { |
igor@26 | 198 if [ $GROUP_WORDS_BY_THREE = NO -a $GROUP_WORDS_BY_TWO = NO ] |
igor@26 | 199 then |
igor@26 | 200 cat |
igor@26 | 201 else |
igor@26 | 202 cat |
igor@26 | 203 |
igor@26 | 204 export GROUP_WORDS_BY_THREE |
igor@26 | 205 export GROUP_WORDS_BY_TWO |
igor@26 | 206 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-two-and-three-XXXXXXXX` |
igor@26 | 207 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME |
igor@26 | 208 #!/usr/bin/perl |
igor@26 | 209 local $/; |
igor@26 | 210 $words=<>; |
igor@26 | 211 $words=~ s@[!?;,:#1-9".]@ @g; |
igor@26 | 212 $words =~ s@\s+@ @g; |
igor@26 | 213 @words = split /\s+/, $words; |
igor@26 | 214 for ($i=0; $i<$#words-3;$i++) { |
igor@26 | 215 my ($a,$b,$c)= ($words[$i],$words[$i+1],$words[$i+2]); |
igor@26 | 216 if ($ENV{GROUP_WORDS_BY_THREE} eq "YES" and ($a && $b && $c)) { |
igor@26 | 217 print "${a}_${b}_${c}\n"; |
igor@26 | 218 }; |
igor@26 | 219 if ($ENV{GROUP_WORDS_BY_TWO} eq "YES" and ($a && $b)) { |
igor@26 | 220 print "${a}_${b}\n"; |
igor@26 | 221 }; |
igor@26 | 222 } |
igor@26 | 223 PERL_SCRIPT |
igor@26 | 224 perl $PERL_SCRIPT_TEMP_NAME "$ORIGINAL_TEXT" |
igor@26 | 225 rm $PERL_SCRIPT_TEMP_NAME |
igor@26 | 226 fi |
igor@26 | 227 } |
igor@26 | 228 |
igor@0 | 229 grep_v_english() |
igor@0 | 230 { |
igor@0 | 231 [ -e "$VOCABULARY" ] || touch "$VOCABULARY" |
igor@0 | 232 eval $(cat $VOCABULARY | tr -d "'" | xargs -n10 echo | tr ' ' '|' | sed 's/^/egrep -xv "RRRRRRR|/' | sed 's/$/"/' | tr '\n' '|')cat |
igor@0 | 233 } |
igor@0 | 234 |
igor@0 | 235 grep_v_english_perl() |
igor@0 | 236 { |
igor@0 | 237 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX` |
igor@0 | 238 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME |
igor@33 | 239 if ($ENV{FILTER_WORDS} eq "NO") { |
igor@33 | 240 while(<>) { print; } |
igor@33 | 241 exit(0); |
igor@33 | 242 } |
igor@27 | 243 $voc_files=$ENV{VOC_FILES}; |
igor@27 | 244 $voc_files=~s@^ @@; |
igor@27 | 245 for $voc_file (split /\s+/,$voc_files) { |
igor@27 | 246 if (open(VOC, $voc_file)) { |
igor@27 | 247 while (<VOC>){ |
igor@27 | 248 chomp; |
igor@27 | 249 #s/'//g; |
igor@27 | 250 $voc{$_}="1"; |
igor@27 | 251 } |
igor@27 | 252 } |
igor@0 | 253 } |
igor@0 | 254 while(<>) { |
igor@0 | 255 chomp; |
igor@0 | 256 if (not defined($voc{$_})) { print "$_\n"; } |
igor@0 | 257 } |
igor@0 | 258 PERL_SCRIPT |
igor@0 | 259 [ -e "$VOCABULARY" ] || touch "$VOCABULARY" |
igor@27 | 260 export VOCABULARY VOC_FILES |
igor@27 | 261 VOC_FILES=$VOCABULARY |
igor@27 | 262 for i in $TAG_NAME |
igor@27 | 263 do |
igor@27 | 264 VOC_FILES="${VOC_FILES} `tag_file_name $i`" |
igor@27 | 265 done |
igor@0 | 266 perl $PERL_SCRIPT_TEMP_NAME |
igor@0 | 267 rm $PERL_SCRIPT_TEMP_NAME |
igor@0 | 268 } |
igor@0 | 269 |
igor@11 | 270 group_words() |
igor@11 | 271 { |
igor@22 | 272 #if [ "$LANGUAGE" != "en" ] |
igor@22 | 273 #then |
igor@22 | 274 # cat |
igor@22 | 275 # return |
igor@22 | 276 #fi |
igor@11 | 277 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-group-words-XXXXXXXX` |
igor@11 | 278 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME |
igor@11 | 279 #!/usr/bin/perl |
igor@11 | 280 |
igor@29 | 281 use Encode; |
igor@29 | 282 use utf8; |
igor@32 | 283 use Lingua::Stem::Snowball qw(stem); |
igor@29 | 284 |
igor@12 | 285 eval { |
igor@12 | 286 # http://stackoverflow.com/questions/251694/how-can-i-check-if-i-have-a-perl-module-before-using-it |
igor@12 | 287 require String::Similarity; |
igor@12 | 288 String::Similarity->import(); |
igor@12 | 289 }; |
igor@12 | 290 unless($@) |
igor@12 | 291 { |
igor@12 | 292 our $HAVE_String_Similarity=1; |
igor@12 | 293 } |
igor@12 | 294 |
igor@32 | 295 |
igor@32 | 296 sub load_notes_dict() |
igor@32 | 297 { |
igor@32 | 298 my %dict; |
igor@32 | 299 if (open(NOTES, $ENV{NOTES_FILE})) { |
igor@32 | 300 while(<NOTES>) { |
igor@32 | 301 $_ = decode( "utf8", $_); |
igor@32 | 302 chomp; |
igor@32 | 303 s/^\s+//; |
igor@32 | 304 my ($a,$b)=split /\s+/,$_,2; |
igor@32 | 305 $dict{$a}=$b; |
igor@32 | 306 } |
igor@32 | 307 } |
igor@32 | 308 return %dict; |
igor@32 | 309 } |
igor@32 | 310 |
igor@12 | 311 sub similar($$){ |
igor@12 | 312 my $a=shift; |
igor@12 | 313 my $b=shift; |
igor@12 | 314 if ($HAVE_String_Similarity) { |
igor@12 | 315 return $Similarity{"$a $b"}; |
igor@12 | 316 } |
igor@12 | 317 else { |
igor@12 | 318 return 0; |
igor@12 | 319 } |
igor@12 | 320 } |
igor@12 | 321 |
igor@32 | 322 |
igor@32 | 323 sub normalize_without_linked($) |
igor@11 | 324 { |
igor@22 | 325 if ( $ENV{LANGUAGE} eq "en" ) { return normalize_english(shift); } |
igor@22 | 326 elsif ( $ENV{LANGUAGE} eq "de" ) { return normalize_german(shift); } |
igor@29 | 327 elsif ( $ENV{LANGUAGE} eq "uk" ) { return normalize_ukrainian(shift); } |
igor@31 | 328 elsif ( $ENV{LANGUAGE} eq "io" ) { return normalize_esperanto(shift); } |
igor@22 | 329 else { return shift ; } |
igor@22 | 330 } |
igor@22 | 331 |
igor@32 | 332 sub normalize_with_linked($) |
igor@32 | 333 { |
igor@32 | 334 my $word = normalize_without_linked(shift); |
igor@32 | 335 #return $word; |
igor@32 | 336 if ($linked_words{$word}) { |
igor@32 | 337 return $linked_words{$word}; |
igor@32 | 338 } |
igor@32 | 339 else { |
igor@32 | 340 return $word; |
igor@32 | 341 } |
igor@32 |