new-words
view new-words.sh @ 21:190d4ac6b07c
-m and -n switches
| author | Igor Chubin <igor@chub.in> |
|---|---|
| date | Thu May 06 21:05:20 2010 +0300 (2010-05-06) |
| parents | 23f949c03f95 |
| children | 46e987f4636d |
line source
1 #!/bin/sh
3 show_usage()
4 {
5 cat <<HELP > /dev/stderr
7 USAGE:
9 new-words [ -l lang ] [ -s ] [ ARG ]
11 SWITCHES:
13 -h print this screen
14 -k put higher words that are similar to the known words (only for English)
15 -l lang override language settings
16 -n non-interactive mode (don't run vi)
17 -m don't add marks (and don't save marks added by user)
18 -s show the text statistics (percentage of known words and so on) and exit
20 The language of the text can be specified also
21 by name of the program new-words (correspondent link must be created before).
22 For example, these calls are equivalent:
24 de-words URL
25 new-words -l de URL
27 HELP
28 }
30 if [ "$1" = "-h" ]
31 then
32 show_usage
33 exit 0
34 fi
36 WORK_DIR=~/.new-words/
37 TEMP1=`mktemp /tmp/new-words-XXXXXXXXXX-temp1`
38 TEMP2=`mktemp /tmp/new-words-XXXXXXXXXX-temp2`
39 export ORIGINAL_TEXT=`mktemp /tmp/new-words-XXXXXXXXXX-orig`
40 editor=${EDITOR:-vim}
42 #----------------------------------------------------
43 # command line options processing
45 LANGUAGE=en
46 my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `"
47 if echo "$1" | grep -q http://...wikipedia.org/wiki/
48 then
49 LANGUAGE="`echo $1 | sed s@http://@@ | sed s@.wikipedia.*@@`"
50 fi
51 [ "${my_name}" = "new" ] || LANGUAGE="$my_name"
52 if [ "$1" = "-l" ]
53 then
54 LANGUAGE="$2"
55 VOCABULARY="$LANGUAGE".txt
56 shift 2
57 fi
58 VOCABULARY=${LANGUAGE}.txt
59 NOTES_FILE=notes-${LANGUAGE}.txt
61 STAT_ONLY=NO
62 if [ "$1" = "-s" ]
63 then
64 STAT_ONLY=YES
65 shift
66 fi
68 NEED_TO_USE_VOCABULARY_WHEN_SORT=NO
69 if [ "$1" = "-k" ]
70 then
71 NEED_TO_USE_VOCABULARY_WHEN_SORT=YES
72 shift
73 fi
75 DONT_ADD_MARKS=NO
76 if [ "$1" = "-m" ]
77 then
78 DONT_ADD_MARKS=YES
79 shift
80 fi
82 NON_INTERACTIVE_MODE=NO
83 if [ "$1" = "-n" ]
84 then
85 NON_INTERACTIVE_MODE=YES
86 shift
87 fi
89 #----------------------------------------------------
91 get_words()
92 {
93 tr ' ' '\n' | sed 's/--/ /g' \
94 | sed "s/'/__APOSTROPHE__/g" \
95 | tr '—·-' '-----' \
96 | tr '*\r,.:#@()+=—<>$;"?!|·[]^%&' ' ' \
97 | tr ' ' '\n' \
98 | grep -x '[[:alpha:]'"'"'-]*' \
99 | tee "$1" \
100 | grep_v_english_perl \
101 | sort | uniq -c | awk '{if ($2!="") print;}' | sort -rn
102 }
104 add_stat()
105 {
106 before="$1"
107 after=${before}2
108 cat > "$after"
109 total="`wc -w $1 | awk '{print $1}'`"
110 total_unknown="`cat $after|awk '{s=s+$1}END{print s}'`"
111 total_known="`echo $total-$total_unknown|bc`"
112 percentage="`echo '100*('$total-$total_unknown')'/$total | bc -l | sed 's/\\.\(.\).*/.\1/'`"
113 #sentences="`cat $after | perl -e 'local $/; $_=<>; s@http://[a-zA-Z&_.:/0-9%?=,\#+()\[\]~-]*@@g; s@\n@@g; s@(Mr|Mrs)\.@\1POINT@g; @sentences=split /\\./;print $#sentences;'`"
114 sentences="`cat $ORIGINAL_TEXT | perl -e 'local $/; $_=<>; s/[^.]//msg; print length($_);'`"
117 if [ "$STAT_ONLY" = "YES" ]
118 then
119 echo "LANG KNOWN% UNKNOWN% KNOWN TOTAL WPS UWPS*10"
120 echo "$LANGUAGE $percentage `echo \(100-$percentage\) | bc -l` $total_known $total `echo $total/$sentences|bc` `echo 10*$total_unknown/$sentences|bc` "
121 rm $after
122 return 0
123 else
124 echo "# $LANGUAGE, $percentage, <$total_known/$total>"
125 fi
127 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
128 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
129 my $total=shift(@ARGV);
130 my $total_known=shift(@ARGV);
131 my $s=0;
132 my $mark_line=int($total_known*100/$total/5)*5;
133 if ($mark_line>=90) {
134 $mark_line=int($total_known*100/$total)+1;
135 } else { $mark_line +=5; };
136 while(<>)
137 {
138 print;
139 /^\s*([0-9]*)\s*/;
140 $s+=$1;
141 if (($total_known+$s)*100/$total>=$mark_line) {
142 print "# $mark_line\n";
143 if ($mark_line>=90) { $mark_line+=1; } else { $mark_line +=5; };
144 }
145 }
146 PERL_SCRIPT
147 perl $PERL_SCRIPT_TEMP_NAME "$total" "$total_known" "$after"
148 rm $PERL_SCRIPT_TEMP_NAME
149 rm $after
150 }
152 grep_v_english()
153 {
154 [ -e "$VOCABULARY" ] || touch "$VOCABULARY"
155 eval $(cat $VOCABULARY | tr -d "'" | xargs -n10 echo | tr ' ' '|' | sed 's/^/egrep -xv "RRRRRRR|/' | sed 's/$/"/' | tr '\n' '|')cat
156 }
158 grep_v_english_perl()
159 {
160 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
161 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
162 open(VOC, $ENV{VOCABULARY})
163 or die "Can't open VOCABULARY";
164 while (<VOC>){
165 chomp;
166 #s/'//g;
167 $voc{$_}="1";
168 }
169 while(<>) {
170 chomp;
171 if (not defined($voc{$_})) { print "$_\n"; }
172 }
173 PERL_SCRIPT
174 [ -e "$VOCABULARY" ] || touch "$VOCABULARY"
175 export VOCABULARY
176 perl $PERL_SCRIPT_TEMP_NAME
177 rm $PERL_SCRIPT_TEMP_NAME
178 }
180 group_words()
181 {
182 if [ "$LANGUAGE" != "en" ]
183 then
184 cat
185 return
186 fi
187 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-group-words-XXXXXXXX`
188 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
189 #!/usr/bin/perl
191 eval {
192 # http://stackoverflow.com/questions/251694/how-can-i-check-if-i-have-a-perl-module-before-using-it
193 require String::Similarity;
194 String::Similarity->import();
195 };
196 unless($@)
197 {
198 our $HAVE_String_Similarity=1;
199 }
201 sub similar($$){
202 my $a=shift;
203 my $b=shift;
204 if ($HAVE_String_Similarity) {
205 return $Similarity{"$a $b"};
206 }
207 else {
208 return 0;
209 }
210 }
212 sub normalize($)
213 {
214 $_=lc(shift);
216 s/s$//;
217 s/ed$//;
218 s/en$//;
219 s/er$//;
220 s/est$//;
221 s/ing$//;
223 s/ism$//; s/ist$//; s/ful$//; s/able$//; s/ably$//;
224 s/ation$//; s/ness$//; s/ship$//; s/ally$//;
225 s/ment$//; s/ify$//; s/ity$//; s/fy$//; s/ly$//;
226 s/ise$//; s/ize$//;
228 s/e$//;
229 return $_;
230 }
233 sub compare($$)
234 {
235 my $a=shift;
236 my $b=shift;
237 $a =~ s/^\s*//;
238 $b =~ s/^\s*//;
239 my ($a1, $a2)= split /\s+/,$a,2;
240 my ($b1, $b2)= split /\s+/,$b,2;
242 my $cmp = $group_weight{normalize($a2)} <=> $group_weight{normalize($b2)};
244 if ($cmp) {
245 return $cmp;
246 }
247 else {
248 if (normalize($a2) ne normalize($b2)) {
249 return normalize($a2) cmp normalize($b2);
250 }
251 else {
252 return $a1 <=> $b1;
253 }
254 }
255 }
257 our %Vocabulary;
258 open(VOC, $ENV{VOCABULARY})
259 or die "Can't open VOCABULARY";
260 while (<VOC>){
261 chomp;
262 #s/'//g;
263 $Vocabulary{normalize($_)}="1";
264 }
265 close(VOC);
267 @lines=<STDIN>;
268 for $L (@lines) {
269 chomp($L);
270 $l=$L;
271 $l =~ s/^\s*//;
272 my ($a, $b)=split(/\s+/,$l,2);
273 $group_weight{normalize($b)}+=$a;
274 }
275 if ($ENV{NEED_TO_USE_VOCABULARY_WHEN_SORT} eq "YES") {
276 for $k (keys %group_weight) {
277 if (defined($Vocabulary{$k})) {
278 $group_weight{$k} *= 2;
279 }
280 }
281 }
282 @lines2 = sort { compare($b,$a) } @lines;
283 for $l (@lines2) {
284 print "$l\n";
285 }
286 PERL_SCRIPT
287 export VOCABULARY
288 export NEED_TO_USE_VOCABULARY_WHEN_SORT
289 perl $PERL_SCRIPT_TEMP_NAME
290 rm $PERL_SCRIPT_TEMP_NAME
291 }
293 text_from_url()
294 {
295 lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@'
296 }
298 add_marks()
299 {
300 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
301 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
302 $file = $ARGV[0];
303 our $dict;
304 if (open(NOTES, $ENV{NOTES_FILE})) {
305 while(<NOTES>) {
306 chomp;
307 s/^\s+//;
308 my ($a,$b)=split /\s+/,$_,2;
309 $dict{$a}=$b;
310 }
311 }
312 if (open(F, $file)) {
313 @lines=<F>;
314 close(F);
316 if (open(F, ">$file")) {
317 for (@lines) {
318 m/\s+\S+\s+(\S+)/;
319 $name=$1;
320 if (not /^#/ and defined($dict{$name})) {
321 chomp;
322 $mark=$dict{$name};
323 $space=" "x(30-length($_));
324 print F "$_$space$mark\n";
325 }
326 else {
327 print F "$_";
328 }
329 }
330 close(F);
331 }
332 }
333 PERL_SCRIPT
334 [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE"
335 export NOTES_FILE
336 perl $PERL_SCRIPT_TEMP_NAME "$1"
337 rm $PERL_SCRIPT_TEMP_NAME
338 }
340 remove_marks()
341 {
342 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
343 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
344 $file = $ARGV[0];
345 our %dict;
346 if (open(F, $file)) {
347 @lines=<F>;
348 close(F);
350 if (open(F, ">$file")) {
351 for (@lines) {
352 chomp;
353 if (not /^#/ and m/(\s+)(\S+)(\s+)(\S+)(\s+)(.*)/) {
354 my $name=$4;
355 my $comment=$6;
356 $dict{$name}=$comment;
357 print F "$1$2$3$4\n";
358 }
359 else {
360 print F "$_\n";
361 }
362 }
363 }
364 }
365 if (($ENV{DONT_ADD_MARKS} ne "YES") and open(NOTES, $ENV{NOTES_FILE})) {
366 @lines=<NOTES>;
367 close(NOTES);
369 if (open(NOTES, ">".$ENV{NOTES_FILE})) {
370 for (@lines) {
371 chomp;
372 s/^\s+//;
373 my ($a,$b)=split /\s+/,$_,2;
374 if (not defined($dict{$a}) || ($dict{$a} eq $b)) {
375 print NOTES "$_\n";
376 if (defined($dict{$a})) { unset($dict{$a}); }
377 }
378 }
379 for (keys %dict) {
380 $mark=$dict{$_};
381 $space=" "x(30-length($_));
382 print NOTES "$_$space$mark\n";
383 }
384 }
385 }
386 PERL_SCRIPT
387 [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE"
388 export NOTES_FILE
389 export DONT_ADD_MARKS
390 perl $PERL_SCRIPT_TEMP_NAME "$1"
391 rm $PERL_SCRIPT_TEMP_NAME
392 }
394 mkdir -p $WORK_DIR
395 oldpwd="$PWD"
396 cd $WORK_DIR
397 if echo "$1" | grep -q http:
398 then
399 text_from_url "$1"
400 elif [ "$#" != 0 ]
401 then
402 if echo $1 | grep -q ^/
403 then
404 cat "$1"
405 else
406 cat "$oldpwd/$1"
407 fi
408 else
409 cat
410 fi \
411 | tee $ORIGINAL_TEXT \
412 | get_words ${TEMP1}-full \
413 | group_words \
414 | add_stat ${TEMP1}-full \
415 | tee "$TEMP1" > "$TEMP2"
417 if [ "$STAT_ONLY" = "YES" ]
418 then
419 cat "$TEMP1"
420 elif [ "$NON_INTERACTIVE_MODE" = "YES" ]
421 then
422 cat "$TEMP1"
423 else
424 [ "$DONT_ADD_MARKS" = "YES" ] || add_marks "$TEMP2"
425 if [ "$editor" = vim ]
426 then
427 vim -c 'set keywordprg='"$LANGUAGE" -c 'set iskeyword=@,48-57,/,.,-,_,+,,,#,$,%,~,=' "$TEMP2" < /dev/tty > /dev/tty
428 else
429 echo 2
430 $editor "$TEMP2"
431 fi
432 remove_marks "$TEMP2"
433 fi
435 diff "$TEMP1" "$TEMP2" | awk '{print $3}' | sort -u >> "$VOCABULARY"
436 rm -f "$TEMP1" "$TEMP2" "${TEMP1}-full" "$ORIGINAL_TEXT"