new-words
view new-words.sh @ 19:416394a87d9f
minifix
| author | Igor Chubin <igor@chub.in> |
|---|---|
| date | Tue Apr 20 21:15:19 2010 +0400 (2010-04-20) |
| parents | 7e3a52db54ad |
| children | 23f949c03f95 |
line source
1 #!/bin/sh
2 cat <<HELP > /dev/null
4 USAGE:
6 new-words [ -l lang ] [ -s ] [ ARG ]
8 SWITCHES:
10 -s show text statistics and exit
11 -l lang override language settings
13 Поддержка нескольких языков:
15 new-words -l lang URL
17 Например, для немецких текстов:
19 new-words -l de URL
21 Или, предварительно создав соответствующую ссылку:
23 de-words URL
25 HELP
27 WORK_DIR=~/.new-words/
28 TEMP1=`mktemp /tmp/new-words-XXXXXXXXXX-temp1`
29 TEMP2=`mktemp /tmp/new-words-XXXXXXXXXX-temp2`
30 export ORIGINAL_TEXT=`mktemp /tmp/new-words-XXXXXXXXXX-orig`
31 editor=${EDITOR:-vim}
33 LANGUAGE=en
34 my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `"
35 if echo "$1" | grep -q http://...wikipedia.org/wiki/
36 then
37 LANGUAGE="`echo $1 | sed s@http://@@ | sed s@.wikipedia.*@@`"
38 fi
39 [ "${my_name}" = "new" ] || LANGUAGE="$my_name"
40 if [ "$1" = "-l" ]
41 then
42 LANGUAGE="$2"
43 VOCABULARY="$LANGUAGE".txt
44 shift 2
45 fi
46 VOCABULARY=${LANGUAGE}.txt
47 NOTES_FILE=notes-${LANGUAGE}.txt
49 STAT_ONLY=NO
50 if [ "$1" = "-s" ]
51 then
52 STAT_ONLY=YES
53 shift
54 fi
56 get_words()
57 {
58 tr ' ' '\n' | sed 's/--/ /g' \
59 | sed "s/'/__APOSTROPHE__/g" \
60 | tr '—·-' '-----' \
61 | tr '*\r,.:#@()+=—<>$;"?!|·[]^%&' ' ' \
62 | tr ' ' '\n' \
63 | grep -x '[[:alpha:]'"'"'-]*' \
64 | tee "$1" \
65 | grep_v_english_perl \
66 | sort | uniq -c | awk '{if ($2!="") print;}' | sort -rn
67 }
69 add_stat()
70 {
71 before="$1"
72 after=${before}2
73 cat > "$after"
74 total="`wc -w $1 | awk '{print $1}'`"
75 total_unknown="`cat $after|awk '{s=s+$1}END{print s}'`"
76 total_known="`echo $total-$total_unknown|bc`"
77 percentage="`echo '100*('$total-$total_unknown')'/$total | bc -l | sed 's/\\.\(.\).*/.\1/'`"
78 #sentences="`cat $after | perl -e 'local $/; $_=<>; s@http://[a-zA-Z&_.:/0-9%?=,\#+()\[\]~-]*@@g; s@\n@@g; s@(Mr|Mrs)\.@\1POINT@g; @sentences=split /\\./;print $#sentences;'`"
79 sentences="`cat $ORIGINAL_TEXT | perl -e 'local $/; $_=<>; s/[^.]//msg; print length($_);'`"
82 if [ "$STAT_ONLY" = "YES" ]
83 then
84 echo "LANG KNOWN% UNKNOWN% KNOWN TOTAL WPS UWPS*10"
85 echo "$LANGUAGE $percentage `echo \(100-$percentage\) | bc -l` $total_known $total `echo $total/$sentences|bc` `echo 10*$total_unknown/$sentences|bc` "
86 rm $after
87 return 0
88 else
89 echo "# $LANGUAGE, $percentage, <$total_known/$total>"
90 fi
92 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
93 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
94 my $total=shift(@ARGV);
95 my $total_known=shift(@ARGV);
96 my $s=0;
97 my $mark_line=int($total_known*100/$total/5)*5;
98 if ($mark_line>=90) {
99 $mark_line=int($total_known*100/$total)+1;
100 } else { $mark_line +=5; };
101 while(<>)
102 {
103 print;
104 /^\s*([0-9]*)\s*/;
105 $s+=$1;
106 if (($total_known+$s)*100/$total>=$mark_line) {
107 print "# $mark_line\n";
108 if ($mark_line>=90) { $mark_line+=1; } else { $mark_line +=5; };
109 }
110 }
111 PERL_SCRIPT
112 perl $PERL_SCRIPT_TEMP_NAME "$total" "$total_known" "$after"
113 rm $PERL_SCRIPT_TEMP_NAME
114 rm $after
115 }
117 grep_v_english()
118 {
119 [ -e "$VOCABULARY" ] || touch "$VOCABULARY"
120 eval $(cat $VOCABULARY | tr -d "'" | xargs -n10 echo | tr ' ' '|' | sed 's/^/egrep -xv "RRRRRRR|/' | sed 's/$/"/' | tr '\n' '|')cat
121 }
123 grep_v_english_perl()
124 {
125 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
126 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
127 open(VOC, $ENV{VOCABULARY})
128 or die "Can't open VOCABULARY";
129 while (<VOC>){
130 chomp;
131 #s/'//g;
132 $voc{$_}="1";
133 }
134 while(<>) {
135 chomp;
136 if (not defined($voc{$_})) { print "$_\n"; }
137 }
138 PERL_SCRIPT
139 [ -e "$VOCABULARY" ] || touch "$VOCABULARY"
140 export VOCABULARY
141 perl $PERL_SCRIPT_TEMP_NAME
142 rm $PERL_SCRIPT_TEMP_NAME
143 }
145 group_words()
146 {
147 if [ "$LANGUAGE" != "en" ]
148 then
149 cat
150 return
151 fi
152 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-group-words-XXXXXXXX`
153 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
154 #!/usr/bin/perl
156 eval {
157 # http://stackoverflow.com/questions/251694/how-can-i-check-if-i-have-a-perl-module-before-using-it
158 require String::Similarity;
159 String::Similarity->import();
160 };
161 unless($@)
162 {
163 our $HAVE_String_Similarity=1;
164 }
166 sub similar($$){
167 my $a=shift;
168 my $b=shift;
169 if ($HAVE_String_Similarity) {
170 return $Similarity{"$a $b"};
171 }
172 else {
173 return 0;
174 }
175 }
177 sub normalize($)
178 {
179 $_=lc(shift);
181 s/s$//;
182 s/ed$//;
183 s/en$//;
184 s/er$//;
185 s/est$//;
186 s/ing$//;
188 s/ism$//; s/ist$//; s/ful$//; s/able$//; s/ably$//;
189 s/ation$//; s/ness$//; s/ship$//; s/ally$//;
190 s/ment$//; s/ify$//; s/ity$//; s/fy$//; s/ly$//;
191 s/ise$//; s/ize$//;
193 s/e$//;
194 return $_;
195 }
198 sub compare($$)
199 {
200 my $a=shift;
201 my $b=shift;
202 $a =~ s/^\s*//;
203 $b =~ s/^\s*//;
204 my ($a1, $a2)= split /\s+/,$a,2;
205 my ($b1, $b2)= split /\s+/,$b,2;
207 my $cmp = $group_weight{normalize($a2)} <=> $group_weight{normalize($b2)};
209 if ($cmp) {
210 return $cmp;
211 }
212 else {
213 if (normalize($a2) ne normalize($b2)) {
214 return normalize($a2) cmp normalize($b2);
215 }
216 else {
217 return $a1 <=> $b1;
218 }
219 }
220 }
222 @lines=<>;
223 for $L (@lines) {
224 chomp($L);
225 $l=$L;
226 $l =~ s/^\s*//;
227 my ($a, $b)=split(/\s+/,$l,2);
228 $group_weight{normalize($b)}+=$a;
229 }
230 @lines2 = sort { compare($b,$a) } @lines;
231 for $l (@lines2) {
232 print "$l\n";
233 }
234 PERL_SCRIPT
235 perl $PERL_SCRIPT_TEMP_NAME
236 rm $PERL_SCRIPT_TEMP_NAME
237 }
239 text_from_url()
240 {
241 lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@'
242 }
244 add_marks()
245 {
246 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
247 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
248 $file = $ARGV[0];
249 our $dict;
250 if (open(NOTES, $ENV{NOTES_FILE})) {
251 while(<NOTES>) {
252 chomp;
253 s/^\s+//;
254 my ($a,$b)=split /\s+/,$_,2;
255 $dict{$a}=$b;
256 }
257 }
258 if (open(F, $file)) {
259 @lines=<F>;
260 close(F);
262 if (open(F, ">$file")) {
263 for (@lines) {
264 m/\s+\S+\s+(\S+)/;
265 $name=$1;
266 if (not /^#/ and defined($dict{$name})) {
267 chomp;
268 $mark=$dict{$name};
269 $space=" "x(30-length($_));
270 print F "$_$space$mark\n";
271 }
272 else {
273 print F "$_";
274 }
275 }
276 close(F);
277 }
278 }
279 PERL_SCRIPT
280 [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE"
281 export NOTES_FILE
282 perl $PERL_SCRIPT_TEMP_NAME "$1"
283 rm $PERL_SCRIPT_TEMP_NAME
284 }
286 remove_marks()
287 {
288 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
289 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
290 $file = $ARGV[0];
291 our %dict;
292 if (open(F, $file)) {
293 @lines=<F>;
294 close(F);
296 if (open(F, ">$file")) {
297 for (@lines) {
298 chomp;
299 if (not /^#/ and m/(\s+)(\S+)(\s+)(\S+)(\s+)(.*)/) {
300 my $name=$4;
301 my $comment=$6;
302 $dict{$name}=$comment;
303 print F "$1$2$3$4\n";
304 }
305 else {
306 print F "$_\n";
307 }
308 }
309 }
310 }
311 if (open(NOTES, $ENV{NOTES_FILE})) {
312 @lines=<NOTES>;
313 close(NOTES);
315 if (open(NOTES, ">".$ENV{NOTES_FILE})) {
316 for (@lines) {
317 chomp;
318 s/^\s+//;
319 my ($a,$b)=split /\s+/,$_,2;
320 if (not defined($dict{$a}) || ($dict{$a} eq $b)) {
321 print NOTES "$_\n";
322 if (defined($dict{$a})) { unset($dict{$a}); }
323 }
324 }
325 for (keys %dict) {
326 $mark=$dict{$_};
327 $space=" "x(30-length($_));
328 print NOTES "$_$space$mark\n";
329 }
330 }
331 }
332 PERL_SCRIPT
333 [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE"
334 export NOTES_FILE
335 perl $PERL_SCRIPT_TEMP_NAME "$1"
336 rm $PERL_SCRIPT_TEMP_NAME
337 }
339 mkdir -p $WORK_DIR
340 oldpwd="$PWD"
341 cd $WORK_DIR
342 if echo "$1" | grep -q http:
343 then
344 text_from_url "$1"
345 elif [ "$#" != 0 ]
346 then
347 if echo $1 | grep -q ^/
348 then
349 cat "$1"
350 else
351 cat "$oldpwd/$1"
352 fi
353 else
354 cat
355 fi \
356 | tee $ORIGINAL_TEXT \
357 | get_words ${TEMP1}-full \
358 | group_words \
359 | add_stat ${TEMP1}-full \
360 | tee "$TEMP1" > "$TEMP2"
362 if [ "$STAT_ONLY" = "YES" ]
363 then
364 cat "$TEMP1"
365 else
366 add_marks "$TEMP2"
367 if [ "$editor" = vim ]
368 then
369 vim -c 'set keywordprg='"$LANGUAGE" -c 'set iskeyword=@,48-57,/,.,-,_,+,,,#,$,%,~,=' "$TEMP2" < /dev/tty > /dev/tty
370 else
371 echo 2
372 $editor "$TEMP2"
373 fi
374 remove_marks "$TEMP2"
375 fi
377 diff "$TEMP1" "$TEMP2" | awk '{print $3}' | sort -u >> "$VOCABULARY"
378 rm -f "$TEMP1" "$TEMP2" "${TEMP1}-full" "$ORIGINAL_TEXT"