new-words
view new-words.sh @ 15:c6efd17741aa
Точка, следующая за Mr./Mrs. не считается окончанием предложения
| author | Igor Chubin <igor@chub.in> |
|---|---|
| date | Sun Apr 04 14:12:35 2010 +0300 (2010-04-04) |
| parents | 975b549364f2 |
| children | c65ffd60cc18 |
line source
1 #!/bin/sh
2 cat <<HELP > /dev/null
4 Поддержка нескольких языков:
6 new-words -l lang URL
8 Например, для немецких текстов:
10 new-words -l de URL
12 Или, предварительно создав соответствующую ссылку:
14 de-words URL
16 HELP
18 WORK_DIR=~/.new-words/
19 TEMP1=`mktemp /tmp/news-words-XXXXXXXXXX`
20 TEMP2=`mktemp /tmp/news-words-XXXXXXXXXX`
21 export ORIGINAL_TEXT=`mktemp /tmp/news-words-XXXXXXXXXX-orig`
22 editor=${EDITOR:-vim}
24 LANGUAGE=en
25 my_name="`echo $0 | sed s@.*/@@ | sed s/-.*// `"
26 if echo "$1" | grep -q http://...wikipedia.org/wiki/
27 then
28 LANGUAGE="`echo $1 | sed s@http://@@ | sed s@.wikipedia.*@@`"
29 fi
30 [ "${my_name}" = "new" ] || LANGUAGE="$my_name"
31 if [ "$1" = "-l" ]
32 then
33 LANGUAGE="$2"
34 VOCABULARY="$LANGUAGE".txt
35 shift 2
36 fi
37 VOCABULARY=${LANGUAGE}.txt
38 NOTES_FILE=notes-${LANGUAGE}.txt
40 get_words()
41 {
42 tr ' ' '\n' | sed 's/--/ /g' \
43 | sed "s/'/__APOSTROPHE__/g" \
44 | tr '—·-' '-----' \
45 | tr '*\r,.:#@()+=—<>$;"?!|·[]^%&' ' ' \
46 | tr ' ' '\n' \
47 | grep -x '[[:alpha:]'"'"'-]*' \
48 | tee "$1" \
49 | grep_v_english_perl \
50 | sort | uniq -c | awk '{if ($2!="") print;}' | sort -rn
51 }
53 add_stat()
54 {
55 before="$1"
56 after=${before}2
57 cat > "$after"
58 total="`wc -w $1 | awk '{print $1}'`"
59 total_unknown="`cat $after|awk '{s=s+$1}END{print s}'`"
60 total_known="`echo $total-$total_unknown|bc`"
61 percentage="`echo '100*('$total-$total_unknown')'/$total | bc -l | sed 's/\\.\(.\).*/.\1/'`"
62 echo "# $LANGUAGE, $percentage, <$total_known/$total>"
63 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
64 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
65 my $total=shift(@ARGV);
66 my $total_known=shift(@ARGV);
67 my $s=0;
68 my $mark_line=int($total_known*100/$total/5)*5+5;
69 while(<>)
70 {
71 print;
72 /^\s*([0-9]*)\s*/;
73 $s+=$1;
74 if (int(($total_known+$s)*100/$total)>=$mark_line) {
75 print "# $mark_line\n";
76 if ($mark_line>=90) { $mark_line+=1; } else { $mark_line +=5; };
77 }
78 }
79 PERL_SCRIPT
80 perl $PERL_SCRIPT_TEMP_NAME "$total" "$total_known" "$after"
81 rm $PERL_SCRIPT_TEMP_NAME
82 rm $after
83 }
85 grep_v_english()
86 {
87 [ -e "$VOCABULARY" ] || touch "$VOCABULARY"
88 eval $(cat $VOCABULARY | tr -d "'" | xargs -n10 echo | tr ' ' '|' | sed 's/^/egrep -xv "RRRRRRR|/' | sed 's/$/"/' | tr '\n' '|')cat
89 }
91 grep_v_english_perl()
92 {
93 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
94 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
95 open(VOC, $ENV{VOCABULARY})
96 or die "Can't open VOCABULARY";
97 while (<VOC>){
98 chomp;
99 #s/'//g;
100 $voc{$_}="1";
101 }
102 while(<>) {
103 chomp;
104 if (not defined($voc{$_})) { print "$_\n"; }
105 }
106 PERL_SCRIPT
107 [ -e "$VOCABULARY" ] || touch "$VOCABULARY"
108 export VOCABULARY
109 perl $PERL_SCRIPT_TEMP_NAME
110 rm $PERL_SCRIPT_TEMP_NAME
111 }
113 group_words()
114 {
115 if [ "$LANGUAGE" != "en" ]
116 then
117 cat
118 return
119 fi
120 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-group-words-XXXXXXXX`
121 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
122 #!/usr/bin/perl
124 eval {
125 # http://stackoverflow.com/questions/251694/how-can-i-check-if-i-have-a-perl-module-before-using-it
126 require String::Similarity;
127 String::Similarity->import();
128 };
129 unless($@)
130 {
131 our $HAVE_String_Similarity=1;
132 }
134 sub similar($$){
135 my $a=shift;
136 my $b=shift;
137 if ($HAVE_String_Similarity) {
138 return $Similarity{"$a $b"};
139 }
140 else {
141 return 0;
142 }
143 }
145 sub normalize($)
146 {
147 $_=lc(shift);
149 s/s$//;
150 s/ed$//;
151 s/en$//;
152 s/er$//;
153 s/est$//;
154 s/ing$//;
156 s/ism$//; s/ist$//; s/ful$//; s/able$//; s/ably$//;
157 s/ation$//; s/ness$//; s/ally$//;
158 s/ment$//; s/ify$//; s/ity$//; s/fy$//; s/ly$//;
159 s/ise$//; s/ize$//;
161 s/e$//;
162 return $_;
163 }
166 sub compare($$)
167 {
168 my $a=shift;
169 my $b=shift;
170 $a =~ s/^\s*//;
171 $b =~ s/^\s*//;
172 my ($a1, $a2)= split /\s+/,$a,2;
173 my ($b1, $b2)= split /\s+/,$b,2;
175 my $cmp = $group_weight{normalize($a2)} <=> $group_weight{normalize($b2)};
177 if ($cmp) {
178 return $cmp;
179 }
180 else {
181 if (normalize($a2) ne normalize($b2)) {
182 return normalize($a2) cmp normalize($b2);
183 }
184 else {
185 return $a1 <=> $b1;
186 }
187 }
188 }
190 @lines=<>;
191 for $L (@lines) {
192 chomp($L);
193 $l=$L;
194 $l =~ s/^\s*//;
195 my ($a, $b)=split(/\s+/,$l,2);
196 $group_weight{normalize($b)}+=$a;
197 }
198 @lines2 = sort { compare($b,$a) } @lines;
199 for $l (@lines2) {
200 print "$l\n";
201 }
202 PERL_SCRIPT
203 perl $PERL_SCRIPT_TEMP_NAME
204 rm $PERL_SCRIPT_TEMP_NAME
205 }
207 text_from_url()
208 {
209 lynx -dump "$1" | perl -p -e 's@http://[a-zA-Z&_.:/0-9%?=,#+()\[\]~-]*@@'
210 }
212 add_marks()
213 {
214 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
215 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
216 $file = $ARGV[0];
217 our $dict;
218 if (open(NOTES, $ENV{NOTES_FILE})) {
219 while(<NOTES>) {
220 chomp;
221 s/^\s+//;
222 my ($a,$b)=split /\s+/,$_,2;
223 $dict{$a}=$b;
224 }
225 }
226 if (open(F, $file)) {
227 @lines=<F>;
228 close(F);
230 if (open(F, ">$file")) {
231 for (@lines) {
232 m/\s+\S+\s+(\S+)/;
233 $name=$1;
234 if (not /^#/ and defined($dict{$name})) {
235 chomp;
236 $mark=$dict{$name};
237 $space=" "x(30-length($_));
238 print F "$_$space$mark\n";
239 }
240 else {
241 print F "$_";
242 }
243 }
244 close(F);
245 }
246 }
247 PERL_SCRIPT
248 [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE"
249 export NOTES_FILE
250 perl $PERL_SCRIPT_TEMP_NAME "$1"
251 rm $PERL_SCRIPT_TEMP_NAME
252 }
254 remove_marks()
255 {
256 PERL_SCRIPT_TEMP_NAME=`mktemp /tmp/perl-grep-v-english-XXXXXXXX`
257 cat <<'PERL_SCRIPT' > $PERL_SCRIPT_TEMP_NAME
258 $file = $ARGV[0];
259 our %dict;
260 if (open(F, $file)) {
261 @lines=<F>;
262 close(F);
264 if (open(F, ">$file")) {
265 for (@lines) {
266 chomp;
267 if (not /^#/ and m/(\s+)(\S+)(\s+)(\S+)(\s+)(.*)/) {
268 my $name=$4;
269 my $comment=$6;
270 $dict{$name}=$comment;
271 print F "$1$2$3$4\n";
272 }
273 else {
274 print F "$_\n";
275 }
276 }
277 }
278 }
279 if (open(NOTES, $ENV{NOTES_FILE})) {
280 @lines=<NOTES>;
281 close(NOTES);
283 if (open(NOTES, ">".$ENV{NOTES_FILE})) {
284 for (@lines) {
285 chomp;
286 s/^\s+//;
287 my ($a,$b)=split /\s+/,$_,2;
288 if (not defined($dict{$a}) || ($dict{$a} eq $b)) {
289 print NOTES "$_\n";
290 if (defined($dict{$a})) { unset($dict{$a}); }
291 }
292 }
293 for (keys %dict) {
294 $mark=$dict{$_};
295 $space=" "x(30-length($_));
296 print NOTES "$_$space$mark\n";
297 }
298 }
299 }
300 PERL_SCRIPT
301 [ -e "$NOTES_FILE" ] || touch "$NOTES_FILE"
302 export NOTES_FILE
303 perl $PERL_SCRIPT_TEMP_NAME "$1"
304 rm $PERL_SCRIPT_TEMP_NAME
305 }
307 mkdir -p $WORK_DIR
308 oldpwd="$PWD"
309 cd $WORK_DIR
310 if echo "$1" | grep -q http:
311 then
312 text_from_url "$1"
313 elif [ "$#" != 0 ]
314 then
315 if echo $1 | grep -q ^/
316 then
317 cat "$1"
318 else
319 cat "$oldpwd/$1"
320 fi
321 else
322 cat
323 fi \
324 | tee $ORIGINAL_TEXT \
325 | get_words ${TEMP1}-full \
326 | group_words \
327 | add_stat ${TEMP1}-full \
328 | tee "$TEMP1" > "$TEMP2"
330 add_marks "$TEMP2"
331 if [ "$editor" = vim ]
332 then
333 vim -c 'set keywordprg='"$LANGUAGE" -c 'set iskeyword=@,48-57,/,.,-,_,+,,,#,$,%,~,=' "$TEMP2" < /dev/tty > /dev/tty
334 else
335 echo 2
336 $editor "$TEMP2"
337 fi
338 remove_marks "$TEMP2"
340 diff "$TEMP1" "$TEMP2" | awk '{print $3}' | sort -u >> "$VOCABULARY"
341 rm -f "$TEMP1" "$TEMP2" "${TEMP1}-full" "$ORIGINAL_TEXT"