new-words
view new-words.py @ 41:4629e08b0d87
minifix: skip numbers as words
| author | Igor Chubin <igor@chub.in> |
|---|---|
| date | Sun Jan 23 18:18:19 2011 +0100 (2011-01-23) |
| parents | c3a50c0d2400 |
| children | 3ec83a7cc544 |
line source
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 from __future__ import with_statement
5 import codecs
6 import logging
7 import os
8 import optparse
9 import re
10 import subprocess
11 import sys
12 import Stemmer
14 config = {
15 'config_directory': os.environ['HOME'] + '/.new-words',
16 'language': 'en',
17 }
19 logging.basicConfig(filename='/tmp/new-words-py.log', level=logging.DEBUG)
21 class Normalizator:
22 def __init__(self, language, linked_words={}):
23 stemmer_algorithm = {
24 'de' : 'german',
25 'en' : 'english',
26 'ru' : 'russian',
27 'uk' : 'ukrainian',
28 }
29 self.stemmer = Stemmer.Stemmer(stemmer_algorithm[language])
30 self.linked_words = linked_words
32 def normalize(self, word):
33 word_chain = []
34 while word in self.linked_words and not word in word_chain:
35 word_chain.append(word)
36 word = self.linked_words[word]
37 return self.stemmer.stemWord(word.lower())
39 parser = optparse.OptionParser()
41 parser.add_option(
42 "-a", "--no-marks",
43 help="don't add marks (and don't save marks added by user)",
44 action="store_true",
45 dest="no_marks")
47 parser.add_option(
48 "-c", "--compressed",
49 help="show compressed wordlist: one word per group",
50 action="store_true",
51 dest="compressed")
53 parser.add_option(
54 "-k", "--known-words",
55 help="put higher words that are similar to the known words (only for English)",
56 action="store_true",
57 dest="compressed")
59 parser.add_option(
60 "-l", "--language",
61 help="specify language of text",
62 action="store",
63 dest="language")
65 parser.add_option(
66 "-f", "--function",
67 help="filter through subsystem [INTERNAL]",
68 action="store",
69 dest="function")
71 parser.add_option(
72 "-m", "--merge-tag",
73 help="merge words tagged with specified tag into the main vocabulary",
74 action="store",
75 dest="merge_tag")
77 parser.add_option(
78 "-M", "--merge-tagged",
79 help="merge words tagged with ANY tag into the main vocabulary",
80 action="store_true",
81 dest="merge_tagged")
83 parser.add_option(
84 "-n", "--non-interactive",
85 help="non-interactive mode (don't run vi)",
86 action="store_true",
87 dest="non_interactive")
89 parser.add_option(
90 "-N", "--no-filter",
91 help="switch off known words filtering",
92 action="store_true",
93 dest="no_filter")
95 parser.add_option(
96 "-p", "--pages",
97 help="work with specified pages only (pages = start-stop/total )",
98 action="store",
99 dest="pages")
101 parser.add_option(
102 "-r", "--remove-tag",
103 help="remove subvocabulary of specified tag",
104 action="store",
105 dest="remove_tag")
107 parser.add_option(
108 "-s", "--text-stats",
109 help="show the text statistics (percentage of known words and so on) and exit",
110 action="store_true",
111 dest="text_stats")
113 parser.add_option(
114 "-S", "--voc-stats",
115 help="show your vocabulary statistics (number of words and word groups)",
116 action="store_true",
117 dest="voc_stats")
119 parser.add_option(
120 "-t", "--tag",
121 help="tag known words with tag",
122 action="store",
123 dest="tag")
125 parser.add_option(
126 "-T", "--show-tags",
127 help="tag known words with tag",
128 action="store_true",
129 dest="show_tags")
131 parser.add_option(
132 "-2", "--two-words",
133 help="find 2 words' sequences",
134 action="store_true",
135 dest="two_words")
137 parser.add_option(
138 "-3", "--three-words",
139 help="find 3 words' sequences",
140 action="store_true",
141 dest="three_words")
143 def readlines_from_file(filename):
144 res = []
145 with codecs.open(filename, "r", "utf-8") as f:
146 for line in f.readlines():
147 res += [line]
148 return res
150 def readlines_from_stdin():
151 return codecs.getreader("utf-8")(sys.stdin).readlines()
153 def words_from_line(line):
154 line = line.rstrip('\n')
155 #return re.split('(?:\s|[*\r,.:#@()+=<>$;"?!|\[\]^%&~{}«»–])+', line)
156 #return re.split('[^a-zA-ZäöëüßÄËÖÜß]+', line)
157 return re.compile("(?!')(?:\W)+", flags=re.UNICODE).split(line)
159 def get_words(lines):
160 """
161 Returns hash of words in a file
162 word => number
163 """
164 result = {}
165 for line in lines:
166 words = words_from_line(line)
167 for word in words:
168 if re.match('[0-9]*$', word):
169 continue
170 result.setdefault(word, 0)
171 result[word] += 1
172 return result
174 def load_vocabulary():
175 return get_words(readlines_from_file("%s/%s.txt"%(config['config_directory'], config['language'])))
177 def notes_filenames():
178 return ["%s/notes-%s.txt"%(config['config_directory'], config['language'])]
180 def load_notes(files):
181 notes = {}
182 for filename in files:
183 with codecs.open(filename, "r", "utf-8") as f:
184 for line in f.readlines():
185 (word, note) = re.split('\s+', line.rstrip('\n'), maxsplit=1)
186 notes.setdefault(word, {})
187 notes[word][filename] = note
188 return notes
190 def add_notes(lines, notes):
191 notes_filename = notes_filenames()[0]
192 result = []
193 for line in lines:
194 if line.startswith('#'):
195 result += [line]
196 else:
197 match_object = re.search('^\s*\S+\s*(\S+)', line)
198 if match_object:
199 word = match_object.group(1)
200 if word in notes:
201 logging.debug(word)
202 logging.debug(line)
203 if notes_filename in notes[word]:
204 line = line.rstrip('\n')
205 line = "%-30s %s\n" % (line, notes[word][notes_filename])
206 logging.debug(line)
207 result += [line]
208 else:
209 result += [line]
210 else:
211 result += [line]
212 return result
214 def remove_notes(lines, notes_group):
215 notes_filename = notes_filenames()[0]
216 notes = {}
217 for k in notes_group.keys():
218 if notes_filename in notes_group[k]:
219 notes[k] = notes_group[k][notes_filename]
221 result = []
222 for line in lines:
223 line = line.rstrip('\n')
224 match_object = re.match('(\s+)(\S+)(\s+)(\S+)(\s+)(.*)', line)
225 if match_object:
226 result.append("".join([
227 match_object.group(1),
228 match_object.group(2),
229 match_object.group(3),
230 match_object.group(4),
231 "\n"
232 ]))
233 notes[match_object.group(4)] = match_object.group(6)
234 else:
235 result.append(line+"\n")
237 save_notes(notes_filename, notes)
238 return result
240 def save_notes(filename, notes):
241 lines = []
242 saved_words = []
243 with codecs.open(filename, "r", "utf-8") as f:
244 for line in f.readlines():
245 (word, note) = re.split('\s+', line.rstrip('\n'), maxsplit=1)
246 if word in notes:
247 line = "%-29s %s\n" % (word, notes[word])
248 saved_words.append(word)
249 lines.append(line)
250 for word in [x for x in notes.keys() if not x in saved_words]:
251 line = "%-29s %s\n" % (word, notes[word])
252 lines.append(line)
254 with codecs.open(filename, "w", "utf-8") as f:
255 for line in lines:
256 f.write(line)
259 def substract_dictionary(dict1, dict2):
260 """
261 returns dict1 - dict2
262 """
263 result = {}
264 for (k,v) in dict1.items():
265 if not k in dict2:
266 result[k] = v
267 return result
269 def dump_words(words, filename):
270 with codecs.open(filename, "w+", "utf-8") as f:
271 for word in words.keys():
272 f.write(("%s\n"%word)*words[word])
274 def error_message(text):
275 print text
277 def find_wordgroups_weights(word_pairs, normalizator):
278 weight = {}
279 for (num, word) in word_pairs:
280 normalized = normalizator.normalize(word)
281 weight.setdefault(normalized, 0)
282 weight[normalized] += num
283 return weight
285 def find_linked_words(notes):
286 linked_words = {}
287 for word in notes.keys():
288 for note in notes[word].values():
289 if "@" in note:
290 result = re.search(r'\@(\S*)', note)
291 if result:
292 main_word = result.group(1)
293 if main_word:
294 linked_words[word] = main_word
295 return linked_words
297 def compare_word_pairs(pair1, pair2, wgw, normalizator, linked_words):
298 (num1, word1) = pair1
299 (num2, word2) = pair2
301 normalized_word1 = normalizator.normalize(word1)
302 normalized_word2 = normalizator.normalize(word2)
304 cmp_res = cmp(wgw[normalized_word1], wgw[normalized_word2])
305 if cmp_res != 0:
306 return cmp_res
307 else:
308 cmp_res = cmp(normalized_word1, normalized_word2)
309 if cmp_res != 0:
310 return cmp_res
311 else:
312 return cmp(int(num1), int(num2))
314 def print_words_sorted(word_pairs, stats, print_stats=True, stats_only=False):
315 if stats_only:
316 codecs.getwriter("utf-8")(sys.stdout).write("stat_only")
317 return
319 if print_stats:
320 codecs.getwriter("utf-8")(sys.stdout).write(
321 "# %(language)s, %(percentage)s, <%(total_known)s/%(total)s>, <%(groups)s/%(words)s>\n" % stats)
323 level_lines = range(int(float(stats['percentage']))/5*5+5,95,5)+range(90,102)
324 known = int(stats['total_known'])
325 total = int(stats['total'])
326 current_level = 0
327 for word_pair in word_pairs:
328 codecs.getwriter("utf-8")(sys.stdout).write("%10s %s\n" % word_pair)
329 known += word_pair[0]
330 if 100.0*known/total >= level_lines[0]:
331 current_level = level_lines[0]
332 while 100.0*known/total > level_lines[0]:
333 current_level = level_lines[0]
334 level_lines = level_lines[1:]
335 codecs.getwriter("utf-8")(sys.stdout).write("# %s\n" % current_level)
337 def filter_add_notes(args):
338 lines = readlines_from_file(args[0])
339 notes = load_notes(notes_filenames())
340 lines = add_notes(lines, notes)
341 with codecs.open(args[0], "w", "utf-8") as f:
342 for line in lines:
343 f.write(line)
345 def filter_remove_notes(args):
346 lines = readlines_from_file(args[0])
347 notes = load_notes(notes_filenames())
348 lines = remove_notes(lines, notes)
349 with codecs.open(args[0], "w", "utf-8") as f:
350 for line in lines:
351 f.write(line)
353 def filter_get_words_group_words_add_stat(args):
354 vocabulary = load_vocabulary()
355 notes = load_notes(notes_filenames())
356 lines = readlines_from_stdin()
357 words = get_words(lines)
359 stats = {}
360 stats['total'] = sum(words[x] for x in words.keys())
361 words = substract_dictionary(words, vocabulary)
363 stats['total_unknown'] = sum(words[x] for x in words.keys())
364 stats['total_known'] = stats['total'] - stats['total_unknown']
365 stats['percentage'] = "%7.2f"%(100.0*stats['total_known']/stats['total'])
366 stats['groups'] = 0
367 stats['words'] = len(words)
368 stats['sentences'] = 0 #FIXME
369 stats['language'] = config['language']
371 linked_words = find_linked_words(notes)
372 normalizator = Normalizator(config['language'], linked_words)
374 word_pairs = []
375 for k in sorted(words.keys(), key=lambda k: words[k], reverse=True):
376 word_pairs.append((words[k], k))
378 wgw = find_wordgroups_weights(word_pairs, normalizator)
379 word_pairs = sorted(
380 word_pairs,
381 cmp=lambda x,y:compare_word_pairs(x,y, wgw, normalizator, linked_words),
382 reverse=True)
384 print_words_sorted(word_pairs, stats)
386 (options, args) = parser.parse_args()
387 if options.language:
388 config['language'] = options.language
390 if options.function:
391 function_names = {
392 'add_notes' : filter_add_notes,
393 'remove_notes': filter_remove_notes,
394 'get_words_group_words_add_stat': filter_get_words_group_words_add_stat,
395 }
396 if options.function in function_names:
397 function_names[options.function](args)
398 else:
399 error_message("Unkown function %s.\nAvailable functions:\n%s" % (
400 options.function, "".join([" "+x for x in sorted(function_names.keys())])))
401 sys.exit(1)
406 #os.system("vim")