26 #include "../utility.h"
27 #include "../predictorRegistry.h"
28 #include "../tokenizer/forwardTokenizer.h"
44 separatorChars (tChars),
45 blankspaceChars(bChars),
46 controlChars (cChars),
47 predictorRegistry (registry),
48 logger (
"ContextTracker", std::cerr),
50 lowercase_mode (true),
92 logger << INFO <<
"SLIDING_WINDOWS_SIZE: " << value <<
endl;
98 logger << INFO <<
"LOWERCASE_MODE: " << value <<
endl;
123 std::stringstream change;
127 logger << INFO <<
"update(): change: " << change.str() <<
endl;
130 std::vector<std::string> change_tokens;
135 logger << INFO <<
"update(): tokenized change: ";
138 change_tokens.push_back(token);
139 logger << INFO << token <<
'|';
143 if (! change_tokens.empty()) {
145 change_tokens.pop_back();
148 logger << INFO <<
"update(): sanitized change tokens: ";
149 for (std::vector<std::string>::const_iterator it = change_tokens.begin();
150 it != change_tokens.end();
152 logger << INFO << *it <<
'|';
161 predictor = it.
next();
162 predictor->
learn(change_tokens);
182 while (tokenizer.hasMoreTokens() && i <= index) {
183 token = tokenizer.nextToken();
240 return getToken(index + change.size());
260 if (completion.find(prefix) == 0) {
269 if(
wordChars.find(c, 0) != std::string::npos)
326 logger << DEBUG <<
"Notification received: "