Skip to content

Commit 267bc47

Browse files
committed
autoformatted and remove print statements
1 parent 89e637d commit 267bc47

File tree

2 files changed

+5430
-5047
lines changed

2 files changed

+5430
-5047
lines changed

common.cpp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -277,10 +277,6 @@ void gpt_split_words(std::string str, std::vector<std::string>& words) {
277277
str = m.suffix();
278278
}
279279

280-
for (auto word: words)
281-
{
282-
std::cout << word << std::endl;
283-
}
284280
}
285281

286282
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
@@ -325,7 +321,6 @@ std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::stri
325321
for (int i = 0; i < (int) word.size(); ){
326322
for (int j = word.size() - 1; j >= i; j--){
327323
auto cand = word.substr(i, j-i+1);
328-
std::cout << cand << std::endl;
329324
auto it = vocab.token_to_id.find(cand);
330325
if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab
331326
tokens.push_back(it->second);

0 commit comments

Comments
 (0)