TrainNeuralNetWork main() 运行流程
参数:
--train_file /home/li***/workspace/nplm/example/work/train.ngrams --validation_file /home/li***/workspace/nplm/example/work/validation.ngrams --num_epochs 10 --words_file /home/li***/workspace/nplm/example/work/words --model_prefix /home/li***/workspace/nplm/example/work/inferno.nnlm --learning_rate 1 --minibatch_size 8
1) 读取并获取各参数
2) 读取训练数据
// Read training data
vector<int> training_data_flat;
vec * training_data_flat_mmap;
data_size_t training_data_size; //num_tokens;
ip::managed_mapped_file mmap_file;
if (use_mmap_file == false) {
cerr << "Reading data from regular text file " << endl;
readDataFile(myParam.train_file, myParam.ngram_size, training_data_flat,
myParam.minibatch_size);
training_data_size = training_data_flat.size() / myParam.ngram_size;
}
readDataFile()方法会读取train.ngrams中所有的数据, 并放入到vector training_data_flat中, 即training_data_flat[0 ~ 2]为第一个ngram (这里ngram = 3-gram), training_data_flat[3~ 5]为第二个ngram, 依此类推.
Matrix<int, Dynamic, Dynamic> training_data;
//(training_data_flat.data(), myParam.ngram_size, training_data_size);
#ifdef MAP
cerr<<"Setting up eigen map"<<endl;
if (use_mmap_file == false) {
training_data = Map< Matrix<int,Dynamic,Dynamic> >(training_data_flat.data(), myParam.ngram_size, training_data_size);
} else {
training_data = Map< Matrix<int,Dynamic,Dynamic> >(training_data_flat_mmap->data().get(), myParam.ngram_size, training_data_size);
}
cerr<<"Created eigen map"<<endl;
#else
if (use_mmap_file == false) {
training_data = Map<Matrix<int, Dynamic, Dynamic> >(
training_data_flat.data(), myParam.ngram_size, training_data_size);
}
#endif
由于MAP没有预定义,运行training_data = Map< Matrix<int dynamic=""> >(training_data_flat.data(), myParam.ngram_size, training_data_size);得到的结果是,生成一个int类型的二维矩陈,其中行数为myParam.ngram_size (即3), 列数为training_data_size (即ngram的样例数). 二维矩阵的值可能根据vector training_data_flat进行初始化.
接着,
if (use_mmap_file == false && randomize == true) {
cerr << "Randomly shuffling data..." << endl;
// Randomly shuffle training data to improve learning
for (data_size_t i = training_data_size - 1; i > 0; i--) {
data_size_t j = uniform_int_distribution<data_size_t>(0, i - 1)(rng);
training_data.col(i).swap(training_data.col(j));
}
}
对training_data中的一些进行替换, 即对数据进行重新洗牌.
3) 读取校正数据
// Read validation data
vector<int> validation_data_flat;
int validation_data_size = 0;
if (myParam.validation_file != "") {
readDataFile(myParam.validation_file, myParam.ngram_size,
validation_data_flat);
validation_data_size = validation_data_flat.size() / myParam.ngram_size;
cerr << "Number of validation instances: " << validation_data_size << endl;
}
Map<Matrix<int, Dynamic, Dynamic> > validation_data(
validation_data_flat.data(), myParam.ngram_size, validation_data_size);
与训练数据处理的类似, 得到校正数据的二维矩阵validation_data
4) 读取输入单词/和输出单词文件
vector<string> input_words;
if (myParam.input_words_file != "") {
readWordsFile(myParam.input_words_file, input_words);
if (myParam.input_vocab_size == 0)
myParam.input_vocab_size = input_words.size();
}
vector<string> output_words;
if (myParam.output_words_file != "") {
readWordsFile(myParam.output_words_file, output_words);
if (myParam.output_vocab_size == 0)
myParam.output_vocab_size = output_words.size();
}
对每个ngram, 前n-1项数字对应的单词可根据输入单词文件找到原型. 而第n项数字对应的单词可根据输出单词文件找到原型.
5)
vector<data_size_t> unigram_counts(myParam.output_vocab_size);
for (data_size_t train_id = 0; train_id < training_data_size; train_id++) {
int output_word;
if (use_mmap_file == false) {
output_word = training_data(myParam.ngram_size - 1, train_id);
} else {
//cerr<<"mmap word is "<<training_data_flat_mmap->at((train_id+1)*myParam.ngram_size - 1)<<endl;
output_word = training_data_flat_mmap->at(
(train_id + 1) * myParam.ngram_size - 1);
}
//cerr<<"output word is "<<output_word<<endl;
unigram_counts[output_word] += 1;
}
multinomial<data_size_t> unigram(unigram_counts);
unigram_counts统计每个输出单词出现的次数, unigram_counts[i]表示编号为i的单词出现在ngram最后一项的次数.
unigram为一个多项式分布变量,以unigram_counts初始化每个输出单词的概率.
5) 模型训练
model nn;
// IF THE MODEL FILE HAS BEEN DEFINED, THEN
// LOAD THE NEURAL NETWORK MODEL
if (myParam.model_file != "") {
nn.read(myParam.model_file);
cerr << "reading the model" << endl;
} else {
nn.resize(myParam.ngram_size, myParam.input_vocab_size,
myParam.output_vocab_size, myParam.input_embedding_dimension,
myParam.num_hidden, myParam.output_embedding_dimension);
nn.initialize(rng, myParam.init_normal, myParam.init_range,
-log(myParam.output_vocab_size), myParam.parameter_update,
myParam.adagrad_epsilon);
nn.set_activation_function(
string_to_activation_function(myParam.activation_function));
}
loss_function_type loss_function = string_to_loss_function(
myParam.loss_function);
在model类(model.h)中, 定义了:
Matrix<double,Dynamic,Dynamic,Eigen::RowMajor> output_embedding_matrix,
input_embedding_matrix,
input_and_output_embedding_matrix;
Input_word_embeddings input_layer;
Output_word_embeddings output_layer;
其中input_embedding_matrix和output_embedding_matrix都是动态维数的double类型矩阵.
