Go to the documentation of this file.
18 #define _USE_MATH_DEFINES // needed to get definition of M_SQRT1_2
22 #include "config_auto.h"
28 #include "allheaders.h"
36 #ifdef INCLUDE_TENSORFLOW
75 : randomly_rotate_(false),
83 sub_trainer_(nullptr) {
91 const char* model_base,
const char* checkpoint_name,
92 int debug_interval, int64_t max_memory)
93 : randomly_rotate_(false),
94 training_data_(max_memory),
95 file_reader_(file_reader),
96 file_writer_(file_writer),
97 checkpoint_reader_(checkpoint_reader),
98 checkpoint_writer_(checkpoint_writer),
99 sub_trainer_(nullptr),
130 const char* old_traineddata) {
133 tprintf(
"Loaded file %s, unpacking...\n", filename);
136 if (((old_traineddata ==
nullptr || *old_traineddata ==
'\0') &&
138 filename == old_traineddata) {
143 if (old_traineddata ==
nullptr || *old_traineddata ==
'\0') {
144 tprintf(
"Must supply the old traineddata for code conversion!\n");
156 std::vector<int> code_map =
MapRecoder(old_chset, old_recoder);
173 int net_flags,
float weight_range,
174 float learning_rate,
float momentum,
182 append_index, net_flags, weight_range,
187 tprintf(
"Built network:%s from request %s\n",
190 "Training parameters:\n Debug interval = %d,"
191 " weights = %g, learning rate = %g, momentum=%g\n",
199 #ifdef INCLUDE_TENSORFLOW
202 TFNetwork* tf_net =
new TFNetwork(
"TensorFlow");
205 tprintf(
"InitFromProtoStr failed!!\n");
229 for (
int i = 0; i <
ET_COUNT; ++i) {
242 const ImageData* trainingdata,
int iteration,
double min_dict_ratio,
243 double dict_ratio_step,
double max_dict_ratio,
double min_cert_offset,
244 double cert_offset_step,
double max_cert_offset,
STRING* results) {
266 for (
double r = min_dict_ratio; r < max_dict_ratio; r += dict_ratio_step) {
267 for (
double c = min_cert_offset; c < max_cert_offset;
268 c += cert_offset_step) {
270 search.ExtractBestPathAsLabels(&ocr_labels, &xcoords);
275 if ((r == min_dict_ratio && c == min_cert_offset) ||
276 !std::isfinite(word_error)) {
279 tprintf(
"r=%g, c=%g, truth=%s, ocr=%s, wderr=%g, truth[0]=%d\n", r, c,
300 bool randomly_rotate) {
336 *log_msg +=
UpdateErrorGraph(iteration, error_rate, rec_model_data, tester);
349 *log_msg +=
" failed to write best model:";
351 *log_msg +=
" wrote best model:";
354 *log_msg += best_model_name;
359 *log_msg +=
UpdateErrorGraph(iteration, error_rate, rec_model_data, tester);
363 *log_msg +=
"\nDivergence! ";
380 result = sub_trainer_result !=
STR_NONE;
388 *log_msg +=
" failed to write checkpoint.";
390 *log_msg +=
" wrote checkpoint.";
411 *log_msg += intro_str;
437 if (!error_buffer.Serialize(fp))
return false;
441 uint8_t amount = serialize_amount;
442 if (!fp->
Serialize(&amount))
return false;
443 if (serialize_amount ==
LIGHT)
return true;
458 if (!sub_data.
Serialize(fp))
return false;
472 tprintf(
"Warning: LSTMTrainer deserialized an LSTMRecognizer!\n");
481 if (!error_buffer.DeSerialize(fp))
return false;
487 if (amount ==
LIGHT)
return true;
501 if (sub_data.
empty()) {
519 *log_msg +=
" Failed to revert to previous best for trial!";
523 log_msg->
add_str_int(
" Trial sub_trainer_ from iteration ",
548 double sub_margin = (training_error - sub_error) / sub_error;
557 int target_iteration =
562 STRING batch_log =
"Sub:";
566 *log_msg += batch_log;
568 sub_margin = (training_error - sub_error) / sub_error;
576 log_msg->
add_str_int(
" Sub trainer wins at iteration ",
593 log_msg->
add_str_int(
"\nReduced learning rate on layers: ", num_reduced);
615 int num_layers = layers.
size();
620 for (
int i = 0; i < LR_COUNT; ++i) {
624 double momentum_factor = 1.0 / (1.0 -
momentum_);
627 for (
int i = 0; i < num_layers; ++i) {
632 for (
int s = 0; s < num_samples; ++s) {
634 for (
int ww = 0; ww < LR_COUNT; ++ww) {
636 float ww_factor = momentum_factor;
637 if (ww == LR_DOWN) ww_factor *= factor;
644 for (
int i = 0; i < num_layers; ++i) {
645 if (num_weights[i] == 0)
continue;
653 if (trainingdata ==
nullptr)
continue;
657 for (
int i = 0; i < num_layers; ++i) {
658 if (num_weights[i] == 0)
continue;
666 layer->
Update(0.0, 0.0, 0.0, 0);
670 float before_bad = bad_sums[ww][i];
671 float before_ok = ok_sums[ww][i];
673 &ok_sums[ww][i], &bad_sums[ww][i]);
675 bad_sums[ww][i] + ok_sums[ww][i] - before_bad - before_ok;
677 bad_frac = (bad_sums[ww][i] - before_bad) / bad_frac;
683 for (
int i = 0; i < num_layers; ++i) {
684 if (num_weights[i] == 0)
continue;
687 double total_down = bad_sums[LR_DOWN][i] + ok_sums[LR_DOWN][i];
688 double total_same = bad_sums[LR_SAME][i] + ok_sums[LR_SAME][i];
689 double frac_down = bad_sums[LR_DOWN][i] / total_down;
690 double frac_same = bad_sums[LR_SAME][i] / total_same;
692 lr * factor, 100.0 * frac_down, lr, 100.0 * frac_same);
701 if (num_lowered == 0) {
703 for (
int i = 0; i < num_layers; ++i) {
704 if (num_weights[i] > 0) {
720 tprintf(
"Empty truth string!\n");
728 if (unicharset.
encode_string(cleaned.c_str(),
true, &internal_labels,
nullptr,
731 for (
int i = 0; i < internal_labels.
size(); ++i) {
732 if (recoder !=
nullptr) {
737 for (
int j = 0; j < len; ++j) {
751 if (success)
return true;
753 tprintf(
"Encoding of string failed! Failure bytes:");
754 while (err_index < cleaned.size()) {
755 tprintf(
" %x", cleaned[err_index++]);
784 #ifndef GRAPHICS_DISABLED
788 #endif // GRAPHICS_DISABLED
799 if (trainingdata ==
nullptr) {
800 tprintf(
"Null trainingdata.\n");
808 tprintf(
"Can't encode transcription: '%s' in language '%s'\n",
813 bool upside_down =
false;
823 for (
int c = 0; c < truth_labels.
size(); ++c) {
831 while (w < truth_labels.
size() &&
834 if (w == truth_labels.
size()) {
835 tprintf(
"Blank transcription: %s\n",
842 if (!
RecognizeLine(*trainingdata, invert, debug, invert, upside_down,
843 &image_scale, &inputs, fwd_outputs)) {
844 tprintf(
"Image not trainable\n");
851 tprintf(
"Compute simple targets failed!\n");
854 }
else if (loss_type ==
LT_CTC) {
856 tprintf(
"Compute CTC targets failed!\n");
860 tprintf(
"Logistic outputs not implemented yet!\n");
867 if (loss_type !=
LT_CTC) {
879 if (truth_text != ocr_text) {
880 tprintf(
"Iteration %d: BEST OCR TEXT : %s\n",
889 trainingdata->
page_number(), delta_error == 0.0 ?
"(Perfect)" :
"");
891 if (delta_error == 0.0)
return PERFECT;
910 const char* data,
int size) {
912 tprintf(
"Warning: data size is 0 in LSTMTrainer::ReadLocalTrainingDump\n");
925 recognizer_data.
size());
944 filename +=
".checkpoint";
961 std::vector<int> code_map(num_new_codes, -1);
962 for (
int c = 0; c < num_new_codes; ++c) {
966 for (
int uid = 0; uid <= num_new_unichars; ++uid) {
970 while (code_index < length && codes(code_index) != c) ++code_index;
971 if (code_index == length)
continue;
974 uid < num_new_unichars
976 : old_chset.
size() - 1;
977 if (old_uid == INVALID_UNICHAR_ID)
continue;
980 if (code_index < old_recoder.
EncodeUnichar(old_uid, &old_codes)) {
981 old_code = old_codes(code_index);
985 code_map[c] = old_code;
998 "Must provide a traineddata containing lstm_unicharset and"
999 " lstm_recoder!\n" !=
nullptr);
1035 if (truth_text.
string() ==
nullptr || truth_text.
length() <= 0) {
1036 tprintf(
"Empty truth string at decode time!\n");
1045 tprintf(
"Iteration %d: GROUND TRUTH : %s\n",
1047 if (truth_text != text) {
1048 tprintf(
"Iteration %d: ALIGNED TRUTH : %s\n",
1052 tprintf(
"TRAINING activation path for truth string %s\n",
1067 const char* window_name,
ScrollView** window) {
1068 #ifndef GRAPHICS_DISABLED // do nothing if there's no graphics.
1069 int width = targets.
Width();
1073 for (
int c = 0; c < num_features; ++c) {
1077 for (
int t = 0; t < width; ++t) {
1078 double target = targets.
f(t)[c];
1082 (*window)->SetCursor(t - 1, 0);
1085 (*window)->DrawTo(t, target);
1086 }
else if (start_t >= 0) {
1087 (*window)->DrawTo(t, 0);
1088 (*window)->DrawTo(start_t - 1, 0);
1093 (*window)->DrawTo(width, 0);
1094 (*window)->DrawTo(start_t - 1, 0);
1097 (*window)->Update();
1098 #endif // GRAPHICS_DISABLED
1106 if (truth_labels.
size() > targets->
Width()) {
1107 tprintf(
"Error: transcription %s too long to fit into target of width %d\n",
1111 for (
int i = 0; i < truth_labels.
size() && i < targets->Width(); ++i) {
1114 for (
int i = truth_labels.
size(); i < targets->Width(); ++i) {
1135 double char_error,
double word_error) {
1155 double total_error = 0.0;
1156 int width = deltas.
Width();
1158 for (
int t = 0; t < width; ++t) {
1159 const float* class_errs = deltas.
f(t);
1160 for (
int c = 0; c < num_classes; ++c) {
1161 double error = class_errs[c];
1162 total_error += error * error;
1165 return sqrt(total_error / (width * num_classes));
1175 int width = deltas.
Width();
1177 for (
int t = 0; t < width; ++t) {
1178 const float* class_errs = deltas.
f(t);
1179 for (
int c = 0; c < num_classes; ++c) {
1180 float abs_delta = fabs(class_errs[c]);
1183 if (0.5 <= abs_delta)
1187 return static_cast<double>(num_errors) / width;
1196 for (
int i = 0; i < truth_str.
size(); ++i) {
1198 ++label_counts[truth_str[i]];
1202 for (
int i = 0; i < ocr_str.
size(); ++i) {
1204 --label_counts[ocr_str[i]];
1207 int char_errors = 0;
1208 for (
int i = 0; i < label_counts.
size(); ++i) {
1209 char_errors += abs(label_counts[i]);
1211 if (truth_size == 0) {
1212 return (char_errors == 0) ? 0.0 : 1.0;
1214 return static_cast<double>(char_errors) / truth_size;
1220 using StrMap = std::unordered_map<std::string, int, std::hash<std::string>>;
1222 truth_str->
split(
' ', &truth_words);
1223 if (truth_words.
empty())
return 0.0;
1224 ocr_str->
split(
' ', &ocr_words);
1226 for (
int i = 0; i < truth_words.
size(); ++i) {
1227 std::string truth_word(truth_words[i].
string());
1228 auto it = word_counts.find(truth_word);
1229 if (it == word_counts.end())
1230 word_counts.insert(std::make_pair(truth_word, 1));
1234 for (
int i = 0; i < ocr_words.
size(); ++i) {
1235 std::string ocr_word(ocr_words[i].
string());
1236 auto it = word_counts.find(ocr_word);
1237 if (it == word_counts.end())
1238 word_counts.insert(std::make_pair(ocr_word, -1));
1242 int word_recall_errs = 0;
1243 for (StrMap::const_iterator it = word_counts.begin(); it != word_counts.end();
1245 if (it->second > 0) word_recall_errs += it->second;
1247 return static_cast<double>(word_recall_errs) / truth_words.
size();
1257 double buffer_sum = 0.0;
1259 double mean = buffer_sum / mean_count;
1273 tprintf(
"Mean rms=%g%%, delta=%g%%, train=%g%%(%g%%), skip ratio=%g%%\n",
1321 double two_percent_more = error_rate + 2.0;
1328 tprintf(
"2 Percent improvement time=%d, best error was %g @ %d\n",
1333 if (tester !=
nullptr) {
int IntCastRounded(double x)
void init_to_size(int size, const T &t)
const double kMinDivergenceRate
virtual R Run(A1, A2, A3)=0
void RecognizeLine(const ImageData &image_data, bool invert, bool debug, double worst_dict_cert, const TBOX &line_box, PointerVector< WERD_RES > *words, int lstm_choice_mode=0)
static std::string CleanupString(const char *utf8_str)
int CurrentTrainingStage() const
bool ReadTrainingDump(const GenericVector< char > &data, LSTMTrainer *trainer) const
bool LoadDataFromFile(const char *filename, GenericVector< char > *data)
const char * string() const
void SubtractAllFromFloat(const NetworkIO &src)
NetworkScratch scratch_space_
void SaveRecognitionDump(GenericVector< char > *data) const
const int kNumAdjustmentIterations
void UpdateErrorBuffer(double new_error, ErrorTypes type)
void add_str_int(const char *str, int number)
virtual void CountAlternators(const Network &other, double *same, double *changed) const
LossType OutputLossType() const
bool encode_string(const char *str, bool give_up_on_failure, GenericVector< UNICHAR_ID > *encoding, GenericVector< char > *lengths, int *encoded_length) const
bool DeSerialize(char *data, size_t count=1)
constexpr size_t countof(T const (&)[N]) noexcept
static constexpr float kMinCertainty
double NewSingleError(ErrorTypes type) const
double ComputeRMSError(const NetworkIO &deltas)
double learning_rate() const
double ComputeWordError(STRING *truth_str, STRING *ocr_str)
bool AnySuspiciousTruth(float confidence_thr) const
bool TransitionTrainingStage(float error_threshold)
void DisplayForward(const NetworkIO &inputs, const GenericVector< int > &labels, const GenericVector< int > &label_coords, const char *window_name, ScrollView **window)
GenericVector< char > best_trainer_
SVEvent * AwaitEvent(SVEventType type)
virtual void SetEnableTraining(TrainingState state)
virtual StaticShape InputShape() const
int last_perfect_training_iteration_
CheckPointReader checkpoint_reader_
void SetVersionString(const std::string &v_str)
void Resize(const NetworkIO &src, int num_features)
SubTrainerResult UpdateSubtrainer(STRING *log_msg)
bool SaveFile(const STRING &filename, FileWriter writer) const
virtual STRING spec() const
int sample_iteration() const
const int kMinStartedErrorRate
void ScaleLayerLearningRate(const STRING &id, double factor)
static void ClearWindow(bool tess_coords, const char *window_name, int width, int height, ScrollView **window)
virtual void Update(float learning_rate, float momentum, float adam_beta, int num_samples)
STRING UpdateErrorGraph(int iteration, double error_rate, const GenericVector< char > &model_data, TestCallback tester)
bool ReadLocalTrainingDump(const TessdataManager *mgr, const char *data, int size)
bool DeSerialize(const TessdataManager *mgr, TFile *fp)
void LogIterations(const char *intro_str, STRING *log_msg) const
CheckPointWriter checkpoint_writer_
const STRING & name() const
void LabelsFromOutputs(const NetworkIO &outputs, GenericVector< int > *labels, GenericVector< int > *xcoords)
DLLSYM void tprintf(const char *format,...)
double error_rates_[ET_COUNT]
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
bool LoadAllTrainingData(const GenericVector< STRING > &filenames, CachingStrategy cache_strategy, bool randomly_rotate)
void SetActivations(int t, int label, float ok_score)
bool DeSerialize(const TessdataManager *mgr, TFile *fp)
static bool ComputeCTCTargets(const GenericVector< int > &truth_labels, int null_char, const GENERIC_2D_ARRAY< float > &outputs, NetworkIO *targets)
void StartSubtrainer(STRING *log_msg)
void SetIteration(int iteration)
double ComputeCharError(const GenericVector< int > &truth_str, const GenericVector< int > &ocr_str)
bool Serialize(SerializeAmount serialize_amount, const TessdataManager *mgr, TFile *fp) const
bool SaveTrainingDump(SerializeAmount serialize_amount, const LSTMTrainer *trainer, GenericVector< char > *data) const
int checkpoint_iteration_
void DebugActivationPath(const NetworkIO &outputs, const GenericVector< int > &labels, const GenericVector< int > &xcoords)
bool ComputeTextTargets(const NetworkIO &outputs, const GenericVector< int > &truth_labels, NetworkIO *targets)
bool LoadDocuments(const GenericVector< STRING > &filenames, CachingStrategy cache_strategy, FileReader reader)
bool Serialize(FILE *fp) const
double worst_error_rates_[ET_COUNT]
const double kLearningRateDecay
_ConstTessMemberResultCallback_5_0< false, R, T1, P1, P2, P3, P4, P5 >::base * NewPermanentTessCallback(const T1 *obj, R(T2::*member)(P1, P2, P3, P4, P5) const, typename Identity< P1 >::type p1, typename Identity< P2 >::type p2, typename Identity< P3 >::type p3, typename Identity< P4 >::type p4, typename Identity< P5 >::type p5)
DocumentCache training_data_
LIST search(LIST list, void *key, int_compare is_equal)
bool DeSerialize(TFile *fp)
std::vector< int > MapRecoder(const UNICHARSET &old_chset, const UnicharCompress &old_recoder) const
GenericVector< double > best_error_history_
GenericVector< double > error_buffers_[ET_COUNT]
int prev_sample_iteration_
bool has_special_codes() const
float error_rate_of_last_saved_best_
int learning_iteration() const
void FillErrorBuffer(double new_error, ErrorTypes type)
virtual void DebugWeights()=0
const ImageData * TrainOnLine(LSTMTrainer *samples_trainer, bool batch)
void Decode(const NetworkIO &output, double dict_ratio, double cert_offset, double worst_dict_cert, const UNICHARSET *charset, int lstm_choice_mode=0)
int32_t improvement_steps_
bool MaintainCheckpoints(TestCallback tester, STRING *log_msg)
bool SaveTraineddata(const STRING &filename)
GenericVector< char > worst_model_data_
const GenericVector< TBOX > & boxes() const
bool(*)(const STRING &, GenericVector< char > *) FileReader
const int kNumPagesPerBatch
const STRING & transcription() const
const double kImprovementFraction
std::string VersionString() const
bool Serialize(const char *data, size_t count=1)
bool InitNetwork(const STRING &network_spec, int append_index, int net_flags, float weight_range, float learning_rate, float momentum, float adam_beta)
void OpenWrite(GenericVector< char > *data)
const double kStageTransitionThreshold
LSTMTrainer * sub_trainer_
int training_iteration() const
const GENERIC_2D_ARRAY< float > & float_array() const
bool DeSerialize(bool swap, FILE *fp)
bool EncodeString(const STRING &str, GenericVector< int > *labels) const
virtual R Run(A1, A2, A3, A4)=0
double ComputeWinnerError(const NetworkIO &deltas)
double SignedRand(double range)
static void NormalizeProbs(NetworkIO *probs)
bool(*)(const GenericVector< char > &, const STRING &) FileWriter
bool SimpleTextOutput() const
const STRING & language() const
bool TestFlag(NetworkFlags flag) const
Network * GetLayer(const STRING &id) const
GenericVector< STRING > EnumerateLayers() const
void DisplayTargets(const NetworkIO &targets, const char *window_name, ScrollView **window)
bool load_from_file(const char *const filename, bool skip_fragments)
void PrepareLogMsg(STRING *log_msg) const
virtual int RemapOutputs(int old_no, const std::vector< int > &code_map)
bool TryLoadingCheckpoint(const char *filename, const char *old_traineddata)
void ReduceLearningRates(LSTMTrainer *samples_trainer, STRING *log_msg)
const int kErrorGraphInterval
virtual bool Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas)=0
bool DebugLSTMTraining(const NetworkIO &inputs, const ImageData &trainingdata, const NetworkIO &fwd_outputs, const GenericVector< int > &truth_labels, const NetworkIO &outputs)
void add_str_double(const char *str, double number)
virtual StaticShape OutputShape(const StaticShape &input_shape) const
const double kBestCheckpointFraction
bool ComputeCTCTargets(const GenericVector< int > &truth_labels, NetworkIO *outputs, NetworkIO *targets)
const UNICHARSET & GetUnicharset() const
Trainability GridSearchDictParams(const ImageData *trainingdata, int iteration, double min_dict_ratio, double dict_ratio_step, double max_dict_ratio, double min_cert_offset, double cert_offset_step, double max_cert_offset, STRING *results)
float GetLayerLearningRate(const STRING &id) const
bool GetComponent(TessdataType type, TFile *fp)
bool LoadCharsets(const TessdataManager *mgr)
int InitTensorFlowNetwork(const std::string &tf_proto)
void ExtractBestPathAsLabels(GenericVector< int > *labels, GenericVector< int > *xcoords) const
GenericVector< char > best_model_data_
Trainability PrepareForBackward(const ImageData *trainingdata, NetworkIO *fwd_outputs, NetworkIO *targets)
static const int kRollingBufferSize_
bool Open(const STRING &filename, FileReader reader)
int EncodeUnichar(int unichar_id, RecodedCharID *code) const
int32_t sample_iteration_
bool Init(const char *data_file_name)
const int kMinStallIterations
int ReduceLayerLearningRates(double factor, int num_samples, LSTMTrainer *samples_trainer)
bool Serialize(const TessdataManager *mgr, TFile *fp) const
GenericVector< int > best_error_iterations_
STRING DecodeLabels(const GenericVector< int > &labels)
void ScaleLearningRate(double factor)
double best_error_rates_[ET_COUNT]
void OverwriteEntry(TessdataType type, const char *data, int size)
const double kHighConfidence
int32_t training_iteration_
STRING DumpFilename() const
const char * c_str() const
bool SaveDataToFile(const GenericVector< char > &data, const STRING &filename)
const STRING & imagefilename() const
void split(char c, GenericVector< STRING > *splited)
@ TESSDATA_LSTM_UNICHARSET
static bool InitNetwork(int num_outputs, STRING network_spec, int append_index, int net_flags, float weight_range, TRand *randomizer, Network **network)
const double kSubTrainerMarginFraction
double ComputeErrorRates(const NetworkIO &deltas, double char_error, double word_error)