21 #include "config_auto.h"
26 #if defined(__MINGW32__)
32 #include "allheaders.h"
59 int* max_x,
int* max_y) {
62 for (
int f = 0; f < features.
size(); ++f) {
63 if (features[f].x_ > *max_x) *max_x = features[f].x_;
64 if (features[f].y_ > *max_y) *max_y = features[f].y_;
71 #ifndef GRAPHICS_DISABLED
72 for (
int f = 0; f < features.
size(); ++f) {
73 FCOORD pos(features[f].x_, features[f].y_);
75 dir.from_direction(features[f].dir_);
103 for (
int i = 0; i < word_features.
size(); ++i) {
105 f.
x = word_features[i].x();
106 f.
y = word_features[i].y();
107 f.
dir = word_features[i].dir();
118 int x_diff = f1->
x_bucket - f2->x_bucket;
119 if (x_diff == 0)
return f1->y - f2->y;
127 : page_number_(0), vertical_text_(vertical) {
136 const char* imagedata,
int imagedatasize,
137 const char* truth_text,
const char* box_text) {
144 memcpy(&
image_data->image_data_[0], imagedata, imagedatasize);
146 if (truth_text ==
nullptr || truth_text[0] ==
'\0') {
147 tprintf(
"Error: No text corresponding to page %d from image %s!\n",
157 }
else if (truth_text !=
nullptr && truth_text[0] !=
'\0' &&
167 if (!imagefilename_.
Serialize(fp))
return false;
168 if (!fp->
Serialize(&page_number_))
return false;
169 if (!image_data_.
Serialize(fp))
return false;
170 if (!language_.
Serialize(fp))
return false;
171 if (!transcription_.
Serialize(fp))
return false;
175 int8_t vertical = vertical_text_;
192 vertical_text_ = vertical != 0;
214 SetPixInternal(pix, &image_data_);
219 return GetPixInternal(image_data_);
229 int* scaled_width,
int* scaled_height,
232 int input_height = 0;
235 input_width = pixGetWidth(src_pix);
236 input_height = pixGetHeight(src_pix);
237 if (target_height == 0) {
238 target_height = std::min(input_height, max_height);
240 float im_factor =
static_cast<float>(target_height) / input_height;
241 if (scaled_width !=
nullptr)
243 if (scaled_height !=
nullptr)
244 *scaled_height = target_height;
246 Pix* pix = pixScale(src_pix, im_factor, im_factor);
247 if (pix ==
nullptr) {
248 tprintf(
"Scaling pix of size %d, %d by factor %g made null pix!!\n",
249 input_width, input_height, im_factor);
251 if (scaled_width !=
nullptr) *scaled_width = pixGetWidth(pix);
252 if (scaled_height !=
nullptr) *scaled_height = pixGetHeight(pix);
253 pixDestroy(&src_pix);
254 if (
boxes !=
nullptr) {
257 for (
int b = 0; b < boxes_.
size(); ++b) {
258 TBOX box = boxes_[b];
259 box.
scale(im_factor);
264 TBOX box(0, 0, im_factor * input_width, target_height);
268 if (scale_factor !=
nullptr) *scale_factor = im_factor;
273 return image_data_.
size();
278 #ifndef GRAPHICS_DISABLED
279 const int kTextSize = 64;
282 if (pix ==
nullptr)
return;
283 int width = pixGetWidth(pix);
284 int height = pixGetHeight(pix);
285 auto* win =
new ScrollView(
"Imagedata", 100, 100,
286 2 * (width + 2 * kTextSize),
287 2 * (height + 4 * kTextSize),
288 width + 10, height + 3 * kTextSize,
true);
289 win->Image(pix, 0, height - 1);
294 int text_size = kTextSize;
295 if (!boxes_.
empty() && boxes_[0].height() * 2 < text_size)
296 text_size = boxes_[0].height() * 2;
297 win->TextAttributes(
"Arial", text_size,
false,
false,
false);
298 if (!boxes_.
empty()) {
299 for (
int b = 0; b < boxes_.
size(); ++b) {
301 win->Text(boxes_[b].left(), height + kTextSize, box_texts_[b].
string());
306 win->Text(0, height + kTextSize * 2, transcription_.
string());
319 for (
int i = 0; i < box_pages.
size(); ++i) {
320 if (page_number_ >= 0 && box_pages[i] != page_number_)
continue;
321 transcription_ += texts[i];
334 ret = pixWriteMem(&data, &size, pix, IFF_PNG);
336 ret = pixWriteMem(&data, &size, pix, IFF_PNM);
350 reinterpret_cast<const unsigned char*
>(&
image_data[0]);
364 true, &
boxes, &texts,
nullptr,
369 tprintf(
"Error: No boxes for page %d from image %s!\n",
370 page_number_, imagefilename_.
string());
379 document_data->ReCachePages();
384 : document_name_(name),
401 pages_offset_ = start_page;
402 return ReCachePages();
410 document_name_ = filename;
412 max_memory_ = max_memory;
421 if (!pages_.Serialize(&fp) || !fp.
CloseWrite(filename, writer)) {
422 tprintf(
"Serialize failed: %s\n", filename);
431 return pages_.Serialize(&fp);
437 pages_.push_back(page);
447 if (pages_offset_ == index)
return;
448 pages_offset_ = index;
460 bool needs_loading = pages_offset_ != index;
465 #if defined(__MINGW32__)
468 std::this_thread::sleep_for(std::chrono::seconds(1));
480 if (num_pages == 0 || index < 0) {
485 index =
Modulo(index, num_pages);
486 if (pages_offset_ <= index && index < pages_offset_ + pages_.size()) {
487 *page = pages_[index - pages_offset_];
503 tprintf(
"Unloaded document %s, saving %" PRId64
" memory\n",
504 document_name_.
string(), memory_saved);
514 int num_pages = pages_.size();
516 for (
int i = 0; i < num_pages; ++i) {
517 int src = random.
IntRand() % num_pages;
519 std::swap(pages_[src], pages_[
dest]);
525 bool DocumentData::ReCachePages() {
530 int loaded_pages = 0;
533 if (!fp.
Open(document_name_, reader_) ||
536 tprintf(
"Deserialize header failed: %s\n", document_name_.
string());
539 pages_offset_ %= loaded_pages;
543 for (page = 0; page < loaded_pages; ++page) {
544 if (page < pages_offset_ ||
545 (max_memory_ > 0 &&
memory_used() > max_memory_)) {
547 tprintf(
"Deserializeskip failed\n");
551 if (!pages_.DeSerializeElement(&fp))
break;
552 ImageData* image_data = pages_.back();
553 if (image_data->imagefilename().length() == 0) {
554 image_data->set_imagefilename(document_name_);
555 image_data->set_page_number(page);
557 set_memory_used(
memory_used() + image_data->MemoryUsed());
560 if (page < loaded_pages) {
561 tprintf(
"Deserialize failed: %s read %d/%d lines\n",
562 document_name_.
string(), page, loaded_pages);
565 tprintf(
"Loaded %d/%d lines (%d-%d) of document %s\n", pages_.size(),
566 loaded_pages, pages_offset_ + 1, pages_offset_ + pages_.size(),
569 set_total_pages(loaded_pages);
570 return !pages_.empty();
575 : num_pages_per_doc_(0), max_memory_(max_memory) {}
583 cache_strategy_ = cache_strategy;
584 int64_t fair_share_memory = 0;
589 fair_share_memory = max_memory_ / filenames.
size();
590 for (
int arg = 0; arg < filenames.
size(); ++arg) {
591 STRING filename = filenames[arg];
593 document->SetDocument(filename.
string(), fair_share_memory, reader);
596 if (!documents_.empty()) {
599 tprintf(
"Load of page 0 failed!\n");
606 documents_.push_back(data);
612 for (
int i = 0; i < documents_.size(); ++i) {
613 if (documents_[i]->document_name() == document_name)
614 return documents_[i];
625 if (num_pages_per_doc_ == 0) GetPageSequential(0);
626 return num_pages_per_doc_ * documents_.size();
629 int num_docs = documents_.size();
630 for (
int d = 0; d < num_docs; ++d) {
632 documents_[d]->GetPage(0);
633 total_pages += documents_[d]->NumPages();
641 const ImageData* DocumentCache::GetPageRoundRobin(
int serial) {
642 int num_docs = documents_.size();
643 int doc_index = serial % num_docs;
644 const ImageData* doc = documents_[doc_index]->GetPage(serial / num_docs);
645 for (
int offset = 1; offset <=
kMaxReadAhead && offset < num_docs; ++offset) {
646 doc_index = (serial + offset) % num_docs;
647 int page = (serial + offset) / num_docs;
648 documents_[doc_index]->LoadPageInBackground(page);
656 const ImageData* DocumentCache::GetPageSequential(
int serial) {
657 int num_docs = documents_.size();
659 if (num_pages_per_doc_ == 0) {
661 documents_[0]->GetPage(0);
662 num_pages_per_doc_ = documents_[0]->NumPages();
663 if (num_pages_per_doc_ == 0) {
664 tprintf(
"First document cannot be empty!!\n");
668 if (serial / num_pages_per_doc_ % num_docs > 0) documents_[0]->UnCache();
670 int doc_index = serial / num_pages_per_doc_ % num_docs;
671 const ImageData* doc =
672 documents_[doc_index]->GetPage(serial % num_pages_per_doc_);
675 int64_t total_memory = 0;
676 for (
int d = 0; d < num_docs; ++d) {
677 total_memory += documents_[d]->memory_used();
679 if (total_memory >= max_memory_) {
685 int num_in_front = CountNeighbourDocs(doc_index, 1);
686 for (
int offset = num_in_front - 2;
687 offset > 1 && total_memory >= max_memory_; --offset) {
688 int next_index = (doc_index + offset) % num_docs;
689 total_memory -= documents_[next_index]->UnCache();
694 int num_behind = CountNeighbourDocs(doc_index, -1);
695 for (
int offset = num_behind; offset < 0 && total_memory >= max_memory_;
697 int next_index = (doc_index + offset + num_docs) % num_docs;
698 total_memory -= documents_[next_index]->UnCache();
701 int next_index = (doc_index + 1) % num_docs;
702 if (!documents_[next_index]->IsCached() && total_memory < max_memory_) {
703 documents_[next_index]->LoadPageInBackground(0);
710 int DocumentCache::CountNeighbourDocs(
int index,
int dir) {
711 int num_docs = documents_.size();
712 for (
int offset = dir; abs(offset) < num_docs; offset += dir) {
713 int offset_index = (index + offset + num_docs) % num_docs;
714 if (!documents_[offset_index]->IsCached())
return offset - dir;