tesseract  4.1.1
boxread.cpp
Go to the documentation of this file.
1 /**********************************************************************
2  * File: boxread.cpp
3  * Description: Read data from a box file.
4  * Author: Ray Smith
5  *
6  * (C) Copyright 2007, Google Inc.
7  ** Licensed under the Apache License, Version 2.0 (the "License");
8  ** you may not use this file except in compliance with the License.
9  ** You may obtain a copy of the License at
10  ** http://www.apache.org/licenses/LICENSE-2.0
11  ** Unless required by applicable law or agreed to in writing, software
12  ** distributed under the License is distributed on an "AS IS" BASIS,
13  ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  ** See the License for the specific language governing permissions and
15  ** limitations under the License.
16  *
17  **********************************************************************/
18 
19 #include "boxread.h"
20 #include <cstring> // for strchr, strcmp, strrchr
21 #include <locale> // for std::locale::classic
22 #include <sstream> // for std::stringstream
23 #include "errcode.h" // for ERRCODE, TESSEXIT
24 #include "fileerr.h" // for CANTOPENFILE
25 #include "genericvector.h" // for GenericVector
26 #include "helpers.h" // for chomp_string
27 #include "rect.h" // for TBOX
28 #include "strngs.h" // for STRING
29 #include "tprintf.h" // for tprintf
30 #include "unichar.h" // for UNICHAR
31 
32 // Special char code used to identify multi-blob labels.
33 static const char* kMultiBlobLabelCode = "WordStr";
34 
35 // Open the boxfile based on the given image filename.
36 FILE* OpenBoxFile(const STRING& fname) {
37  STRING filename = BoxFileName(fname);
38  FILE* box_file = nullptr;
39  if (!(box_file = fopen(filename.string(), "rb"))) {
40  CANTOPENFILE.error("read_next_box", TESSEXIT, "Can't open box file %s",
41  filename.string());
42  }
43  return box_file;
44 }
45 
46 // Reads all boxes from the given filename.
47 // Reads a specific target_page number if >= 0, or all pages otherwise.
48 // Skips blanks if skip_blanks is true.
49 // The UTF-8 label of the box is put in texts, and the full box definition as
50 // a string is put in box_texts, with the corresponding page number in pages.
51 // Each of the output vectors is optional (may be nullptr).
52 // Returns false if no boxes are found.
53 bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING& filename,
54  GenericVector<TBOX>* boxes,
55  GenericVector<STRING>* texts,
56  GenericVector<STRING>* box_texts,
57  GenericVector<int>* pages) {
58  GenericVector<char> box_data;
59  if (!tesseract::LoadDataFromFile(BoxFileName(filename).c_str(), &box_data))
60  return false;
61  // Convert the array of bytes to a string, so it can be used by the parser.
62  box_data.push_back('\0');
63  return ReadMemBoxes(target_page, skip_blanks, &box_data[0],
64  /*continue_on_failure*/ true, boxes, texts, box_texts,
65  pages);
66 }
67 
68 // Reads all boxes from the string. Otherwise, as ReadAllBoxes.
69 bool ReadMemBoxes(int target_page, bool skip_blanks, const char* box_data,
70  bool continue_on_failure,
71  GenericVector<TBOX>* boxes,
72  GenericVector<STRING>* texts,
73  GenericVector<STRING>* box_texts,
74  GenericVector<int>* pages) {
75  STRING box_str(box_data);
77  box_str.split('\n', &lines);
78  if (lines.empty()) return false;
79  int num_boxes = 0;
80  for (int i = 0; i < lines.size(); ++i) {
81  int page = 0;
82  STRING utf8_str;
83  TBOX box;
84  if (!ParseBoxFileStr(lines[i].string(), &page, &utf8_str, &box)) {
85  if (continue_on_failure)
86  continue;
87  else
88  return false;
89  }
90  if (skip_blanks && (utf8_str == " " || utf8_str == "\t")) continue;
91  if (target_page >= 0 && page != target_page) continue;
92  if (boxes != nullptr) boxes->push_back(box);
93  if (texts != nullptr) texts->push_back(utf8_str);
94  if (box_texts != nullptr) {
95  STRING full_text;
96  MakeBoxFileStr(utf8_str.string(), box, target_page, &full_text);
97  box_texts->push_back(full_text);
98  }
99  if (pages != nullptr) pages->push_back(page);
100  ++num_boxes;
101  }
102  return num_boxes > 0;
103 }
104 
105 // Returns the box file name corresponding to the given image_filename.
106 STRING BoxFileName(const STRING& image_filename) {
107  STRING box_filename = image_filename;
108  const char *lastdot = strrchr(box_filename.string(), '.');
109  if (lastdot != nullptr)
110  box_filename.truncate_at(lastdot - box_filename.string());
111 
112  box_filename += ".box";
113  return box_filename;
114 }
115 
116 // TODO(rays) convert all uses of ReadNextBox to use the new ReadAllBoxes.
117 // Box files are used ONLY DURING TRAINING, but by both processes of
118 // creating tr files with tesseract, and unicharset_extractor.
119 // ReadNextBox factors out the code to interpret a line of a box
120 // file so that applybox and unicharset_extractor interpret the same way.
121 // This function returns the next valid box file utf8 string and coords
122 // and returns true, or false on eof (and closes the file).
123 // It ignores the utf8 file signature ByteOrderMark (U+FEFF=EF BB BF), checks
124 // for valid utf-8 and allows space or tab between fields.
125 // utf8_str is set with the unichar string, and bounding box with the box.
126 // If there are page numbers in the file, it reads them all.
127 bool ReadNextBox(int *line_number, FILE* box_file,
128  STRING* utf8_str, TBOX* bounding_box) {
129  return ReadNextBox(-1, line_number, box_file, utf8_str, bounding_box);
130 }
131 
132 // As ReadNextBox above, but get a specific page number. (0-based)
133 // Use -1 to read any page number. Files without page number all
134 // read as if they are page 0.
135 bool ReadNextBox(int target_page, int *line_number, FILE* box_file,
136  STRING* utf8_str, TBOX* bounding_box) {
137  int page = 0;
138  char buff[kBoxReadBufSize]; // boxfile read buffer
139  char *buffptr = buff;
140 
141  while (fgets(buff, sizeof(buff) - 1, box_file)) {
142  (*line_number)++;
143 
144  buffptr = buff;
145  const auto *ubuf = reinterpret_cast<const unsigned char*>(buffptr);
146  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf)
147  buffptr += 3; // Skip unicode file designation.
148  // Check for blank lines in box file
149  if (*buffptr == '\n' || *buffptr == '\0') continue;
150  // Skip blank boxes.
151  if (*buffptr == ' ' || *buffptr == '\t') continue;
152  if (*buffptr != '\0') {
153  if (!ParseBoxFileStr(buffptr, &page, utf8_str, bounding_box)) {
154  tprintf("Box file format error on line %i; ignored\n", *line_number);
155  continue;
156  }
157  if (target_page >= 0 && target_page != page)
158  continue; // Not on the appropriate page.
159  return true; // Successfully read a box.
160  }
161  }
162  fclose(box_file);
163  return false; // EOF
164 }
165 
166 // Parses the given box file string into a page_number, utf8_str, and
167 // bounding_box. Returns true on a successful parse.
168 // The box file is assumed to contain box definitions, one per line, of the
169 // following format for blob-level boxes:
170 // <UTF8 str> <left> <bottom> <right> <top> <page id>
171 // and for word/line-level boxes:
172 // WordStr <left> <bottom> <right> <top> <page id> #<space-delimited word str>
173 // See applyybox.cpp for more information.
174 bool ParseBoxFileStr(const char* boxfile_str, int* page_number,
175  STRING* utf8_str, TBOX* bounding_box) {
176  *bounding_box = TBOX(); // Initialize it to empty.
177  *utf8_str = "";
178  char uch[kBoxReadBufSize];
179  const char *buffptr = boxfile_str;
180  // Read the unichar without messing up on Tibetan.
181  // According to issue 253 the utf-8 surrogates 85 and A0 are treated
182  // as whitespace by sscanf, so it is more reliable to just find
183  // ascii space and tab.
184  int uch_len = 0;
185  // Skip unicode file designation, if present.
186  const auto *ubuf = reinterpret_cast<const unsigned char*>(buffptr);
187  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf)
188  buffptr += 3;
189  // Allow a single blank as the UTF-8 string. Check for empty string and
190  // then blindly eat the first character.
191  if (*buffptr == '\0') return false;
192  do {
193  uch[uch_len++] = *buffptr++;
194  } while (*buffptr != '\0' && *buffptr != ' ' && *buffptr != '\t' &&
195  uch_len < kBoxReadBufSize - 1);
196  uch[uch_len] = '\0';
197  if (*buffptr != '\0') ++buffptr;
198  int x_min = INT_MAX;
199  int y_min = INT_MAX;
200  int x_max = INT_MIN;
201  int y_max = INT_MIN;
202  *page_number = 0;
203  std::stringstream stream(buffptr);
204  stream.imbue(std::locale::classic());
205  stream >> x_min;
206  stream >> y_min;
207  stream >> x_max;
208  stream >> y_max;
209  stream >> *page_number;
210  if (x_max < x_min || y_max < y_min) {
211  tprintf("Bad box coordinates in boxfile string! %s\n", ubuf);
212  return false;
213  }
214  // Test for long space-delimited string label.
215  if (strcmp(uch, kMultiBlobLabelCode) == 0 &&
216  (buffptr = strchr(buffptr, '#')) != nullptr) {
217  strncpy(uch, buffptr + 1, kBoxReadBufSize - 1);
218  uch[kBoxReadBufSize - 1] = '\0'; // Prevent buffer overrun.
219  chomp_string(uch);
220  uch_len = strlen(uch);
221  }
222  // Validate UTF8 by making unichars with it.
223  int used = 0;
224  while (used < uch_len) {
225  tesseract::UNICHAR ch(uch + used, uch_len - used);
226  int new_used = ch.utf8_len();
227  if (new_used == 0) {
228  tprintf("Bad UTF-8 str %s starts with 0x%02x at col %d\n",
229  uch + used, uch[used], used + 1);
230  return false;
231  }
232  used += new_used;
233  }
234  *utf8_str = uch;
235  if (x_min > x_max) Swap(&x_min, &x_max);
236  if (y_min > y_max) Swap(&y_min, &y_max);
237  bounding_box->set_to_given_coords(x_min, y_min, x_max, y_max);
238  return true; // Successfully read a box.
239 }
240 
241 // Creates a box file string from a unichar string, TBOX and page number.
242 void MakeBoxFileStr(const char* unichar_str, const TBOX& box, int page_num,
243  STRING* box_str) {
244  *box_str = unichar_str;
245  box_str->add_str_int(" ", box.left());
246  box_str->add_str_int(" ", box.bottom());
247  box_str->add_str_int(" ", box.right());
248  box_str->add_str_int(" ", box.top());
249  box_str->add_str_int(" ", page_num);
250 }
TBOX
Definition: rect.h:34
rect.h
chomp_string
void chomp_string(char *str)
Definition: helpers.h:77
fileerr.h
tesseract::LoadDataFromFile
bool LoadDataFromFile(const char *filename, GenericVector< char > *data)
Definition: genericvector.h:375
STRING::string
const char * string() const
Definition: strngs.cpp:194
TESSEXIT
@ TESSEXIT
Definition: errcode.h:44
STRING::add_str_int
void add_str_int(const char *str, int number)
Definition: strngs.cpp:377
TBOX::right
int16_t right() const
Definition: rect.h:79
GenericVector< TBOX >
strngs.h
CANTOPENFILE
constexpr ERRCODE CANTOPENFILE("Can't open file")
kBoxReadBufSize
const int kBoxReadBufSize
Definition: boxread.h:32
ERRCODE::error
void error(const char *caller, TessErrorLogCode action, const char *format,...) const
Definition: errcode.cpp:35
ParseBoxFileStr
bool ParseBoxFileStr(const char *boxfile_str, int *page_number, STRING *utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:174
genericvector.h
TBOX::left
int16_t left() const
Definition: rect.h:72
tprintf
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:35
tesseract::UNICHAR
Definition: unichar.h:57
unichar.h
TBOX::top
int16_t top() const
Definition: rect.h:58
helpers.h
Swap
void Swap(T *p1, T *p2)
Definition: helpers.h:95
TBOX::set_to_given_coords
void set_to_given_coords(int x_min, int y_min, int x_max, int y_max)
Definition: rect.h:271
tprintf.h
ReadMemBoxes
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:69
ReadNextBox
bool ReadNextBox(int *line_number, FILE *box_file, STRING *utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:127
GenericVector::size
int size() const
Definition: genericvector.h:72
tesseract::UNICHAR::utf8_len
int utf8_len() const
Definition: unichar.h:77
errcode.h
OpenBoxFile
FILE * OpenBoxFile(const STRING &fname)
Definition: boxread.cpp:36
STRING::truncate_at
void truncate_at(int32_t index)
Definition: strngs.cpp:265
STRING
Definition: strngs.h:45
MakeBoxFileStr
void MakeBoxFileStr(const char *unichar_str, const TBOX &box, int page_num, STRING *box_str)
Definition: boxread.cpp:242
ReadAllBoxes
bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING &filename, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:53
GenericVector::empty
bool empty() const
Definition: genericvector.h:91
BoxFileName
STRING BoxFileName(const STRING &image_filename)
Definition: boxread.cpp:106
TBOX::bottom
int16_t bottom() const
Definition: rect.h:65
boxread.h
STRING::split
void split(char c, GenericVector< STRING > *splited)
Definition: strngs.cpp:282
GenericVector::push_back
int push_back(T object)
Definition: genericvector.h:837