The Battle for Wesnoth  1.15.1+dev
tstring.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2004 by Philippe Plantier <ayin@anathas.org>
3  Copyright (C) 2005 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 /**
17  * @file
18  * Routines for translatable strings.
19  */
20 
21 #include "tstring.hpp"
22 
23 #include "gettext.hpp"
24 #include "log.hpp"
25 
26 #include <boost/multi_index/hashed_index.hpp>
27 
28 #include <map>
29 #include <mutex>
30 #include <vector>
31 
32 static lg::log_domain log_config("config");
33 #define LOG_CF LOG_STREAM(info, log_config)
34 #define ERR_CF LOG_STREAM(err, log_config)
35 
36 static unsigned language_counter = 0;
37 
38 namespace
39 {
40 const char TRANSLATABLE_PART = 0x01;
41 const char UNTRANSLATABLE_PART = 0x02;
42 const char TEXTDOMAIN_SEPARATOR = 0x03;
43 const char ID_TRANSLATABLE_PART = 0x04;
44 const char PLURAL_PART = 0x05;
45 
46 std::vector<std::string> id_to_textdomain;
47 std::map<std::string, unsigned int> textdomain_to_id;
48 }
49 
50 std::size_t t_string_base::hash_value() const
51 {
52  std::size_t seed = 0;
53  boost::hash_combine(seed, value_);
54  boost::hash_combine(seed, translatable_);
55  boost::hash_combine(seed, last_untranslatable_);
56  return seed;
57 }
58 
60  : string_(string.value_)
61  , begin_(0)
62  , end_(string_.size())
63  , textdomain_()
64  , translatable_(false)
65  , countable_(false)
66  , count_(0)
67 {
68  if(string.translatable_) {
69  update();
70  }
71 }
72 
73 static std::string mark = std::string(TRANSLATABLE_PART, 1) + UNTRANSLATABLE_PART + ID_TRANSLATABLE_PART + PLURAL_PART;
74 
76 {
77  unsigned int id;
78 
79  if(begin_ == string_.size()) {
80  return;
81  }
82 
83  switch(string_[begin_]) {
84  case TRANSLATABLE_PART: {
85  // Format: [TRANSLATABLE_PART]textdomain[TEXTDOMAIN_SEPARATOR]msgid[...]
86  std::string::size_type textdomain_end = string_.find(TEXTDOMAIN_SEPARATOR, begin_ + 1);
87 
88  if(textdomain_end == std::string::npos || textdomain_end >= string_.size() - 1) {
89  ERR_CF << "Error: invalid string: " << string_ << std::endl;
90  begin_ = string_.size();
91  return;
92  }
93 
94  end_ = string_.find_first_of(mark, textdomain_end + 1);
95  if(end_ == std::string::npos) {
96  end_ = string_.size();
97  }
98 
99  textdomain_ = std::string(string_, begin_ + 1, textdomain_end - begin_ - 1);
100  translatable_ = true;
101  begin_ = textdomain_end + 1;
102 
103  break;
104  }
105  case ID_TRANSLATABLE_PART:
106  // Format: [ID_TRANSLATABLE_PART][2-byte textdomain ID]msgid[...]
107  if(begin_ + 3 >= string_.size()) {
108  ERR_CF << "Error: invalid string: " << string_ << std::endl;
109  begin_ = string_.size();
110  return;
111  }
112 
113  end_ = string_.find_first_of(mark, begin_ + 3);
114  if(end_ == std::string::npos) {
115  end_ = string_.size();
116  }
117 
118  id = static_cast<unsigned char>(string_[begin_ + 1]) + static_cast<unsigned char>(string_[begin_ + 2]) * 256;
119  if(id >= id_to_textdomain.size()) {
120  ERR_CF << "Error: invalid string: " << string_ << std::endl;
121  begin_ = string_.size();
122  return;
123  }
124 
125  textdomain_ = id_to_textdomain[id];
126  begin_ += 3;
127  translatable_ = true;
128 
129  break;
130 
131  case UNTRANSLATABLE_PART:
132  end_ = string_.find_first_of(mark, begin_ + 1);
133  if(end_ == std::string::npos) {
134  end_ = string_.size();
135  }
136 
137  if(end_ <= begin_ + 1) {
138  ERR_CF << "Error: invalid string: " << string_ << std::endl;
139  begin_ = string_.size();
140  return;
141  }
142 
143  translatable_ = false;
144  textdomain_ = "";
145  begin_ += 1;
146  break;
147 
148  case PLURAL_PART:
149  begin_ = string_.find_first_of(mark, end_ + 5);
150  if(begin_ == std::string::npos) {
151  begin_ = string_.size();
152  }
153 
154  if(string_[begin_] == PLURAL_PART) {
155  ERR_CF << "Error: invalid string: " << string_ << std::endl;
156  begin_ = string_.size();
157  return;
158  }
159 
160  update();
161  break;
162 
163  default:
164  end_ = string_.size();
165  translatable_ = false;
166  textdomain_ = "";
167  break;
168  }
169 
170  if(translatable_ && string_[end_] == PLURAL_PART) {
171  // Format: [PLURAL_PART][4-byte count]msgid_plural[...]
172  if(end_ + 5 >= string_.size()) {
173  ERR_CF << "Error: invalid string: " << string_ << std::endl;
174  begin_ = string_.size();
175  return;
176  }
177 
178  std::string::size_type real_end = string_.find_first_of(mark, end_ + 6);
179  if(real_end < string_.size() && string_[real_end] == PLURAL_PART) {
180  ERR_CF << "Error: invalid string: " << string_ << std::endl;
181  begin_ = string_.size();
182  return;
183  }
184 
185  countable_ = true;
186 
187  union {
188  int32_t count;
189  char data[4];
190  } cvt;
191 
192  std::copy_n(string_.data() + end_ + 1, 4, cvt.data);
193  count_ = cvt.count;
194  } else {
195  countable_ = false;
196  count_ = 0;
197  }
198 }
199 
200 std::string::const_iterator t_string_base::walker::plural_begin() const
201 {
202  if(!countable_) {
203  return begin();
204  }
205 
206  return end() + 5;
207 }
208 
209 std::string::const_iterator t_string_base::walker::plural_end() const
210 {
211  if(!countable_) {
212  return end();
213  }
214 
215  std::string::size_type pl_end = string_.find_first_of(mark, end_ + 5);
216  if(pl_end == std::string::npos) {
217  pl_end = string_.size();
218  }
219 
220  return string_.begin() + pl_end;
221 }
222 
224  : value_()
227  , translatable_(false)
228  , last_untranslatable_(false)
229 {
230 }
231 
233 {
234 }
235 
237  : value_(string.value_)
240  , translatable_(string.translatable_)
242 {
243 }
244 
245 t_string_base::t_string_base(const std::string& string)
246  : value_(string)
249  , translatable_(false)
250  , last_untranslatable_(false)
251 {
252 }
253 
254 t_string_base::t_string_base(const std::string& string, const std::string& textdomain)
255  : value_(1, ID_TRANSLATABLE_PART)
258  , translatable_(true)
259  , last_untranslatable_(false)
260 {
261  if(string.empty()) {
262  value_.clear();
263  translatable_ = false;
264  return;
265  }
266 
267  std::map<std::string, unsigned int>::const_iterator idi = textdomain_to_id.find(textdomain);
268  unsigned int id;
269 
270  if(idi == textdomain_to_id.end()) {
271  id = id_to_textdomain.size();
272  textdomain_to_id[textdomain] = id;
273  id_to_textdomain.push_back(textdomain);
274  } else {
275  id = idi->second;
276  }
277 
278  value_ += static_cast<char>(id & 0xff);
279  value_ += static_cast<char>(id >> 8);
280  value_ += string;
281 }
282 
283 t_string_base::t_string_base(const std::string& sing, const std::string& pl, int count, const std::string& textdomain)
284  : value_(1, ID_TRANSLATABLE_PART)
287  , translatable_(true)
288  , last_untranslatable_(false)
289 {
290  if(sing.empty() && pl.empty()) {
291  value_.clear();
292  translatable_ = false;
293  return;
294  }
295 
296  std::map<std::string, unsigned int>::const_iterator idi = textdomain_to_id.find(textdomain);
297  unsigned int id;
298 
299  if(idi == textdomain_to_id.end()) {
300  id = id_to_textdomain.size();
301  textdomain_to_id[textdomain] = id;
302  id_to_textdomain.push_back(textdomain);
303  } else {
304  id = idi->second;
305  }
306 
307  value_ += static_cast<char>(id & 0xff);
308  value_ += static_cast<char>(id >> 8);
309  value_ += sing;
310  value_ += PLURAL_PART;
311 
312  union {
313  int32_t count;
314  char data[4];
315  } cvt;
316 
317  cvt.count = count;
318  for(char c : cvt.data) {
319  value_ += c;
320  }
321 
322  value_ += pl;
323 }
324 
325 t_string_base::t_string_base(const char* string)
326  : value_(string)
329  , translatable_(false)
330  , last_untranslatable_(false)
331 {
332 }
333 
335 {
336  t_string_base orig(string);
337 
338  if(!string.empty() && (string[0] == TRANSLATABLE_PART || string[0] == UNTRANSLATABLE_PART)) {
339  orig.translatable_ = true;
340  } else {
341  orig.translatable_ = false;
342  }
343 
344  t_string_base res;
345 
346  for(walker w(orig); !w.eos(); w.next()) {
347  std::string substr(w.begin(), w.end());
348 
349  if(w.translatable()) {
350  res += t_string_base(substr, w.textdomain());
351  } else {
352  res += substr;
353  }
354  }
355 
356  return res;
357 }
358 
359 std::string t_string_base::base_str() const
360 {
361  std::string res;
362  for(walker w(*this); !w.eos(); w.next()) {
363  res += std::string(w.begin(), w.end());
364  }
365 
366  return res;
367 }
368 
369 std::string t_string_base::to_serialized() const
370 {
371  t_string_base res;
372 
373  for(walker w(*this); !w.eos(); w.next()) {
374  t_string_base chunk;
375 
376  std::string substr(w.begin(), w.end());
377  if(w.translatable()) {
378  chunk.translatable_ = true;
379  chunk.last_untranslatable_ = false;
380  chunk.value_ = TRANSLATABLE_PART + w.textdomain() + TEXTDOMAIN_SEPARATOR + substr;
381  } else {
382  chunk.translatable_ = false;
383  chunk.value_ = substr;
384  }
385 
386  res += chunk;
387  }
388 
389  return res.value();
390 }
391 
393 {
394  value_ = string.value_;
395  translated_value_ = string.translated_value_;
396  translation_timestamp_ = string.translation_timestamp_;
397  translatable_ = string.translatable_;
398  last_untranslatable_ = string.last_untranslatable_;
399 
400  return *this;
401 }
402 
403 t_string_base& t_string_base::operator=(const std::string& string)
404 {
405  value_ = string;
406  translated_value_ = "";
408  translatable_ = false;
409  last_untranslatable_ = false;
410 
411  return *this;
412 }
413 
415 {
416  value_ = string;
417  translated_value_ = "";
419  translatable_ = false;
420  last_untranslatable_ = false;
421 
422  return *this;
423 }
424 
426 {
427  t_string_base res(*this);
428  res += string;
429  return res;
430 }
431 
432 t_string_base t_string_base::operator+(const std::string& string) const
433 {
434  t_string_base res(*this);
435  res += string;
436  return res;
437 }
438 
439 t_string_base t_string_base::operator+(const char* string) const
440 {
441  t_string_base res(*this);
442  res += string;
443  return res;
444 }
445 
447 {
448  if(string.value_.empty()) {
449  return *this;
450  }
451 
452  if(value_.empty()) {
453  *this = string;
454  return *this;
455  }
456 
457  if(translatable_ || string.translatable_) {
458  if(!translatable_) {
459  value_ = UNTRANSLATABLE_PART + value_;
460  translatable_ = true;
461  last_untranslatable_ = true;
462  } else {
463  translated_value_ = "";
464  }
465 
466  if(string.translatable_) {
467  if(last_untranslatable_ && string.value_[0] == UNTRANSLATABLE_PART) {
468  value_.append(string.value_.begin() + 1, string.value_.end());
469  } else {
470  value_ += string.value_;
471  }
472 
473  last_untranslatable_ = string.last_untranslatable_;
474  } else {
475  if(!last_untranslatable_) {
476  value_ += UNTRANSLATABLE_PART;
477  last_untranslatable_ = true;
478  }
479 
480  value_ += string.value_;
481  }
482  } else {
483  value_ += string.value_;
484  }
485 
486  return *this;
487 }
488 
489 t_string_base& t_string_base::operator+=(const std::string& string)
490 {
491  if(string.empty()) {
492  return *this;
493  }
494 
495  if(value_.empty()) {
496  *this = string;
497  return *this;
498  }
499 
500  if(translatable_) {
501  if(!last_untranslatable_) {
502  value_ += UNTRANSLATABLE_PART;
503  last_untranslatable_ = true;
504  }
505 
506  value_ += string;
507  translated_value_ = "";
508  } else {
509  value_ += string;
510  }
511 
512  return *this;
513 }
514 
516 {
517  if(string[0] == 0) {
518  return *this;
519  }
520 
521  if(value_.empty()) {
522  *this = string;
523  return *this;
524  }
525 
526  if(translatable_) {
527  if(!last_untranslatable_) {
528  value_ += UNTRANSLATABLE_PART;
529  last_untranslatable_ = true;
530  }
531 
532  value_ += string;
533  translated_value_ = "";
534  } else {
535  value_ += string;
536  }
537 
538  return *this;
539 }
540 
542 {
543  return that.translatable_ == translatable_ && that.value_ == value_;
544 }
545 
546 bool t_string_base::operator==(const std::string& that) const
547 {
548  return !translatable_ && value_ == that;
549 }
550 
551 bool t_string_base::operator==(const char* that) const
552 {
553  return !translatable_ && value_ == that;
554 }
555 
557 {
558  return value_ < that.value_;
559 }
560 
561 const std::string& t_string_base::str() const
562 {
563  if(!translatable_) {
564  return value_;
565  }
566 
568  return translated_value_;
569  }
570 
571  translated_value_.clear();
572 
573  for(walker w(*this); !w.eos(); w.next()) {
574  std::string part(w.begin(), w.end());
575 
576  if(w.translatable()) {
577  if(w.countable()) {
578  std::string plural(w.plural_begin(), w.plural_end());
580  translation::dsngettext(w.textdomain().c_str(), part.c_str(), plural.c_str(), w.count());
581  } else {
583  translation::dsgettext(w.textdomain().c_str(), part.c_str());
584  }
585  } else {
586  translated_value_ += part;
587  }
588  }
589 
591  return translated_value_;
592 }
593 
595  : val_(new base())
596 {
597 }
598 
600 {
601 }
602 
604  : val_(o.val_)
605 {
606 }
607 
609  : val_(new base(o))
610 {
611 }
612 
613 t_string::t_string(const char* o)
614  : val_(new base(o))
615 {
616 }
617 
618 t_string::t_string(const std::string& o)
619  : val_(new base(o))
620 {
621 }
622 
623 t_string::t_string(const std::string& o, const std::string& textdomain)
624  : val_(new base(o, textdomain))
625 {
626 }
627 
628 t_string::t_string(const std::string& s, const std::string& pl, int c, const std::string& textdomain)
629  : val_(new base(s, pl, c, textdomain))
630 {
631 }
632 
634 {
635  val_ = o.val_;
636  return *this;
637 }
638 
640 {
641  t_string o2(o);
642  swap(o2);
643  return *this;
644 }
645 
646 void t_string::add_textdomain(const std::string& name, const std::string& path)
647 {
648  LOG_CF << "Binding textdomain " << name << " to path " << path << "\n";
649 
650  // Register and (re-)bind this textdomain
651  translation::bind_textdomain(name.c_str(), path.c_str(), "UTF-8");
652 }
653 
655 {
657 }
658 
659 void swap(t_string& lhs, t_string& rhs)
660 {
661  lhs.swap(rhs);
662 }
663 
664 std::ostream& operator<<(std::ostream& stream, const t_string_base& string)
665 {
666  stream << string.str();
667  return stream;
668 }
static unsigned language_counter
Definition: tstring.cpp:36
std::string::const_iterator begin() const
Definition: tstring.hpp:39
std::string base_str() const
Definition: tstring.cpp:359
static void reset_translations()
Definition: tstring.cpp:654
t_string()
Default implementation, but defined out-of-line for efficiency reasons.
Definition: tstring.cpp:594
std::string::const_iterator plural_begin() const
Definition: tstring.cpp:200
void bind_textdomain(const char *domain, const char *directory, const char *)
Definition: gettext.cpp:432
~t_string_base()
Default implementation, but defined out-of-line for efficiency reasons.
Definition: tstring.cpp:232
std::string to_serialized() const
Definition: tstring.cpp:369
#define val_(o)
Definition: lobject.h:123
void swap(t_string &other)
Definition: tstring.hpp:196
std::string dsngettext(const char *domainname, const char *singular, const char *plural, int n)
Definition: gettext.cpp:417
std::string value_
Definition: tstring.hpp:114
std::string::size_type end_
Definition: tstring.hpp:49
const std::string & string_
Definition: tstring.hpp:47
bool operator==(const t_string_base &) const
Definition: tstring.cpp:541
const std::string & value() const
Definition: tstring.hpp:108
bool last_untranslatable_
Definition: tstring.hpp:117
std::string::size_type size() const
Definition: tstring.hpp:99
bool operator<(const t_string_base &string) const
Definition: tstring.cpp:556
const std::string & str() const
Definition: tstring.cpp:561
std::string textdomain_
Definition: tstring.hpp:50
std::string dsgettext(const char *domainname, const char *msgid)
Definition: gettext.cpp:404
t_string_base & operator=(const t_string_base &)
Default implementation, but defined out-of-line for efficiency reasons.
Definition: tstring.cpp:392
unsigned translation_timestamp_
Definition: tstring.hpp:116
std::string translated_value_
Definition: tstring.hpp:115
std::string::size_type begin_
Definition: tstring.hpp:48
static const char * name(const std::vector< SDL_Joystick *> &joysticks, const std::size_t index)
Definition: joystick.cpp:48
static void add_textdomain(const std::string &name, const std::string &path)
Definition: tstring.cpp:646
std::string path
Definition: game_config.cpp:39
std::size_t hash_value() const
Definition: tstring.cpp:50
walker(const t_string_base &string)
Definition: tstring.cpp:59
t_string & operator=(const t_string &)
Default implementation, but defined out-of-line for efficiency reasons.
Definition: tstring.cpp:633
std::string::const_iterator plural_end() const
Definition: tstring.cpp:209
int count() const
Definition: tstring.hpp:37
static lg::log_domain log_config("config")
static std::string mark
Definition: tstring.cpp:73
const route_iterator begin_
Definition: move.cpp:295
~t_string()
Default implementation, but defined out-of-line for efficiency reasons.
Definition: tstring.cpp:599
static map_location::DIRECTION s
int w
bool empty() const
Definition: tstring.hpp:98
std::string::const_iterator end() const
Definition: tstring.hpp:40
std::shared_ptr< const t_string_base > val_
Definition: tstring.hpp:200
#define ERR_CF
Definition: tstring.cpp:34
Standard logging facilities (interface).
t_string_base & operator+=(const t_string_base &)
Definition: tstring.cpp:446
#define LOG_CF
Definition: tstring.cpp:33
t_string_base operator+(const t_string_base &) const
Definition: tstring.cpp:425
mock_char c
bool translatable_
Definition: tstring.hpp:117
std::ostream & operator<<(std::ostream &stream, const t_string_base &string)
Definition: tstring.cpp:664
static t_string_base from_serialized(const std::string &string)
Definition: tstring.cpp:334