The Battle for Wesnoth  1.19.0-dev
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2024
3  by Pauli Nieminen <paniemin@cc.hut.fi>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 #define GETTEXT_DOMAIN "wesnoth-lib"
17 
18 #include "config_cache.hpp"
19 #include "filesystem.hpp"
20 #include "gettext.hpp"
21 #include "log.hpp"
22 #include "hash.hpp"
24 #include "serialization/parser.hpp"
26 #include "game_version.hpp"
27 
28 #include <boost/algorithm/string/replace.hpp>
29 #include <boost/iostreams/filter/gzip.hpp>
30 
31 static lg::log_domain log_cache("cache");
32 #define ERR_CACHE LOG_STREAM(err, log_cache)
33 #define LOG_CACHE LOG_STREAM(info, log_cache)
34 #define DBG_CACHE LOG_STREAM(debug, log_cache)
35 
36 namespace game_config
37 {
38 
39 namespace
40 {
41 
42 void add_builtin_defines(preproc_map& target)
43 {
44 #ifdef __APPLE__
45  target["APPLE"] = preproc_define();
46 #endif
47 
48 #if defined(MOUSE_TOUCH_EMULATION) || defined(TARGET_OS_IPHONE)
49  target["IPHONEOS"] = preproc_define();
50 #endif
51 
52  target["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
53 }
54 
55 }
56 
58 {
59  static config_cache cache;
60  return cache;
61 }
62 
64  : force_valid_cache_(false)
65  , use_cache_(true)
66  , fake_invalid_cache_(false)
67  , defines_map_()
68  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
69 {
70  // To set-up initial defines map correctly
71  clear_defines();
72 }
73 
75 {
76  return defines_map_;
77 }
78 
80 {
81  LOG_CACHE << "Clearing defines map!";
82 
83  defines_map_.clear();
84 
85  //
86  // Set-up default defines map.
87  //
88 
89  add_builtin_defines(defines_map_);
90 }
91 
92 void config_cache::get_config(const std::string& file_path, config& cfg, abstract_validator* validator)
93 {
94  load_configs(file_path, cfg, validator);
95 }
96 
97 void config_cache::write_file(std::string file_path, const config& cfg)
98 {
101  writer.write(cfg);
102 }
103 
104 void config_cache::write_file(std::string file_path, const preproc_map& defines)
105 {
106  if(defines.empty()) {
107  if(filesystem::file_exists(file_path)) {
108  filesystem::delete_directory(file_path);
109  }
110  return;
111  }
112 
115 
116  // Write all defines to stream.
117  for(const preproc_map::value_type& define : defines) {
118  define.second.write(writer, define.first);
119  }
120 }
121 
122 void config_cache::read_file(const std::string& file_path, config& cfg)
123 {
125  read_gz(cfg, *stream);
126 }
127 
129 {
131  // HACK: copy_map doesn't have built-in defines in some cases (issue #1924)
132  // or they may be out of date, so brute-force them in.
133  add_builtin_defines(res);
134  return res;
135 }
136 
138 {
140 }
141 
142 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map, abstract_validator* validator)
143 {
144  //read the file and then write to the cache
145  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
146  read(cfg, *stream, validator);
147 }
148 
149 void config_cache::read_cache(const std::string& file_path, config& cfg, abstract_validator* validator)
150 {
151  static const std::string extension = ".gz";
152 
153  std::stringstream defines_string;
154  defines_string << file_path;
155 
156  bool is_valid = true;
157 
158  for(const preproc_map::value_type& d : defines_map_) {
159  //
160  // Only WESNOTH_VERSION is allowed to be non-empty.
161  //
162  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
163  d.first != "WESNOTH_VERSION")
164  {
165  is_valid = false;
166  ERR_CACHE << "Invalid preprocessor define: " << d.first;
167  break;
168  }
169 
170  defines_string << " " << d.first;
171  }
172 
173  // Do cache check only if define map is valid and
174  // caching is allowed.
175  const std::string& cache_path = filesystem::get_cache_dir();
176 
177  if(is_valid && !cache_path.empty()) {
178  // Use a hash for a shorter display of the defines.
179  const std::string fname = cache_path + "/" +
181  utils::md5(defines_string.str()).hex_digest();
182  const std::string fname_checksum = fname + ".checksum" + extension;
183 
184  filesystem::file_tree_checksum dir_checksum;
185 
187  try {
188  if(filesystem::file_exists(fname_checksum)) {
189  config checksum_cfg;
190 
191  DBG_CACHE << "Reading checksum: " << fname_checksum;
192  read_file(fname_checksum, checksum_cfg);
193 
194  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
195  }
196  } catch(const config::error&) {
197  ERR_CACHE << "cache checksum is corrupt";
198  } catch(const filesystem::io_exception&) {
199  ERR_CACHE << "error reading cache checksum";
200  } catch(const std::ios_base::failure&) {
201  ERR_CACHE << "error reading cache checksum";
202  }
203  }
204 
205  if(force_valid_cache_) {
206  LOG_CACHE << "skipping cache validation (forced)";
207  }
208 
209  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
210  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str();
211  log_scope("read cache");
212 
213  try {
214  read_file(fname + extension,cfg);
215  const std::string define_file = fname + ".define" + extension;
216 
217  if(filesystem::file_exists(define_file)) {
219  }
220 
221  return;
222  } catch(const config::error& e) {
223  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message;
224  } catch(const filesystem::io_exception&) {
225  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files";
226  } catch (const boost::iostreams::gzip_error& e) {
227  //read_file -> ... -> read_gz can throw this exception.
228  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error();
229  }
230  }
231 
232  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'";
233 
234  // Now we need queued defines so read them to memory
236 
237  preproc_map copy_map(make_copy_map());
238 
239  read_configs(file_path, cfg, copy_map, validator);
240  add_defines_map_diff(copy_map);
241 
242  try {
243  write_file(fname + extension, cfg);
244  write_file(fname + ".define" + extension, copy_map);
245 
246  config checksum_cfg;
247 
248  filesystem::data_tree_checksum().write(checksum_cfg);
249  write_file(fname_checksum, checksum_cfg);
250  } catch(const filesystem::io_exception&) {
251  ERR_CACHE << "could not write to cache '" << fname << "'";
252  }
253 
254  return;
255  }
256 
257  LOG_CACHE << "Loading plain config instead of cache";
258 
259  preproc_map copy_map(make_copy_map());
260  read_configs(file_path, cfg, copy_map, validator);
261  add_defines_map_diff(copy_map);
262 }
263 
264 void config_cache::read_defines_file(const std::string& file_path)
265 {
266  config cfg;
267  read_file(file_path, cfg);
268 
269  DBG_CACHE << "Reading cached defines from: " << file_path;
270 
271  // use static preproc_define::read_pair(config) to make a object
272  // and pass that object config_cache_transaction::insert_to_active method
273  for(const config::any_child value : cfg.all_children_range()) {
275  preproc_define::read_pair(value.cfg));
276  }
277 }
278 
280 {
281  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
282 
283  for(const std::string &p : files) {
285  }
286 }
287 
288 void config_cache::load_configs(const std::string& config_path, config& cfg, abstract_validator* validator)
289 {
290  // Make sure that we have fake transaction if no real one is going on
292 
293  if (use_cache_) {
294  read_cache(config_path, cfg, validator);
295  } else {
296  preproc_map copy_map(make_copy_map());
297  read_configs(config_path, cfg, copy_map, validator);
298  add_defines_map_diff(copy_map);
299  }
300 }
301 
303 {
304  fake_invalid_cache_ = force;
305 }
306 
308 {
309  use_cache_ = use;
310 }
311 
313 {
314  force_valid_cache_ = force;
315 }
316 
318 {
320 }
321 
322 void config_cache::add_define(const std::string& define)
323 {
324  DBG_CACHE << "adding define: " << define;
325  defines_map_[define] = preproc_define();
326 
328  // we have to add this to active map too
330  }
331 
332 }
333 
334 void config_cache::remove_define(const std::string& define)
335 {
336  DBG_CACHE << "removing define: " << define;
337  defines_map_.erase(define);
338 
340  // we have to remove this from active map too
342  }
343 }
344 
346 {
347  std::vector<std::string> files, dirs;
349 
350  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
351  << dirs.size() << " dirs to check";
352 
353  const std::string& exclude_current = cache_file_prefix_ + "*";
354 
355  bool status = true;
356 
357  status &= delete_cache_files(files, exclude_current);
358  status &= delete_cache_files(dirs, exclude_current);
359 
360  LOG_CACHE << "clean_cache(): done";
361 
362  return status;
363 }
364 
366 {
367  std::vector<std::string> files, dirs;
369 
370  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
371  << dirs.size() << " dirs";
372 
373  bool status = true;
374 
375  status &= delete_cache_files(files);
376  status &= delete_cache_files(dirs);
377 
378  LOG_CACHE << "purge_cache(): done";
379  return status;
380 }
381 
382 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
383  const std::string& exclude_pattern)
384 {
385  const bool delete_everything = exclude_pattern.empty();
386  bool status = true;
387 
388  for(const std::string& file_path : paths)
389  {
390  if(!delete_everything) {
391  const std::string& fn = filesystem::base_name(file_path);
392 
393  if(utils::wildcard_string_match(fn, exclude_pattern)) {
394  LOG_CACHE << "delete_cache_files(): skipping " << file_path
395  << " excluded by '" << exclude_pattern << "'";
396  continue;
397  }
398  }
399 
400  LOG_CACHE << "delete_cache_files(): deleting " << file_path;
401  if(!filesystem::delete_directory(file_path)) {
402  ERR_CACHE << "delete_cache_files(): could not delete "
403  << file_path;
404  status = false;
405  }
406  }
407 
408  return status;
409 }
410 
413 
415  : define_filenames_()
416  , active_map_()
417 {
418  assert(state_ == FREE);
419  state_ = NEW;
420  active_ = this;
421 }
422 
424 {
425  state_ = FREE;
426  active_ = 0;
427 }
428 
430 {
431  state_ = LOCKED;
432 }
433 
434 const std::vector<std::string>& config_cache_transaction::get_define_files() const
435 {
436  return define_filenames_;
437 }
438 
439 void config_cache_transaction::add_define_file(const std::string& file)
440 {
441  define_filenames_.push_back(file);
442 }
443 
445 {
446  if(active_map_.empty()) {
447  active_map_.insert(defines_map.begin(), defines_map.end());
448  if(get_state() == NEW) {
449  state_ = ACTIVE;
450  }
451  }
452 
453  return active_map_;
454 }
455 
456 namespace
457 {
458 
459 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
460 {
461  if(a.first < b.first) {
462  return true;
463  }
464 
465  if(b.first < a.first) {
466  return false;
467  }
468 
469  if(a.second < b.second) {
470  return true;
471  }
472 
473  return false;
474 }
475 
476 } // end anonymous namespace
477 
479 {
480  if(get_state() == ACTIVE) {
481  preproc_map temp;
482  std::set_difference(new_map.begin(),
483  new_map.end(),
484  active_map_.begin(),
485  active_map_.end(),
486  std::insert_iterator<preproc_map>(temp,temp.begin()),
487  &compare_define);
488 
489  for(const preproc_map::value_type &def : temp) {
490  insert_to_active(def);
491  }
492 
493  temp.swap(new_map);
494  } else if (get_state() == LOCKED) {
495  new_map.clear();
496  }
497 }
498 
499 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
500 {
501  active_map_[def.first] = def.second;
502 }
503 
504 }
Used in parsing config file.
Definition: validator.hpp:38
Class for writing a config out to a file in pieces.
void write(const config &cfg)
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:159
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:887
Used to share macros between cache objects You have to create transaction object to load all macros t...
void add_defines_map_diff(preproc_map &defines_map)
std::vector< std::string > define_filenames_
static config_cache_transaction & instance()
const std::vector< std::string > & get_define_files() const
void lock()
Lock the transaction so no more macros are added.
preproc_map & get_active_map(const preproc_map &defines_map)
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
static config_cache_transaction * active_
void add_define_file(const std::string &file)
Singleton class to manage game config file caching.
void load_configs(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
void add_define(const std::string &define)
Add a entry to preproc defines map.
void recheck_filetree_checksum()
Force cache checksum validation.
void read_cache(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
bool purge_cache()
Deletes all cache files.
void set_use_cache(bool use)
Enable/disable caching.
static config_cache & instance()
Get reference to the singleton object.
preproc_map & make_copy_map()
const preproc_map & get_preproc_map() const
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
void clear_defines()
Clear stored defines map to default values.
void write_file(std::string file, const config &cfg)
void read_configs(const std::string &path, config &cfg, preproc_map &defines, abstract_validator *validator=nullptr)
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_file(const std::string &file, config &cfg)
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read_defines_file(const std::string &path)
void add_defines_map_diff(preproc_map &)
void set_force_valid_cache(bool force)
Enable/disable cache validation.
void get_config(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
Gets a config object from given path.
Holds a fake cache transaction if no real one is used.
virtual std::string hex_digest() const override
Definition: hash.cpp:120
#define LOG_CACHE
static lg::log_domain log_cache("cache")
#define DBG_CACHE
#define ERR_CACHE
Declarations for File-IO.
Interfaces for manipulating version numbers of engine, add-ons, etc.
Standard logging facilities (interface).
#define log_scope(description)
Definition: log.hpp:274
std::string get_cache_dir()
Definition: filesystem.cpp:880
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Get a list of all files and/or directories in a given directory.
Definition: filesystem.cpp:404
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
static bool file_exists(const bfs::path &fpath)
Definition: filesystem.cpp:318
bool delete_directory(const std::string &dirname, const bool keep_pbl)
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:50
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:51
Game configuration data as global variables.
Definition: build_info.cpp:60
const version_info wesnoth_version(VERSION)
int cache_compression_level
Definition: filesystem.cpp:95
const std::string revision
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using '*' as any number of characters (including none), '+' as one or more characters,...
fake
For describing the type of faked display, if any.
Definition: video.hpp:44
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
std::map< std::string, struct preproc_define > preproc_map
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:627
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:683
An exception object used when an IO error occurs.
Definition: filesystem.hpp:64
static preproc_map::value_type read_pair(const config &)
mock_party p
#define d
#define e
#define a
#define b