The Battle for Wesnoth  1.17.0-dev
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2021
3  by Pauli Nieminen <paniemin@cc.hut.fi>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 #define GETTEXT_DOMAIN "wesnoth-lib"
17 
18 #include "config_cache.hpp"
19 #include "filesystem.hpp"
20 #include "gettext.hpp"
21 #include "game_config.hpp"
22 #include "log.hpp"
23 #include "hash.hpp"
25 #include "serialization/parser.hpp"
27 #include "game_version.hpp"
28 
29 #include <boost/algorithm/string/replace.hpp>
30 #include <boost/iostreams/filter/gzip.hpp>
31 
32 static lg::log_domain log_cache("cache");
33 #define ERR_CACHE LOG_STREAM(err, log_cache)
34 #define LOG_CACHE LOG_STREAM(info, log_cache)
35 #define DBG_CACHE LOG_STREAM(debug, log_cache)
36 
37 namespace game_config
38 {
39 
40 namespace
41 {
42 
43 void add_builtin_defines(preproc_map& target)
44 {
45 #ifdef __APPLE__
46  target["APPLE"] = preproc_define();
47 #endif
48 
49 #if defined(MOUSE_TOUCH_EMULATION) || defined(TARGET_OS_IPHONE)
50  target["IPHONEOS"] = preproc_define();
51 #endif
52 
53  target["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
54 }
55 
56 }
57 
59 {
60  static config_cache cache;
61  return cache;
62 }
63 
65  : force_valid_cache_(false)
66  , use_cache_(true)
67  , fake_invalid_cache_(false)
68  , defines_map_()
69  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
70 {
71  // To set-up initial defines map correctly
72  clear_defines();
73 }
74 
76 {
77  return defines_map_;
78 }
79 
81 {
82  LOG_CACHE << "Clearing defines map!" << std::endl;
83 
84  defines_map_.clear();
85 
86  //
87  // Set-up default defines map.
88  //
89 
90  add_builtin_defines(defines_map_);
91 }
92 
93 void config_cache::get_config(const std::string& file_path, config& cfg, abstract_validator* validator)
94 {
95  load_configs(file_path, cfg, validator);
96 }
97 
98 void config_cache::write_file(std::string file_path, const config& cfg)
99 {
102  writer.write(cfg);
103 }
104 
105 void config_cache::write_file(std::string file_path, const preproc_map& defines)
106 {
107  if(defines.empty()) {
108  if(filesystem::file_exists(file_path)) {
109  filesystem::delete_directory(file_path);
110  }
111  return;
112  }
113 
116 
117  // Write all defines to stream.
118  for(const preproc_map::value_type& define : defines) {
119  define.second.write(writer, define.first);
120  }
121 }
122 
123 void config_cache::read_file(const std::string& file_path, config& cfg)
124 {
126  read_gz(cfg, *stream);
127 }
128 
130 {
132  // HACK: copy_map doesn't have built-in defines in some cases (issue #1924)
133  // or they may be out of date, so brute-force them in.
134  add_builtin_defines(res);
135  return res;
136 }
137 
139 {
141 }
142 
143 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map, abstract_validator* validator)
144 {
145  //read the file and then write to the cache
146  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
147  read(cfg, *stream, validator);
148 }
149 
150 void config_cache::read_cache(const std::string& file_path, config& cfg, abstract_validator* validator)
151 {
152  static const std::string extension = ".gz";
153 
154  std::stringstream defines_string;
155  defines_string << file_path;
156 
157  bool is_valid = true;
158 
159  for(const preproc_map::value_type& d : defines_map_) {
160  //
161  // Only WESNOTH_VERSION is allowed to be non-empty.
162  //
163  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
164  d.first != "WESNOTH_VERSION")
165  {
166  is_valid = false;
167  ERR_CACHE << "Invalid preprocessor define: " << d.first << '\n';
168  break;
169  }
170 
171  defines_string << " " << d.first;
172  }
173 
174  // Do cache check only if define map is valid and
175  // caching is allowed.
176  const std::string& cache_path = filesystem::get_cache_dir();
177 
178  if(is_valid && !cache_path.empty()) {
179  // Use a hash for a shorter display of the defines.
180  const std::string fname = cache_path + "/" +
182  utils::sha1(defines_string.str()).hex_digest();
183  const std::string fname_checksum = fname + ".checksum" + extension;
184 
185  filesystem::file_tree_checksum dir_checksum;
186 
188  try {
189  if(filesystem::file_exists(fname_checksum)) {
190  config checksum_cfg;
191 
192  DBG_CACHE << "Reading checksum: " << fname_checksum << "\n";
193  read_file(fname_checksum, checksum_cfg);
194 
195  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
196  }
197  } catch(const config::error&) {
198  ERR_CACHE << "cache checksum is corrupt" << std::endl;
199  } catch(const filesystem::io_exception&) {
200  ERR_CACHE << "error reading cache checksum" << std::endl;
201  } catch(const std::ios_base::failure&) {
202  ERR_CACHE << "error reading cache checksum" << std::endl;
203  }
204  }
205 
206  if(force_valid_cache_) {
207  LOG_CACHE << "skipping cache validation (forced)\n";
208  }
209 
210  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
211  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str() << "\n";
212  log_scope("read cache");
213 
214  try {
215  read_file(fname + extension,cfg);
216  const std::string define_file = fname + ".define" + extension;
217 
218  if(filesystem::file_exists(define_file)) {
220  }
221 
222  return;
223  } catch(const config::error& e) {
224  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message << std::endl;
225  } catch(const filesystem::io_exception&) {
226  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files" << std::endl;
227  } catch (const boost::iostreams::gzip_error& e) {
228  //read_file -> ... -> read_gz can throw this exception.
229  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error() << std::endl;
230  }
231  }
232 
233  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'\n";
234 
235  // Now we need queued defines so read them to memory
237 
238  preproc_map copy_map(make_copy_map());
239 
240  read_configs(file_path, cfg, copy_map, validator);
241  add_defines_map_diff(copy_map);
242 
243  try {
244  write_file(fname + extension, cfg);
245  write_file(fname + ".define" + extension, copy_map);
246 
247  config checksum_cfg;
248 
249  filesystem::data_tree_checksum().write(checksum_cfg);
250  write_file(fname_checksum, checksum_cfg);
251  } catch(const filesystem::io_exception&) {
252  ERR_CACHE << "could not write to cache '" << fname << "'" << std::endl;
253  }
254 
255  return;
256  }
257 
258  LOG_CACHE << "Loading plain config instead of cache\n";
259 
260  preproc_map copy_map(make_copy_map());
261  read_configs(file_path, cfg, copy_map, validator);
262  add_defines_map_diff(copy_map);
263 }
264 
265 void config_cache::read_defines_file(const std::string& file_path)
266 {
267  config cfg;
268  read_file(file_path, cfg);
269 
270  DBG_CACHE << "Reading cached defines from: " << file_path << "\n";
271 
272  // use static preproc_define::read_pair(config) to make a object
273  // and pass that object config_cache_transaction::insert_to_active method
274  for(const config::any_child value : cfg.all_children_range()) {
276  preproc_define::read_pair(value.cfg));
277  }
278 }
279 
281 {
282  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
283 
284  for(const std::string &p : files) {
286  }
287 }
288 
289 void config_cache::load_configs(const std::string& config_path, config& cfg, abstract_validator* validator)
290 {
291  // Make sure that we have fake transaction if no real one is going on
292  fake_transaction fake;
293 
294  if (use_cache_) {
295  read_cache(config_path, cfg, validator);
296  } else {
297  preproc_map copy_map(make_copy_map());
298  read_configs(config_path, cfg, copy_map, validator);
299  add_defines_map_diff(copy_map);
300  }
301 }
302 
304 {
305  fake_invalid_cache_ = force;
306 }
307 
309 {
310  use_cache_ = use;
311 }
312 
314 {
315  force_valid_cache_ = force;
316 }
317 
319 {
321 }
322 
323 void config_cache::add_define(const std::string& define)
324 {
325  DBG_CACHE << "adding define: " << define << "\n";
326  defines_map_[define] = preproc_define();
327 
329  // we have to add this to active map too
331  }
332 
333 }
334 
335 void config_cache::remove_define(const std::string& define)
336 {
337  DBG_CACHE << "removing define: " << define << "\n";
338  defines_map_.erase(define);
339 
341  // we have to remove this from active map too
343  }
344 }
345 
347 {
348  std::vector<std::string> files, dirs;
350 
351  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
352  << dirs.size() << " dirs to check\n";
353 
354  const std::string& exclude_current = cache_file_prefix_ + "*";
355 
356  bool status = true;
357 
358  status &= delete_cache_files(files, exclude_current);
359  status &= delete_cache_files(dirs, exclude_current);
360 
361  LOG_CACHE << "clean_cache(): done\n";
362 
363  return status;
364 }
365 
367 {
368  std::vector<std::string> files, dirs;
370 
371  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
372  << dirs.size() << " dirs\n";
373 
374  bool status = true;
375 
376  status &= delete_cache_files(files);
377  status &= delete_cache_files(dirs);
378 
379  LOG_CACHE << "purge_cache(): done\n";
380  return status;
381 }
382 
383 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
384  const std::string& exclude_pattern)
385 {
386  const bool delete_everything = exclude_pattern.empty();
387  bool status = true;
388 
389  for(const std::string& file_path : paths)
390  {
391  if(!delete_everything) {
392  const std::string& fn = filesystem::base_name(file_path);
393 
394  if(utils::wildcard_string_match(fn, exclude_pattern)) {
395  LOG_CACHE << "delete_cache_files(): skipping " << file_path
396  << " excluded by '" << exclude_pattern << "'\n";
397  continue;
398  }
399  }
400 
401  LOG_CACHE << "delete_cache_files(): deleting " << file_path << '\n';
402  if(!filesystem::delete_directory(file_path)) {
403  ERR_CACHE << "delete_cache_files(): could not delete "
404  << file_path << '\n';
405  status = false;
406  }
407  }
408 
409  return status;
410 }
411 
414 
416  : define_filenames_()
417  , active_map_()
418 {
419  assert(state_ == FREE);
420  state_ = NEW;
421  active_ = this;
422 }
423 
425 {
426  state_ = FREE;
427  active_ = 0;
428 }
429 
431 {
432  state_ = LOCKED;
433 }
434 
435 const std::vector<std::string>& config_cache_transaction::get_define_files() const
436 {
437  return define_filenames_;
438 }
439 
440 void config_cache_transaction::add_define_file(const std::string& file)
441 {
442  define_filenames_.push_back(file);
443 }
444 
446 {
447  if(active_map_.empty()) {
448  active_map_.insert(defines_map.begin(), defines_map.end());
449  if(get_state() == NEW) {
450  state_ = ACTIVE;
451  }
452  }
453 
454  return active_map_;
455 }
456 
457 namespace
458 {
459 
460 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
461 {
462  if(a.first < b.first) {
463  return true;
464  }
465 
466  if(b.first < a.first) {
467  return false;
468  }
469 
470  if(a.second < b.second) {
471  return true;
472  }
473 
474  return false;
475 }
476 
477 } // end anonymous namespace
478 
480 {
481  if(get_state() == ACTIVE) {
482  preproc_map temp;
483  std::set_difference(new_map.begin(),
484  new_map.end(),
485  active_map_.begin(),
486  active_map_.end(),
487  std::insert_iterator<preproc_map>(temp,temp.begin()),
488  &compare_define);
489 
490  for(const preproc_map::value_type &def : temp) {
491  insert_to_active(def);
492  }
493 
494  temp.swap(new_map);
495  } else if (get_state() == LOCKED) {
496  new_map.clear();
497  }
498 }
499 
500 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
501 {
502  active_map_[def.first] = def.second;
503 }
504 
505 }
bool delete_directory(const std::string &dirname, const bool keep_pbl)
Definition: filesystem.cpp:948
static config_cache & instance()
Get reference to the singleton object.
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:978
void write(const config &cfg)
void write_file(std::string file, const config &cfg)
Interfaces for manipulating version numbers of engine, add-ons, etc.
void read_file(const std::string &file, config &cfg)
#define a
void lock()
Lock the transaction so no more macros are added.
static bool file_exists(const bfs::path &fpath)
Definition: filesystem.cpp:263
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void read_cache(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using &#39;*&#39; as any number of characters (including none), &#39;+&#39; as one or more characters, and &#39;?&#39; as any one character.
static config_cache_transaction * active_
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
Holds a fake cache transaction if no real one is used.
#define d
std::vector< std::string > define_filenames_
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:683
static config_cache_transaction & instance()
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
const std::vector< std::string > & get_define_files() const
#define b
Used in parsing config file.
Definition: validator.hpp:37
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:627
Class for writing a config out to a file in pieces.
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_defines_file(const std::string &path)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Get a list of all files and/or directories in a given directory.
Definition: filesystem.cpp:349
void add_define_file(const std::string &file)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
void set_force_valid_cache(bool force)
Enable/disable cache validation.
#define LOG_CACHE
void read_configs(const std::string &path, config &cfg, preproc_map &defines, abstract_validator *validator=nullptr)
static lg::log_domain log_cache("cache")
static int writer(lua_State *L, const void *b, size_t size, void *ud)
Definition: lstrlib.cpp:221
void get_config(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
Gets a config object from given path.
void set_use_cache(bool use)
Enable/disable caching.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
std::string get_cache_dir()
Definition: filesystem.cpp:797
void clear_defines()
Clear stored defines map to default values.
const std::string revision
void add_defines_map_diff(preproc_map &)
mock_party p
Game configuration data as global variables.
Definition: build_info.cpp:59
#define DBG_CACHE
const preproc_map & get_preproc_map() const
An exception object used when an IO error occurs.
Definition: filesystem.hpp:48
static tcache cache
Definition: minimap.cpp:124
preproc_map & get_active_map(const preproc_map &defines_map)
#define log_scope(description)
Definition: log.hpp:218
Declarations for File-IO.
const version_info wesnoth_version(VERSION)
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
Used to share macros between cache objects You have to create transaction object to load all macros t...
#define ERR_CACHE
void load_configs(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
void add_defines_map_diff(preproc_map &defines_map)
Standard logging facilities (interface).
std::string message
Definition: exceptions.hpp:30
std::map< std::string, struct preproc_define > preproc_map
#define e
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:61
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static preproc_map::value_type read_pair(const config &)
preproc_map & make_copy_map()
void add_define(const std::string &define)
Add a entry to preproc defines map.
void recheck_filetree_checksum()
Force cache checksum validation.
bool purge_cache()
Deletes all cache files.
Singleton class to manage game config file caching.
int cache_compression_level
Definition: game_config.cpp:86