The Battle for Wesnoth  1.15.11+dev
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2018 by Pauli Nieminen <paniemin@cc.hut.fi>
3  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
4 
5  This program is free software; you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation; either version 2 of the License, or
8  (at your option) any later version.
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY.
11 
12  See the COPYING file for more details.
13 */
14 
15 #define GETTEXT_DOMAIN "wesnoth-lib"
16 
17 #include "config_cache.hpp"
18 #include "filesystem.hpp"
19 #include "gettext.hpp"
20 #include "game_config.hpp"
21 #include "log.hpp"
22 #include "hash.hpp"
24 #include "serialization/parser.hpp"
26 #include "game_version.hpp"
27 
28 #include <boost/algorithm/string/replace.hpp>
29 #include <boost/iostreams/filter/gzip.hpp>
30 #include <SDL2/SDL_platform.h>
31 
32 static lg::log_domain log_cache("cache");
33 #define ERR_CACHE LOG_STREAM(err, log_cache)
34 #define LOG_CACHE LOG_STREAM(info, log_cache)
35 #define DBG_CACHE LOG_STREAM(debug, log_cache)
36 
37 namespace game_config
38 {
39 
40 namespace
41 {
42 
43 void add_builtin_defines(preproc_map& target)
44 {
45 #ifdef __APPLE__
46  target["APPLE"] = preproc_define();
47 #endif
48 
49 #if defined(MOUSE_TOUCH_EMULATION) || defined(__IPHONEOS__)
50  target["IPHONEOS"] = preproc_define();
51 #endif
52 
53  target["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
54 }
55 
56 }
57 
59 {
60  static config_cache cache;
61  return cache;
62 }
63 
65  : force_valid_cache_(false)
66  , use_cache_(true)
67  , fake_invalid_cache_(false)
68  , defines_map_()
69  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
70 {
71  // To set-up initial defines map correctly
72  clear_defines();
73 }
74 
76 {
77  return defines_map_;
78 }
79 
81 {
82  LOG_CACHE << "Clearing defines map!" << std::endl;
83 
84  defines_map_.clear();
85 
86  //
87  // Set-up default defines map.
88  //
89 
90  add_builtin_defines(defines_map_);
91 }
92 
93 void config_cache::get_config(const std::string& file_path, config& cfg, abstract_validator* validator)
94 {
95  load_configs(file_path, cfg, validator);
96 }
97 
98 void config_cache::write_file(std::string file_path, const config& cfg)
99 {
102  writer.write(cfg);
103 }
104 
105 void config_cache::write_file(std::string file_path, const preproc_map& defines)
106 {
107  if(defines.empty()) {
108  if(filesystem::file_exists(file_path)) {
109  filesystem::delete_directory(file_path);
110  }
111  return;
112  }
113 
116 
117  // Write all defines to stream.
118  for(const preproc_map::value_type& define : defines) {
119  define.second.write(writer, define.first);
120  }
121 }
122 
123 void config_cache::read_file(const std::string& file_path, config& cfg)
124 {
126  read_gz(cfg, *stream);
127 }
128 
130 {
132  // HACK: copy_map doesn't have built-in defines in some cases (issue #1924)
133  // or they may be out of date, so brute-force them in.
134  add_builtin_defines(res);
135  return res;
136 }
137 
139 {
141 }
142 
143 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map, abstract_validator* validator)
144 {
145  //read the file and then write to the cache
146  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
147  read(cfg, *stream, validator);
148 }
149 
150 void config_cache::read_cache(const std::string& file_path, config& cfg, abstract_validator* validator)
151 {
152  static const std::string extension = ".gz";
153 
154  std::stringstream defines_string;
155  defines_string << file_path;
156 
157  bool is_valid = true;
158 
159  for(const preproc_map::value_type& d : defines_map_) {
160  //
161  // Only WESNOTH_VERSION is allowed to be non-empty.
162  //
163  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
164  d.first != "WESNOTH_VERSION")
165  {
166  is_valid = false;
167  ERR_CACHE << "Invalid preprocessor define: " << d.first << '\n';
168  break;
169  }
170 
171  defines_string << " " << d.first;
172  }
173 
174  // Do cache check only if define map is valid and
175  // caching is allowed.
176  const std::string& cache_path = filesystem::get_cache_dir();
177 
178  if(is_valid && !cache_path.empty()) {
179  // Use a hash for a shorter display of the defines.
180  const std::string fname = cache_path + "/" +
182  utils::sha1(defines_string.str()).hex_digest();
183  const std::string fname_checksum = fname + ".checksum" + extension;
184 
185  filesystem::file_tree_checksum dir_checksum;
186 
188  try {
189  if(filesystem::file_exists(fname_checksum)) {
190  config checksum_cfg;
191 
192  DBG_CACHE << "Reading checksum: " << fname_checksum << "\n";
193  read_file(fname_checksum, checksum_cfg);
194 
195  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
196  }
197  } catch(const config::error&) {
198  ERR_CACHE << "cache checksum is corrupt" << std::endl;
199  } catch(const filesystem::io_exception&) {
200  ERR_CACHE << "error reading cache checksum" << std::endl;
201  } catch(const std::ios_base::failure&) {
202  ERR_CACHE << "error reading cache checksum" << std::endl;
203  }
204  }
205 
206  if(force_valid_cache_) {
207  LOG_CACHE << "skipping cache validation (forced)\n";
208  }
209 
210  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
211  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str() << "\n";
212  log_scope("read cache");
213 
214  try {
215  read_file(fname + extension,cfg);
216  const std::string define_file = fname + ".define" + extension;
217 
218  if(filesystem::file_exists(define_file)) {
220  }
221 
222  return;
223  } catch(const config::error& e) {
224  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message << std::endl;
225  } catch(const filesystem::io_exception&) {
226  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files" << std::endl;
227  } catch (const boost::iostreams::gzip_error& e) {
228  //read_file -> ... -> read_gz can throw this exception.
229  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error() << std::endl;
230  }
231  }
232 
233  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'\n";
234 
235  // Now we need queued defines so read them to memory
237 
238  preproc_map copy_map(make_copy_map());
239 
240  read_configs(file_path, cfg, copy_map, validator);
241  add_defines_map_diff(copy_map);
242 
243  try {
244  write_file(fname + extension, cfg);
245  write_file(fname + ".define" + extension, copy_map);
246 
247  config checksum_cfg;
248 
249  filesystem::data_tree_checksum().write(checksum_cfg);
250  write_file(fname_checksum, checksum_cfg);
251  } catch(const filesystem::io_exception&) {
252  ERR_CACHE << "could not write to cache '" << fname << "'" << std::endl;
253  }
254 
255  return;
256  }
257 
258  LOG_CACHE << "Loading plain config instead of cache\n";
259 
260  preproc_map copy_map(make_copy_map());
261  read_configs(file_path, cfg, copy_map, validator);
262  add_defines_map_diff(copy_map);
263 }
264 
265 void config_cache::read_defines_file(const std::string& file_path)
266 {
267  config cfg;
268  read_file(file_path, cfg);
269 
270  DBG_CACHE << "Reading cached defines from: " << file_path << "\n";
271 
272  // use static preproc_define::read_pair(config) to make a object
273  // and pass that object config_cache_transaction::insert_to_active method
274  for(const config::any_child &value : cfg.all_children_range()) {
276  preproc_define::read_pair(value.cfg));
277  }
278 }
279 
281 {
282  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
283 
284  for(const std::string &p : files) {
286  }
287 }
288 
289 void config_cache::load_configs(const std::string& config_path, config& cfg, abstract_validator* validator)
290 {
291  // Make sure that we have fake transaction if no real one is going on
292  fake_transaction fake;
293 
294  if (use_cache_) {
295  read_cache(config_path, cfg, validator);
296  } else {
297  preproc_map copy_map(make_copy_map());
298  read_configs(config_path, cfg, copy_map, validator);
299  add_defines_map_diff(copy_map);
300  }
301 }
302 
304 {
305  fake_invalid_cache_ = force;
306 }
307 
309 {
310  use_cache_ = use;
311 }
312 
314 {
315  force_valid_cache_ = force;
316 }
317 
319 {
321 }
322 
323 void config_cache::add_define(const std::string& define)
324 {
325  DBG_CACHE << "adding define: " << define << "\n";
326  defines_map_[define] = preproc_define();
327 
329  // we have to add this to active map too
331  }
332 
333 }
334 
335 void config_cache::remove_define(const std::string& define)
336 {
337  DBG_CACHE << "removing define: " << define << "\n";
338  defines_map_.erase(define);
339 
341  // we have to remove this from active map too
343  }
344 }
345 
347 {
348  std::vector<std::string> files, dirs;
350 
351  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
352  << dirs.size() << " dirs to check\n";
353 
354  const std::string& exclude_current = cache_file_prefix_ + "*";
355 
356  bool status = true;
357 
358  status &= delete_cache_files(files, exclude_current);
359  status &= delete_cache_files(dirs, exclude_current);
360 
361  LOG_CACHE << "clean_cache(): done\n";
362 
363  return status;
364 }
365 
367 {
368  std::vector<std::string> files, dirs;
370 
371  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
372  << dirs.size() << " dirs\n";
373 
374  bool status = true;
375 
376  status &= delete_cache_files(files);
377  status &= delete_cache_files(dirs);
378 
379  LOG_CACHE << "purge_cache(): done\n";
380  return status;
381 }
382 
383 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
384  const std::string& exclude_pattern)
385 {
386  const bool delete_everything = exclude_pattern.empty();
387  bool status = true;
388 
389  for(const std::string& file_path : paths)
390  {
391  if(!delete_everything) {
392  const std::string& fn = filesystem::base_name(file_path);
393 
394  if(utils::wildcard_string_match(fn, exclude_pattern)) {
395  LOG_CACHE << "delete_cache_files(): skipping " << file_path
396  << " excluded by '" << exclude_pattern << "'\n";
397  continue;
398  }
399  }
400 
401  LOG_CACHE << "delete_cache_files(): deleting " << file_path << '\n';
402  if(!filesystem::delete_directory(file_path)) {
403  ERR_CACHE << "delete_cache_files(): could not delete "
404  << file_path << '\n';
405  status = false;
406  }
407  }
408 
409  return status;
410 }
411 
414 
416  : define_filenames_()
417  , active_map_()
418 {
419  assert(state_ == FREE);
420  state_ = NEW;
421  active_ = this;
422 }
423 
425 {
426  state_ = FREE;
427  active_ = 0;
428 }
429 
431 {
432  state_ = LOCKED;
433 }
434 
435 const std::vector<std::string>& config_cache_transaction::get_define_files() const
436 {
437  return define_filenames_;
438 }
439 
440 void config_cache_transaction::add_define_file(const std::string& file)
441 {
442  define_filenames_.push_back(file);
443 }
444 
446 {
447  if(active_map_.empty()) {
448  active_map_.insert(defines_map.begin(), defines_map.end());
449  if(get_state() == NEW) {
450  state_ = ACTIVE;
451  }
452  }
453 
454  return active_map_;
455 }
456 
457 namespace
458 {
459 
460 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
461 {
462  if(a.first < b.first) {
463  return true;
464  }
465 
466  if(b.first < a.first) {
467  return false;
468  }
469 
470  if(a.second < b.second) {
471  return true;
472  }
473 
474  return false;
475 }
476 
477 } // end anonymous namespace
478 
480 {
481  if(get_state() == ACTIVE) {
482  preproc_map temp;
483  std::set_difference(new_map.begin(),
484  new_map.end(),
485  active_map_.begin(),
486  active_map_.end(),
487  std::insert_iterator<preproc_map>(temp,temp.begin()),
488  &compare_define);
489 
490  for(const preproc_map::value_type &def : temp) {
491  insert_to_active(def);
492  }
493 
494  temp.swap(new_map);
495  } else if (get_state() == LOCKED) {
496  new_map.clear();
497  }
498 }
499 
500 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
501 {
502  active_map_[def.first] = def.second;
503 }
504 
505 }
bool delete_directory(const std::string &dirname, const bool keep_pbl)
Definition: filesystem.cpp:945
static config_cache & instance()
Get reference to the singleton object.
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:953
void write(const config &cfg)
void write_file(std::string file, const config &cfg)
Interfaces for manipulating version numbers of engine, add-ons, etc.
void read_file(const std::string &file, config &cfg)
#define a
void lock()
Lock the transaction so no more macros are added.
static bool file_exists(const bfs::path &fpath)
Definition: filesystem.cpp:263
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void read_cache(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using &#39;*&#39; as any number of characters (including none), &#39;+&#39; as one or more characters, and &#39;?&#39; as any one character.
static config_cache_transaction * active_
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
Holds a fake cache transaction if no real one is used.
#define d
std::vector< std::string > define_filenames_
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:682
static config_cache_transaction & instance()
const std::vector< std::string > & get_define_files() const
#define b
Used in parsing config file.
Definition: validator.hpp:35
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:626
Class for writing a config out to a file in pieces.
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_defines_file(const std::string &path)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:349
void add_define_file(const std::string &file)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:36
void set_force_valid_cache(bool force)
Enable/disable cache validation.
#define LOG_CACHE
void read_configs(const std::string &path, config &cfg, preproc_map &defines, abstract_validator *validator=nullptr)
static lg::log_domain log_cache("cache")
static int writer(lua_State *L, const void *b, size_t size, void *ud)
Definition: lstrlib.cpp:221
void get_config(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
Gets a config object from given path.
void set_use_cache(bool use)
Enable/disable caching.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:37
std::string get_cache_dir()
Definition: filesystem.cpp:794
void clear_defines()
Clear stored defines map to default values.
const std::string revision
void add_defines_map_diff(preproc_map &)
mock_party p
Game configuration data as global variables.
Definition: build_info.cpp:58
#define DBG_CACHE
const preproc_map & get_preproc_map() const
An exception object used when an IO error occurs.
Definition: filesystem.hpp:45
static tcache cache
Definition: minimap.cpp:123
preproc_map & get_active_map(const preproc_map &defines_map)
#define log_scope(description)
Definition: log.hpp:206
Declarations for File-IO.
const version_info wesnoth_version(VERSION)
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
Used to share macros between cache objects You have to create transaction object to load all macros t...
#define ERR_CACHE
void load_configs(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
void add_defines_map_diff(preproc_map &defines_map)
Standard logging facilities (interface).
std::string message
Definition: exceptions.hpp:29
std::map< std::string, struct preproc_define > preproc_map
#define e
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:59
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static preproc_map::value_type read_pair(const config &)
preproc_map & make_copy_map()
void add_define(const std::string &define)
Add a entry to preproc defines map.
void recheck_filetree_checksum()
Force cache checksum validation.
bool purge_cache()
Deletes all cache files.
Singleton class to manage game config file caching.
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory)
int cache_compression_level
Definition: game_config.cpp:84