The Battle for Wesnoth  1.15.0-dev
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2018 by Pauli Nieminen <paniemin@cc.hut.fi>
3  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
4 
5  This program is free software; you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation; either version 2 of the License, or
8  (at your option) any later version.
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY.
11 
12  See the COPYING file for more details.
13 */
14 
15 #define GETTEXT_DOMAIN "wesnoth-lib"
16 
17 #include "config_cache.hpp"
18 #include "filesystem.hpp"
19 #include "gettext.hpp"
20 #include "game_config.hpp"
21 #include "log.hpp"
22 #include "hash.hpp"
24 #include "serialization/parser.hpp"
26 #include "version.hpp"
27 
28 #include <boost/algorithm/string/replace.hpp>
29 #include <boost/iostreams/filter/gzip.hpp>
30 
31 static lg::log_domain log_cache("cache");
32 #define ERR_CACHE LOG_STREAM(err, log_cache)
33 #define LOG_CACHE LOG_STREAM(info, log_cache)
34 #define DBG_CACHE LOG_STREAM(debug, log_cache)
35 
36 namespace game_config
37 {
38 
39 namespace
40 {
41 
42 void add_builtin_defines(preproc_map& target)
43 {
44 #ifdef __APPLE__
45  target["APPLE"] = preproc_define();
46 #endif
47 
48  target["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
49 }
50 
51 }
52 
54 {
55  static config_cache cache;
56  return cache;
57 }
58 
60  : force_valid_cache_(false)
61  , use_cache_(true)
62  , fake_invalid_cache_(false)
63  , defines_map_()
64  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
65 {
66  // To set-up initial defines map correctly
67  clear_defines();
68 }
69 
71 {
72  return defines_map_;
73 }
74 
76 {
77  LOG_CACHE << "Clearing defines map!" << std::endl;
78 
79  defines_map_.clear();
80 
81  //
82  // Set-up default defines map.
83  //
84 
85  add_builtin_defines(defines_map_);
86 }
87 
88 void config_cache::get_config(const std::string& file_path, config& cfg)
89 {
90  load_configs(file_path, cfg);
91 }
92 
93 void config_cache::write_file(std::string file_path, const config& cfg)
94 {
97  writer.write(cfg);
98 }
99 
100 void config_cache::write_file(std::string file_path, const preproc_map& defines)
101 {
102  if(defines.empty()) {
103  if(filesystem::file_exists(file_path)) {
104  filesystem::delete_directory(file_path);
105  }
106  return;
107  }
108 
111 
112  // Write all defines to stream.
113  for(const preproc_map::value_type& define : defines) {
114  define.second.write(writer, define.first);
115  }
116 }
117 
118 void config_cache::read_file(const std::string& file_path, config& cfg)
119 {
121  read_gz(cfg, *stream);
122 }
123 
125 {
127  // HACK: copy_map doesn't have built-in defines in some cases (issue #1924)
128  // or they may be out of date, so brute-force them in.
129  add_builtin_defines(res);
130  return res;
131 }
132 
134 {
136 }
137 
138 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map)
139 {
140  //read the file and then write to the cache
141  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
142  read(cfg, *stream);
143 }
144 
145 void config_cache::read_cache(const std::string& file_path, config& cfg)
146 {
147  static const std::string extension = ".gz";
148 
149  std::stringstream defines_string;
150  defines_string << file_path;
151 
152  bool is_valid = true;
153 
154  for(const preproc_map::value_type& d : defines_map_) {
155  //
156  // Only WESNOTH_VERSION is allowed to be non-empty.
157  //
158  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
159  d.first != "WESNOTH_VERSION")
160  {
161  is_valid = false;
162  ERR_CACHE << "Invalid preprocessor define: " << d.first << '\n';
163  break;
164  }
165 
166  defines_string << " " << d.first;
167  }
168 
169  // Do cache check only if define map is valid and
170  // caching is allowed.
171  const std::string& cache_path = filesystem::get_cache_dir();
172 
173  if(is_valid && !cache_path.empty()) {
174  // Use a hash for a shorter display of the defines.
175  const std::string fname = cache_path + "/" +
177  utils::sha1(defines_string.str()).hex_digest();
178  const std::string fname_checksum = fname + ".checksum" + extension;
179 
180  filesystem::file_tree_checksum dir_checksum;
181 
183  try {
184  if(filesystem::file_exists(fname_checksum)) {
185  config checksum_cfg;
186 
187  DBG_CACHE << "Reading checksum: " << fname_checksum << "\n";
188  read_file(fname_checksum, checksum_cfg);
189 
190  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
191  }
192  } catch(const config::error&) {
193  ERR_CACHE << "cache checksum is corrupt" << std::endl;
194  } catch(const filesystem::io_exception&) {
195  ERR_CACHE << "error reading cache checksum" << std::endl;
196  }
197  }
198 
199  if(force_valid_cache_) {
200  LOG_CACHE << "skipping cache validation (forced)\n";
201  }
202 
203  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
204  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str() << "\n";
205  log_scope("read cache");
206 
207  try {
208  read_file(fname + extension,cfg);
209  const std::string define_file = fname + ".define" + extension;
210 
211  if(filesystem::file_exists(define_file)) {
213  }
214 
215  return;
216  } catch(const config::error& e) {
217  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message << std::endl;
218  } catch(const filesystem::io_exception&) {
219  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files" << std::endl;
220  } catch (const boost::iostreams::gzip_error& e) {
221  //read_file -> ... -> read_gz can throw this exception.
222  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error() << std::endl;
223  }
224  }
225 
226  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'\n";
227 
228  // Now we need queued defines so read them to memory
230 
231  preproc_map copy_map(make_copy_map());
232 
233  read_configs(file_path, cfg, copy_map);
234  add_defines_map_diff(copy_map);
235 
236  try {
237  write_file(fname + extension, cfg);
238  write_file(fname + ".define" + extension, copy_map);
239 
240  config checksum_cfg;
241 
242  filesystem::data_tree_checksum().write(checksum_cfg);
243  write_file(fname_checksum, checksum_cfg);
244  } catch(const filesystem::io_exception&) {
245  ERR_CACHE << "could not write to cache '" << fname << "'" << std::endl;
246  }
247 
248  return;
249  }
250 
251  LOG_CACHE << "Loading plain config instead of cache\n";
252 
253  preproc_map copy_map(make_copy_map());
254  read_configs(file_path, cfg, copy_map);
255  add_defines_map_diff(copy_map);
256 }
257 
258 void config_cache::read_defines_file(const std::string& file_path)
259 {
260  config cfg;
261  read_file(file_path, cfg);
262 
263  DBG_CACHE << "Reading cached defines from: " << file_path << "\n";
264 
265  // use static preproc_define::read_pair(config) to make a object
266  // and pass that object config_cache_transaction::insert_to_active method
267  for(const config::any_child &value : cfg.all_children_range()) {
269  preproc_define::read_pair(value.cfg));
270  }
271 }
272 
274 {
275  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
276 
277  for(const std::string &p : files) {
279  }
280 }
281 
282 void config_cache::load_configs(const std::string& config_path, config& cfg)
283 {
284  // Make sure that we have fake transaction if no real one is going on
285  fake_transaction fake;
286 
287  if (use_cache_) {
288  read_cache(config_path, cfg);
289  } else {
290  preproc_map copy_map(make_copy_map());
291  read_configs(config_path, cfg, copy_map);
292  add_defines_map_diff(copy_map);
293  }
294 }
295 
297 {
298  fake_invalid_cache_ = force;
299 }
300 
302 {
303  use_cache_ = use;
304 }
305 
307 {
308  force_valid_cache_ = force;
309 }
310 
312 {
314 }
315 
316 void config_cache::add_define(const std::string& define)
317 {
318  DBG_CACHE << "adding define: " << define << "\n";
319  defines_map_[define] = preproc_define();
320 
322  // we have to add this to active map too
324  }
325 
326 }
327 
328 void config_cache::remove_define(const std::string& define)
329 {
330  DBG_CACHE << "removing define: " << define << "\n";
331  defines_map_.erase(define);
332 
334  // we have to remove this from active map too
336  }
337 }
338 
340 {
341  std::vector<std::string> files, dirs;
343 
344  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
345  << dirs.size() << " dirs to check\n";
346 
347  const std::string& exclude_current = cache_file_prefix_ + "*";
348 
349  bool status = true;
350 
351  status &= delete_cache_files(files, exclude_current);
352  status &= delete_cache_files(dirs, exclude_current);
353 
354  LOG_CACHE << "clean_cache(): done\n";
355 
356  return status;
357 }
358 
360 {
361  std::vector<std::string> files, dirs;
363 
364  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
365  << dirs.size() << " dirs\n";
366 
367  bool status = true;
368 
369  status &= delete_cache_files(files);
370  status &= delete_cache_files(dirs);
371 
372  LOG_CACHE << "purge_cache(): done\n";
373  return status;
374 }
375 
376 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
377  const std::string& exclude_pattern)
378 {
379  const bool delete_everything = exclude_pattern.empty();
380  bool status = true;
381 
382  for(const std::string& file_path : paths)
383  {
384  if(!delete_everything) {
385  const std::string& fn = filesystem::base_name(file_path);
386 
387  if(utils::wildcard_string_match(fn, exclude_pattern)) {
388  LOG_CACHE << "delete_cache_files(): skipping " << file_path
389  << " excluded by '" << exclude_pattern << "'\n";
390  continue;
391  }
392  }
393 
394  LOG_CACHE << "delete_cache_files(): deleting " << file_path << '\n';
395  if(!filesystem::delete_directory(file_path)) {
396  ERR_CACHE << "delete_cache_files(): could not delete "
397  << file_path << '\n';
398  status = false;
399  }
400  }
401 
402  return status;
403 }
404 
407 
409  : define_filenames_()
410  , active_map_()
411 {
412  assert(state_ == FREE);
413  state_ = NEW;
414  active_ = this;
415 }
416 
418 {
419  state_ = FREE;
420  active_ = 0;
421 }
422 
424 {
425  state_ = LOCKED;
426 }
427 
428 const std::vector<std::string>& config_cache_transaction::get_define_files() const
429 {
430  return define_filenames_;
431 }
432 
433 void config_cache_transaction::add_define_file(const std::string& file)
434 {
435  define_filenames_.push_back(file);
436 }
437 
439 {
440  if(active_map_.empty()) {
441  active_map_.insert(defines_map.begin(), defines_map.end());
442  if(get_state() == NEW) {
443  state_ = ACTIVE;
444  }
445  }
446 
447  return active_map_;
448 }
449 
450 namespace
451 {
452 
453 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
454 {
455  if(a.first < b.first) {
456  return true;
457  }
458 
459  if(b.first < a.first) {
460  return false;
461  }
462 
463  if(a.second < b.second) {
464  return true;
465  }
466 
467  return false;
468 }
469 
470 } // end anonymous namespace
471 
473 {
474  if(get_state() == ACTIVE) {
475  preproc_map temp;
476  std::set_difference(new_map.begin(),
477  new_map.end(),
478  active_map_.begin(),
479  active_map_.end(),
480  std::insert_iterator<preproc_map>(temp,temp.begin()),
481  &compare_define);
482 
483  for(const preproc_map::value_type &def : temp) {
484  insert_to_active(def);
485  }
486 
487  temp.swap(new_map);
488  } else if (get_state() == LOCKED) {
489  new_map.clear();
490  }
491 }
492 
493 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
494 {
495  active_map_[def.first] = def.second;
496 }
497 
498 }
bool delete_directory(const std::string &dirname, const bool keep_pbl)
Definition: filesystem.cpp:836
static config_cache & instance()
Get reference to the singleton object.
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:874
void write(const config &cfg)
void write_file(std::string file, const config &cfg)
void read_configs(const std::string &path, config &cfg, preproc_map &defines)
void read_file(const std::string &file, config &cfg)
static bool file_exists(const fs::path &fpath)
Definition: filesystem.cpp:300
#define a
void lock()
Lock the transaction so no more macros are added.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
Definition: filesystem.cpp:894
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using &#39;*&#39; as any number of characters (including none), &#39;+&#39; as one or more characters, and &#39;?&#39; as any one character.
static config_cache_transaction * active_
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
Holds a fake cache transaction if no real one is used.
#define d
std::vector< std::string > define_filenames_
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:668
static config_cache_transaction & instance()
const std::vector< std::string > & get_define_files() const
#define b
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:612
Class for writing a config out to a file in pieces.
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_defines_file(const std::string &path)
void get_config(const std::string &path, config &cfg)
Gets a config object from given path.
void add_define_file(const std::string &file)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
void set_force_valid_cache(bool force)
Enable/disable cache validation.
#define LOG_CACHE
static lg::log_domain log_cache("cache")
void set_use_cache(bool use)
Enable/disable caching.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
void load_configs(const std::string &path, config &cfg)
std::string get_cache_dir()
Definition: filesystem.cpp:754
void clear_defines()
Clear stored defines map to default values.
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, file_name_option mode, file_filter_option filter, file_reorder_option reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:386
const std::string revision
Definition: version.cpp:42
void add_defines_map_diff(preproc_map &)
mock_party p
Game configuration data as global variables.
Definition: build_info.cpp:46
#define DBG_CACHE
const preproc_map & get_preproc_map() const
An exception object used when an IO error occurs.
Definition: filesystem.hpp:48
preproc_map & get_active_map(const preproc_map &defines_map)
#define log_scope(description)
Definition: log.hpp:186
void read_cache(const std::string &path, config &cfg)
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
const version_info wesnoth_version(VERSION)
Definition: version.hpp:210
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
Used to share macros between cache objects You have to create transaction object to load all macros t...
#define ERR_CACHE
void add_defines_map_diff(preproc_map &defines_map)
Standard logging facilities (interface).
std::string message
Definition: exceptions.hpp:31
std::map< std::string, struct preproc_define > preproc_map
#define e
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:68
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
preproc_map & make_copy_map()
Interfaces for manipulating version numbers of engine, add-ons, etc.
void add_define(const std::string &define)
Add a entry to preproc defines map.
void recheck_filetree_checksum()
Force cache checksum validation.
bool purge_cache()
Deletes all cache files.
Singleton class to manage game config file caching.
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory)
Definition: filesystem.cpp:932
int cache_compression_level
Definition: game_config.cpp:84