The Battle for Wesnoth  1.13.11+dev
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2018 by Pauli Nieminen <paniemin@cc.hut.fi>
3  Part of the Battle for Wesnoth Project http://www.wesnoth.org/
4 
5  This program is free software; you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation; either version 2 of the License, or
8  (at your option) any later version.
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY.
11 
12  See the COPYING file for more details.
13 */
14 
15 #define GETTEXT_DOMAIN "wesnoth-lib"
16 
17 #include "config_cache.hpp"
18 #include "filesystem.hpp"
19 #include "gettext.hpp"
20 #include "game_config.hpp"
21 #include "log.hpp"
22 #include "hash.hpp"
24 #include "serialization/parser.hpp"
26 #include "version.hpp"
27 
28 #include <boost/algorithm/string/replace.hpp>
29 #include <boost/iostreams/filter/gzip.hpp>
30 
31 static lg::log_domain log_cache("cache");
32 #define ERR_CACHE LOG_STREAM(err, log_cache)
33 #define LOG_CACHE LOG_STREAM(info, log_cache)
34 #define DBG_CACHE LOG_STREAM(debug, log_cache)
35 
36 namespace game_config
37 {
38 
40 {
41  static config_cache cache;
42  return cache;
43 }
44 
46  : force_valid_cache_(false)
47  , use_cache_(true)
48  , fake_invalid_cache_(false)
49  , defines_map_()
50  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
51 {
52  // To set-up initial defines map correctly
53  clear_defines();
54 }
55 
57 {
58  return defines_map_;
59 }
60 
62 {
63  LOG_CACHE << "Clearing defines map!" << std::endl;
64 
65  defines_map_.clear();
66 
67  //
68  // Set-up default defines map.
69  //
70 
71 #ifdef __APPLE__
72  defines_map_["APPLE"] = preproc_define();
73 #endif
74 
75  defines_map_["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
76 }
77 
78 void config_cache::get_config(const std::string& file_path, config& cfg)
79 {
80  load_configs(file_path, cfg);
81 }
82 
83 void config_cache::write_file(std::string file_path, const config& cfg)
84 {
87  writer.write(cfg);
88 }
89 
90 void config_cache::write_file(std::string file_path, const preproc_map& defines)
91 {
92  if(defines.empty()) {
93  if(filesystem::file_exists(file_path)) {
95  }
96  return;
97  }
98 
101 
102  // Write all defines to stream.
103  for(const preproc_map::value_type& define : defines) {
104  define.second.write(writer, define.first);
105  }
106 }
107 
108 void config_cache::read_file(const std::string& file_path, config& cfg)
109 {
111  read_gz(cfg, *stream);
112 }
113 
115 {
117 }
118 
120 {
122 }
123 
124 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map)
125 {
126  //read the file and then write to the cache
127  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
128  read(cfg, *stream);
129 }
130 
131 void config_cache::read_cache(const std::string& file_path, config& cfg)
132 {
133  static const std::string extension = ".gz";
134 
135  std::stringstream defines_string;
136  defines_string << file_path;
137 
138  bool is_valid = true;
139 
140  for(const preproc_map::value_type& d : defines_map_) {
141  //
142  // Only WESNOTH_VERSION is allowed to be non-empty.
143  //
144  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
145  d.first != "WESNOTH_VERSION")
146  {
147  is_valid = false;
148  ERR_CACHE << "Invalid preprocessor define: " << d.first << '\n';
149  break;
150  }
151 
152  defines_string << " " << d.first;
153  }
154 
155  // Do cache check only if define map is valid and
156  // caching is allowed.
157  const std::string& cache_path = filesystem::get_cache_dir();
158 
159  if(is_valid && !cache_path.empty()) {
160  // Use a hash for a shorter display of the defines.
161  const std::string fname = cache_path + "/" +
163  utils::sha1(defines_string.str()).hex_digest();
164  const std::string fname_checksum = fname + ".checksum" + extension;
165 
166  filesystem::file_tree_checksum dir_checksum;
167 
169  try {
170  if(filesystem::file_exists(fname_checksum)) {
171  config checksum_cfg;
172 
173  DBG_CACHE << "Reading checksum: " << fname_checksum << "\n";
174  read_file(fname_checksum, checksum_cfg);
175 
176  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
177  }
178  } catch(config::error&) {
179  ERR_CACHE << "cache checksum is corrupt" << std::endl;
180  } catch(filesystem::io_exception&) {
181  ERR_CACHE << "error reading cache checksum" << std::endl;
182  }
183  }
184 
185  if(force_valid_cache_) {
186  LOG_CACHE << "skipping cache validation (forced)\n";
187  }
188 
189  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
190  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str() << "\n";
191  log_scope("read cache");
192 
193  try {
194  read_file(fname + extension,cfg);
195  const std::string define_file = fname + ".define" + extension;
196 
197  if(filesystem::file_exists(define_file)) {
199  }
200 
201  return;
202  } catch(config::error& e) {
203  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message << std::endl;
204  } catch(filesystem::io_exception&) {
205  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files" << std::endl;
206  } catch (boost::iostreams::gzip_error& e) {
207  //read_file -> ... -> read_gz can throw this exception.
208  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error() << std::endl;
209  }
210  }
211 
212  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'\n";
213 
214  // Now we need queued defines so read them to memory
216 
217  preproc_map copy_map(make_copy_map());
218 
219  read_configs(file_path, cfg, copy_map);
220  add_defines_map_diff(copy_map);
221 
222  try {
223  write_file(fname + extension, cfg);
224  write_file(fname + ".define" + extension, copy_map);
225 
226  config checksum_cfg;
227 
228  filesystem::data_tree_checksum().write(checksum_cfg);
229  write_file(fname_checksum, checksum_cfg);
230  } catch(filesystem::io_exception&) {
231  ERR_CACHE << "could not write to cache '" << fname << "'" << std::endl;
232  }
233 
234  return;
235  }
236 
237  LOG_CACHE << "Loading plain config instead of cache\n";
238 
239  preproc_map copy_map(make_copy_map());
240  read_configs(file_path, cfg, copy_map);
241  add_defines_map_diff(copy_map);
242 }
243 
245 {
246  config cfg;
247  read_file(file_path, cfg);
248 
249  DBG_CACHE << "Reading cached defines from: " << file_path << "\n";
250 
251  // use static preproc_define::read_pair(config) to make a object
252  // and pass that object config_cache_transaction::insert_to_active method
253  for(const config::any_child &value : cfg.all_children_range()) {
255  preproc_define::read_pair(value.cfg));
256  }
257 }
258 
260 {
261  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
262 
263  for(const std::string &p : files) {
265  }
266 }
267 
268 void config_cache::load_configs(const std::string& config_path, config& cfg)
269 {
270  // Make sure that we have fake transaction if no real one is going on
271  fake_transaction fake;
272 
273  if (use_cache_) {
274  read_cache(config_path, cfg);
275  } else {
276  preproc_map copy_map(make_copy_map());
277  read_configs(config_path, cfg, copy_map);
278  add_defines_map_diff(copy_map);
279  }
280 }
281 
283 {
284  fake_invalid_cache_ = force;
285 }
286 
288 {
289  use_cache_ = use;
290 }
291 
293 {
294  force_valid_cache_ = force;
295 }
296 
298 {
300 }
301 
303 {
304  DBG_CACHE << "adding define: " << define << "\n";
305  defines_map_[define] = preproc_define();
306 
308  // we have to add this to active map too
310  }
311 
312 }
313 
315 {
316  DBG_CACHE << "removing define: " << define << "\n";
317  defines_map_.erase(define);
318 
320  // we have to remove this from active map too
322  }
323 }
324 
326 {
327  std::vector<std::string> files, dirs;
329 
330  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
331  << dirs.size() << " dirs to check\n";
332 
333  const std::string& exclude_current = cache_file_prefix_ + "*";
334 
335  bool status = true;
336 
337  status &= delete_cache_files(files, exclude_current);
338  status &= delete_cache_files(dirs, exclude_current);
339 
340  LOG_CACHE << "clean_cache(): done\n";
341 
342  return status;
343 }
344 
346 {
347  std::vector<std::string> files, dirs;
349 
350  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
351  << dirs.size() << " dirs\n";
352 
353  bool status = true;
354 
355  status &= delete_cache_files(files);
356  status &= delete_cache_files(dirs);
357 
358  LOG_CACHE << "purge_cache(): done\n";
359  return status;
360 }
361 
362 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
363  const std::string& exclude_pattern)
364 {
365  const bool delete_everything = exclude_pattern.empty();
366  bool status = true;
367 
368  for(const std::string& file_path : paths)
369  {
370  if(!delete_everything) {
371  const std::string& fn = filesystem::base_name(file_path);
372 
373  if(utils::wildcard_string_match(fn, exclude_pattern)) {
374  LOG_CACHE << "delete_cache_files(): skipping " << file_path
375  << " excluded by '" << exclude_pattern << "'\n";
376  continue;
377  }
378  }
379 
380  LOG_CACHE << "delete_cache_files(): deleting " << file_path << '\n';
381  if(!filesystem::delete_directory(file_path)) {
382  ERR_CACHE << "delete_cache_files(): could not delete "
383  << file_path << '\n';
384  status = false;
385  }
386  }
387 
388  return status;
389 }
390 
393 
395  : define_filenames_()
396  , active_map_()
397 {
398  assert(state_ == FREE);
399  state_ = NEW;
400  active_ = this;
401 }
402 
404 {
405  state_ = FREE;
406  active_ = 0;
407 }
408 
410 {
411  state_ = LOCKED;
412 }
413 
414 const std::vector<std::string>& config_cache_transaction::get_define_files() const
415 {
416  return define_filenames_;
417 }
418 
420 {
421  define_filenames_.push_back(file);
422 }
423 
425 {
426  if(active_map_.empty()) {
427  active_map_.insert(defines_map.begin(), defines_map.end());
428  if(get_state() == NEW) {
429  state_ = ACTIVE;
430  }
431  }
432 
433  return active_map_;
434 }
435 
436 namespace
437 {
438 
439 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
440 {
441  if(a.first < b.first) {
442  return true;
443  }
444 
445  if(b.first < a.first) {
446  return false;
447  }
448 
449  if(a.second < b.second) {
450  return true;
451  }
452 
453  return false;
454 }
455 
456 } // end anonymous namespace
457 
459 {
460  if(get_state() == ACTIVE) {
461  preproc_map temp;
462  std::set_difference(new_map.begin(),
463  new_map.end(),
464  active_map_.begin(),
465  active_map_.end(),
466  std::insert_iterator<preproc_map>(temp,temp.begin()),
467  &compare_define);
468 
469  for(const preproc_map::value_type &def : temp) {
470  insert_to_active(def);
471  }
472 
473  temp.swap(new_map);
474  } else if (get_state() == LOCKED) {
475  new_map.clear();
476  }
477 }
478 
479 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
480 {
481  active_map_[def.first] = def.second;
482 }
483 
484 }
static config_cache & instance()
Get reference to the singleton object.
std::vector< char_t > string
void write(const config &cfg)
void write_file(std::string file, const config &cfg)
void read_configs(const std::string &path, config &cfg, preproc_map &defines)
void read_file(const std::string &file, config &cfg)
#define a
void lock()
Lock the transaction so no more macros are added.
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using '*' as any number of characters (including none), and '?' as any one character.
static config_cache_transaction * active_
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
Holds a fake cache transaction if no real one is used.
#define d
std::vector< std::string > define_filenames_
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:668
static config_cache_transaction & instance()
const std::vector< std::string > & get_define_files() const
#define b
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error=true)
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:612
Class for writing a config out to a file in pieces.
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_defines_file(const std::string &path)
const preproc_map & get_preproc_map() const
void get_config(const std::string &path, config &cfg)
Gets a config object from given path.
void add_define_file(const std::string &file)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:37
void set_force_valid_cache(bool force)
Enable/disable cache validation.
#define LOG_CACHE
static lg::log_domain log_cache("cache")
void set_use_cache(bool use)
Enable/disable caching.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:38
void load_configs(const std::string &path, config &cfg)
std::string get_cache_dir()
void clear_defines()
Clear stored defines map to default values.
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory=true)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs=nullptr, file_name_option mode=FILE_NAME_ONLY, file_filter_option filter=NO_FILTER, file_reorder_option reorder=DONT_REORDER, file_tree_checksum *checksum=nullptr)
Populates 'files' with all the files and 'dirs' with all the directories in dir.
const std::string revision
Definition: game_config.cpp:50
void add_defines_map_diff(preproc_map &)
mock_party p
Game configuration data as global variables.
Definition: build_info.cpp:53
#define DBG_CACHE
An exception object used when an IO error occurs.
Definition: filesystem.hpp:46
static tcache cache
Definition: minimap.cpp:134
preproc_map & get_active_map(const preproc_map &defines_map)
#define log_scope(description)
Definition: log.hpp:186
void write(config &cfg) const
void read_cache(const std::string &path, config &cfg)
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
const version_info wesnoth_version(VERSION)
bool delete_directory(const std::string &dirname, const bool keep_pbl=false)
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
std::string base_name(const std::string &file, const bool remove_extension=false)
Returns the base filename of a file, with directory name stripped.
Used to share macros between cache objects You have to create transaction object to load all macros t...
#define ERR_CACHE
void add_defines_map_diff(preproc_map &defines_map)
Standard logging facilities (interface).
std::string message
Definition: exceptions.hpp:31
std::map< std::string, struct preproc_define > preproc_map
#define e
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:93
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
preproc_map & make_copy_map()
Interfaces for manipulating version numbers of engine, add-ons, etc.
bool file_exists(const std::string &name)
Returns true if a file or directory with such name already exists.
void add_define(const std::string &define)
Add a entry to preproc defines map.
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:865
void recheck_filetree_checksum()
Force cache checksum validation.
bool purge_cache()
Deletes all cache files.
Singleton class to manage game config file caching.
int cache_compression_level