All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
ratingstat.cc
Go to the documentation of this file.
2 #include "osl/rating/ratingEnv.h"
6 #include "osl/record/csaRecord.h"
8 #include "osl/record/kisen.h"
9 #include "osl/misc/perfmon.h"
10 #include "osl/stat/histogram.h"
11 #include "osl/stat/variance.h"
12 #include "osl/stl/vector.h"
13 
14 #include <boost/format.hpp>
15 #include <string>
16 #include <iostream>
17 #include <iomanip>
18 #include <cmath>
19 using namespace osl;
20 using namespace osl::rating;
21 
22 void usage(const char *prog)
23 {
24  using namespace std;
25  cerr << "Usage: " << prog << " [-v] [-f skip] csafiles or -k kisen-filename -n num\n"
26  << endl;
27  exit(1);
28 }
29 
30 size_t first_skip = 3;
31 int verbose = 0;
32 const char *kisen_filename=0;
33 size_t num_kisen = 4000;
34 size_t kisen_start = 200000;
35 size_t min_rating = 1500;
36 
37 struct KeepMin
38 {
39  int min;
40  explicit KeepMin(int initial = 10000) : min(initial)
41  {
42  }
43  void add(int value) { min = std::min(value, min); }
44  int value() const { return min; }
45 };
46 
47 struct KeepMax
48 {
49  int max;
50  explicit KeepMax(int initial = -10000) : max(initial)
51  {
52  }
53  void add(int value) { max = std::max(value, max); }
54  int value() const { return max; }
55 };
56 
57 struct Histogram8
58 {
59  CArray<stat::Histogram*,8> histograms;
60  const stat::Histogram& operator[](size_t i) const
61  {
62  return *histograms[i];
63  }
64  Histogram8(int width, int length, int start=0)
65  {
66  for (int i=0; i<8; ++i)
67  histograms[i] = new stat::Histogram(width, length, start);
68  }
69  void add(Progress16 progress, int data, double weight = 1.0)
70  {
71  const int min = histograms[0]->start();
72  const int max = histograms[0]->start() + histograms[0]->width()*histograms[0]->length();
73  if (data < min || data >= max) {
74  return;
75  }
76  histograms[progress.value()/2]->add(data, weight);
77  }
78 };
79 
81 const int width = 4, length = 20;
84 const int sc_width = 100, sc_length = 16, sc_start = -400;
97 const int sc_length_2d = sc_length+2;
98 const int sc_start_2d = -100;
99 
100 namespace osl
101 {
102  void showLogProb(const stat::Histogram& numerator, const stat::Histogram& denominator)
103  {
104  assert(numerator.width() == denominator.width());
105  assert(numerator.length() == denominator.length());
106  assert(numerator.start() == denominator.start());
107  stat::Histogram logprob(numerator.width(), numerator.length(), numerator.start());
108  for (size_t i=0; i<numerator.length(); ++i) {
109  const double n = numerator.frequency(i);
110  const double d = denominator.frequency(i);
111  const double prob = n / d;
112  logprob.frequency(i) = d >= 15 ? static_cast<int>(-100.0*log(prob)/log(2.0)) : 0;
113  }
114  logprob.show(std::cout);
115  }
116  void showLogProb(const stat::Histogram& numerator, const stat::Histogram& denom1, const stat::Histogram& denom2)
117  {
118  assert(numerator.width() == denom1.width());
119  assert(numerator.length() == denom1.length());
120  assert(numerator.start() == denom1.start());
121  assert(denom1.width() == denom2.width() && denom1.length() == denom2.length() && denom1.start() == denom2.start());
122  stat::Histogram l1(numerator.width(), numerator.length(), numerator.start()),
123  l2(numerator.width(), numerator.length(), numerator.start());
124  for (size_t i=0; i<numerator.length(); ++i) {
125  const double n = numerator.frequency(i);
126  const double d1 = denom1.frequency(i);
127  const double d2 = denom2.frequency(i);
128  const double p1 = n / d1;
129  const double p2 = n / d2;
130  l1.frequency(i) = d1 > 20 ? static_cast<int>(-100.0*log(p1)/log(2.0)) : 0;
131  l2.frequency(i) = d2 > 20 ? static_cast<int>(-100.0*log(p2)/log(2.0)) : 0;
132  }
133  int value=l1.start();
134  for (size_t i=0; i<l1.length(); ++i, value+=l1.width()) {
135  std::cout << std::setw(5) << value << " - "
136  << std::setw(5) << value+(int)l1.width();
137  std::cout << " " << std::setw(8) << l1.frequency(i)
138  << " " << std::setw(8) << l2.frequency(i) << "\n";
139  }
140  }
141  void showLogProb(const Histogram8& numerator, const Histogram8& denom1, const Histogram8& denom2)
142  {
143  assert(numerator[0].width() == denom1[0].width());
144  assert(numerator[0].length() == denom1[0].length());
145  assert(numerator[0].start() == denom1[0].start());
146  assert(denom1[0].width() == denom2[0].width() && denom1[0].length() == denom2[0].length() && denom1[0].start() == denom2[0].start());
147  // for human
148  int value=numerator[0].start();
149  for (size_t i=0; i<numerator[0].length(); ++i, value+=numerator[0].width()) {
150  std::cout << std::setw(4) << value << " - "
151  << std::setw(4) << value+(int)numerator[0].width();
152 
153  for (int p=0; p<8; ++p) {
154  const double n = numerator[p].frequency(i);
155  const double d1 = denom1[p].frequency(i);
156  const double d2 = denom2[p].frequency(i);
157  const double p1 = n / d1;
158  const double p2 = n / d2;
159  const double f1 = n > 20 ? static_cast<int>(-100.0*log(p1)/log(2.0)) : 0;
160  const double f2 = n > 20 ? static_cast<int>(-100.0*log(p2)/log(2.0)) : 0;
161  std::cout << " " << std::setw(5) << f1
162  << " " << std::setw(4) << f2;
163  }
164  std::cout << "\n";
165  }
166  // depth
167  std::cout << "static const osl::CArray2d<int, 8, "
168  << numerator[0].length() << "> xxx_to_depth = {{\n";
169  for (int p=0; p<8; ++p) {
170  std::cout << " { ";
171  for (size_t i=0; i<numerator[0].length(); ++i, value+=numerator[0].width()) {
172  const double n = numerator[p].frequency(i);
173  const double d = denom1[p].frequency(i);
174  const double p = n / d;
175  const double f = n > 20 ? static_cast<int>(-100.0*log(p)/log(2.0)) : 0;
176  std::cout << std::setw(4) << f << ",";
177  if (i % 5 == 4)
178  std::cout << " ";
179  }
180  std::cout << "},\n";
181  }
182  std::cout << "}};\n";
183  // width
184  std::cout << "static const osl::CArray2d<int, 8, "
185  << numerator[0].length() << "> xxx_to_width = {{\n";
186  for (int p=0; p<8; ++p) {
187  std::cout << " { ";
188  for (size_t i=0; i<numerator[0].length(); ++i, value+=numerator[0].width()) {
189  const double n = numerator[p].frequency(i);
190  const double d = denom2[p].frequency(i);
191  const double p = n / d;
192  const double f = n > 20 ? static_cast<int>(-100.0*log(p)/log(2.0)) : 0;
193  std::cout << std::setw(4) << f << ",";
194  if (i % 5 == 4)
195  std::cout << " ";
196  }
197  std::cout << "},\n";
198  }
199  std::cout << "}};\n";
200  }
201 
202  enum Property {
203  All,
217  };
218  size_t find(Property property, const NumEffectState& state, const RatingEnv& e,
219  const RatedMoveVector& moves, Move selected)
220  {
221  if (moves.empty())
222  return 1;
223  size_t i = 0;
224  if (property == TakeBack || property == TakeBack2) {
225  if (! e.history.lastMove().isNormal())
226  return moves.size();
227  for (; i<moves.size(); ++i)
228  if (moves[i].move().to() == e.history.lastMove().to())
229  break;
230  }
231  if (property == TakeBack2) {
232  ++i;
233  for (; i<moves.size(); ++i)
234  if (moves[i].move().to() == e.history.lastMove().to())
235  break;
236  }
237  if (property == NoTakeBack) {
238  if (e.history.lastMove().isNormal()) {
239  for (; i<moves.size(); ++i)
240  if (moves[i].move().to() != e.history.lastMove().to())
241  break;
242  }
243  }
244  if (property == SeePlus || property == SeePlus2) {
245  for (; i<moves.size(); ++i) {
246  if (e.history.lastMove().isNormal() && moves[i].move().to() == e.history.lastMove().to())
247  continue;
248  if (PieceEval::computeDiffAfterMoveForRP(state, moves[i].move()) > 0)
249  break;
250  }
251  }
252  if (property == SeePlus2) {
253  ++i;
254  for (; i<moves.size(); ++i) {
255  if (PieceEval::computeDiffAfterMoveForRP(state, moves[i].move()) > 0)
256  break;
257  }
258  }
259  if (property == SeePlusX) {
260  int num_seeplus=0;
261  for (; i<moves.size(); ++i) {
262  if (e.history.lastMove().isNormal() && moves[i].move().to() == e.history.lastMove().to())
263  continue;
264  if (PieceEval::computeDiffAfterMoveForRP(state, moves[i].move()) > 0) {
265  if (++num_seeplus <= 1) // use 2 for 3rd see+
266  continue;
267  }
268  break;
269  }
270  }
271  if (property == NoSeePlus) {
272  for (; i<moves.size(); ++i) {
273  if (e.history.lastMove().isNormal() && moves[i].move().to() == e.history.lastMove().to())
274  continue;
275  if (PieceEval::computeDiffAfterMoveForRP(state, moves[i].move()) > 0)
276  continue;
277  break;
278  }
279  }
280  return i;
281  }
283  struct TopProb
284  {
289  : selected(sc_width,sc_length+1,200), generated(sc_width,sc_length+1,200),
290  generated_all(sc_width,sc_length+1,200),
291  property(p)
292  {
293  }
294  void add(const NumEffectState& state, const RatingEnv& e,
295  const RatedMoveVector& moves, Move selected)
296  {
297  const size_t i = find(property, state, e, moves, selected);
298  if (i >= moves.size())
299  return;
300  generated_all.add(moves[i].rating());
301  const RatedMove *found = moves.find(selected);
302  if (found && (found - &*moves.begin()) <= (int)i) {
303  if (moves[i].move() == selected)
304  this->selected.add(moves[i].rating());
305  generated.add(moves[i].rating());
306  }
307  }
308  void show()
309  {
310  showLogProb(selected, generated, generated_all);
311  }
312  };
315  {
319  CArray<stat::Variance, sc_length_2d*sc_length_2d> variance;
320  size_t first, last;
321 
322  RatingDiffRange(size_t f, size_t l)
323  : selected(1,sc_length_2d*sc_length_2d), generated(1,sc_length_2d*sc_length_2d), all_generated(1,sc_length_2d*sc_length_2d),
324  first(f), last(l)
325  {
326  }
327 
328  static int index(int score, int diff)
329  {
330  const int score_index = std::max(0, std::min((score - sc_start_2d) / sc_width, sc_length_2d-1));
331  const int diff_index = std::min(diff / sc_width, sc_length_2d-1);
332  assert(diff_index >= 0);
333  return score_index*sc_length_2d + diff_index;
334  }
335  void add(const NumEffectState& state, const RatedMoveVector& moves, Move selected)
336  {
337  if (moves.empty() || state.inCheck())
338  return;
339  const int highest = moves[0].rating();
340  const RatedMove *found = moves.find(selected);
341  if (! found)
342  return; // records may contain illegal move
343  const size_t selected_order = found - &*moves.begin();
344  if (first <= selected_order && selected_order < last) {
345  const int selected_index = index(found->rating(), highest - found->rating());
346  this->selected.add(selected_index);
347  }
348  for (size_t i=first; i<std::min(last,moves.size()); ++i) {
349  const int index = this->index(moves[i].rating(), highest - moves[i].rating());
350  all_generated.add(index);
351  if (i <= selected_order)
352  generated.add(index);
353  variance[index].add(i);
354  }
355  }
356 
357  void show(std::ostream& os)
358  {
359  os << "depth\n";
360  for (int i=0; i<sc_length_2d; ++i) {
361  for (int j=0; j<sc_length_2d; ++j) {
362  double s = selected.frequency(i*sc_length_2d+j);
363  double g = generated.frequency(i*sc_length_2d+j);
364  // os << std::setw(5) << (g > 20 ? s/g : 0.0);
365  os << std::setw(5) << (std::min(s,g) > 20 ? static_cast<int>(-100.0*log(s/g)/log(2.0)) : 0);
366  }
367  os << "\n";
368  }
369  os << "width\n";
370  for (int i=0; i<sc_length_2d; ++i) {
371  for (int j=0; j<sc_length_2d; ++j) {
372  double s = selected.frequency(i*sc_length_2d+j);
373  double a = all_generated.frequency(i*sc_length_2d+j);
374  // os << std::setw(5) << (a > 20 ? s/a : 0.0);
375  os << std::setw(5) << (std::min(s,a) > 20 ? static_cast<int>(-100.0*log(s/a)/log(2.0)) : 0);
376  }
377  os << "\n";
378  }
379  os << "order\n";
380  for (int i=0; i<sc_length_2d; ++i) {
381  for (int j=0; j<sc_length_2d; ++j) {
382  // os << std::setw(5) << std::setprecision(3) << variance[i*sc_length_2d+j].getAverage();
383  os << std::setw(5) << static_cast<int>(variance[i*sc_length_2d+j].getAverage());
384  }
385  os << "\n";
386  }
387  }
388  };
389  struct RatingDiff
390  {
393  r0(1,25), r1(25,800), r_all(1,800)
394  {
395  }
396  void add(const NumEffectState& state, const RatedMoveVector& moves, Move selected)
397  {
398 #ifdef SHOW_SPLIT_RATING
399  r0.add(state, moves, selected);
400  r1.add(state, moves, selected);
401 #endif
402  r_all.add(state, moves, selected);
403  }
404  void show(std::ostream& os)
405  {
406 #if SHOW_SPLIT_RATING
407  r0.show(os);
408  r1.show(os);
409 #endif
410  r_all.show(os);
411  }
412  };
413 }
414 
418 CArray<stat::Variance, 8> top_rating_progress;
419 
420 void test_file(const FeatureSet&, const char *filename);
421 void test_record(const FeatureSet& f,
422  const SimpleState& initial,
423  const osl::stl::vector<osl::Move>& moves);
424 
425 int main(int argc, char **argv)
426 {
427  const char *program_name = argv[0];
428  bool error_flag = false;
429  extern char *optarg;
430  extern int optind;
431 
432  char c;
433  while ((c = getopt(argc, argv, "f:k:n:vh")) != EOF)
434  {
435  switch(c)
436  {
437  case 'f': first_skip = atoi(optarg);
438  break;
439  case 'k': kisen_filename = optarg;
440  break;
441  case 'n': num_kisen = atoi(optarg);
442  break;
443  case 'v': ++verbose;
444  break;
445  default: error_flag = true;
446  }
447  }
448  argc -= optind;
449  argv += optind;
450 
451  if (error_flag || (!kisen_filename && argc < 1))
452  usage(program_name);
453 
456 
457  if (kisen_filename) {
458  KisenFile kisen_file(kisen_filename);
459  KisenIpxFile ipx(kisen_file.ipxFileName());
460  size_t skip = 0;
461  for (size_t i=0; i<num_kisen; i++) {
462  if (ipx.getRating(i, BLACK) < min_rating
463  || ipx.getRating(i, WHITE) < min_rating) {
464  ++skip;
465  continue;
466  }
467  if (i % 128 == 0)
468  std::cerr << '.';
469  test_record(f, kisen_file.getInitialState(), kisen_file.getMoves(i+kisen_start));
470  }
471  }
472 
473  for (int i=0; i<argc; ++i)
474  {
475  if (i % 128 == 0)
476  std::cerr << '.';
477  test_file(f, argv[i]);
478  }
479 
480  std::cout << "\n"
481  << "average moves/position " << moves.getAverage() << "\n"
482  << "average order " << order.getAverage() << "\n"
483  << "average selected score " << selected_score.getAverage() << "\n"
484  << "min selected score " << min_selected.value() << "\n"
485  << "average top score " << top_score.getAverage() << "\n"
486  << "min top score " << min_top.value() << "\n"
487  << "max top score (notakeback) " << max_notakeback.value() << "\n"
488  << "max top score (nocapture) " << max_nocapture.value() << "\n";
489  std::cout << "order to logprob (depth, width)\n";
491  std::cout << "score to logprob (all)\n";
493  std::cout << "relative score to logprob (all)\n";
495  std::cout << "score to logprob (takeback)\n";
497  std::cout << "score to logprob (see+)\n";
499  std::cout << "score to logprob (king_escape)\n";
501  if (verbose) {
502  std::cout << "order to logprob (takeback)\n";
504  std::cout << "order to logprob (seeplus)\n";
506  std::cout << "order to logprob (kingescape)\n";
508  rating_diff.show(std::cout);
509  std::cout << "top move\n";
510  top_prob.show();
511  std::cout << "top move (takeback)\n";
513  std::cout << "top move (2nd takeback)\n";
515  if (verbose > 1) {
516  std::cout << "top move (no takeback)\n";
518  }
519  std::cout << "top move (see+)\n";
521  std::cout << "top move (2nd see+)\n";
523  std::cout << "top move (2nd see+ or no see+)\n";
525  }
526  std::cout << "top rating for each progress8\n";
527  for (size_t i=0; i<top_rating_progress.size(); ++i)
528  std::cout << "progress8 " << i << "\tave. " << std::setprecision(3) << top_rating_progress[i].getAverage()
529  << "\tsigma " << sqrt(top_rating_progress[i].variance()) << "\n";
530 }
531 
532 /* ------------------------------------------------------------------------- */
533 
534 size_t num_positions = 0;
535 void test_position(const FeatureSet& f, Move next_move, Move last_move, const RatingEnv& env,
536  const NumEffectState& state, const eval::ProgressEval& eval)
537 {
538  const bool in_check = state.inCheck();
540  f.generateRating(state, env, 2000, moves);
541 
542  if (moves.empty())
543  return;
544  const RatedMove *p = moves.find(next_move);
545  if (! p)
546  return;
547 
548  rating_diff.add(state, moves, next_move);
549  top_prob.add(state, env, moves, next_move);
550  takeback_topprob.add(state, env, moves, next_move);
551  takeback2_topprob.add(state, env, moves, next_move);
552  no_takeback_topprob.add(state, env, moves, next_move);
553  seeplus_topprob.add(state, env, moves, next_move);
554  seeplus2_topprob.add(state, env, moves, next_move);
555  seeplusx_topprob.add(state, env, moves, next_move);
556 
557  bool notakeback_added = in_check, nocapture_added = in_check;
558  const int highest = moves[0].rating();
559  min_top.add(highest);
560  top_score.add(highest);
561  if (! in_check) {
562  size_t index = find(NoSeePlus, state, env, moves, next_move);
563  if (index < moves.size())
564  top_rating_progress[eval.progress16().value()/2].add(moves[index].rating());
565  }
566  if (! notakeback_added
567  && moves[0].move().to() != last_move.to()) {
568  nocapture_added = true;
569  max_notakeback.add(highest);
570  }
571  if (! nocapture_added
572  && moves[0].move().capturePtype() == PTYPE_EMPTY
573  && ! moves[0].move().isPromotion()) {
574  nocapture_added = true;
575  max_nocapture.add(highest);
576  }
577 
578  const int count = moves.size();
579  const int order = p ? p - &*moves.begin() +1 : count;
580  ::order.add(order);
581  const double selected_weight = 1.0-1.0/(moves.size()-order+1);
582  const double other_weight = 1.0; // -1.0/moves.size();
583 
584  if (in_check) {
585  for (int i=0; i<count; ++i) {
586  if (i < order) {
587  king_escape_histogram.add(moves[i].rating(), other_weight);
589  if (moves[i].move() == next_move) {
590  selected_king_escape.add(moves[i].rating(), selected_weight);
592  }
593  }
595  }
596  return;
597  }
598  selected_histogram.add(env.progress, order, selected_weight);
599  selected_score.add(p->rating());
600  min_selected.add(p->rating());
601  if (p->rating() < -2000) {
602  std::cerr << state << "selected " << *p << "\n" << moves;
603  }
604  for (int i=0; i<order; ++i)
605  moves_histogram.add(env.progress, i, other_weight);
606  for (size_t i=0; i<moves.size(); ++i)
607  all_moves_histogram.add(env.progress, i, other_weight);
608  ::moves.add(count);
609  ++num_positions;
610 
611  int j=0;
612  for (int i=0; i<count; ++i) {
613  if (moves[i].move().to() != last_move.to())
614  continue;
615  if (i < order) {
616  takeback_histogram.add(moves[i].rating(), other_weight);
617  takeback_order.add(j);
618  if (moves[i].move() == next_move) {
619  selected_takeback.add(moves[i].rating(), selected_weight);
621  }
622  }
624  ++j;
625  }
626  j=0;
627  for (int i=0; i<count; ++i) {
628  if (moves[i].move().to() == last_move.to())
629  continue;
630  if (! (moves[i].move().capturePtype() != PTYPE_EMPTY
631  || moves[i].move().isPromotion())
632  || PieceEval::computeDiffAfterMoveForRP(state, moves[i].move()) <= 0)
633  continue;
634  if (i<order) {
635  seeplus_histogram.add(moves[i].rating(), other_weight);
636  seeplus_order.add(j);
637  if (moves[i].move() == next_move) {
638  selected_seeplus.add(moves[i].rating(), selected_weight);
640  }
641  }
643  ++j;
644  }
645 
646  for (int i=0; i<order; ++i) {
647  score_histogram.add(env.progress, moves[i].rating(), other_weight);
648  if (moves[i].move() == next_move)
649  selected_score_histogram.add(env.progress, moves[i].rating(), selected_weight);
650  }
651  for (size_t i=0; i<moves.size(); ++i) {
652  all_score_histogram.add(env.progress, moves[i].rating(), other_weight);
653  if (! notakeback_added && moves[i].move().to() != last_move.to()) {
654  notakeback_added = true;
655  max_notakeback.add(moves[i].rating());
656  }
657  if (! nocapture_added && moves[i].move().capturePtype() == PTYPE_EMPTY
658  && ! moves[i].move().isPromotion()) {
659  nocapture_added = true;
660  max_nocapture.add(moves[i].rating());
661  }
662  }
663  if (moves[0].move() != next_move) {
664  const int top_score = moves[0].rating();
665  for (int i=1; i<order; ++i) {
666  rscore_histogram.add(env.progress, top_score - moves[i].rating(), other_weight);
667  if (moves[i].move() == next_move)
668  rselected_score_histogram.add(env.progress, top_score - moves[i].rating(), selected_weight);
669  }
670  for (size_t i=1; i<moves.size(); ++i) {
671  rall_score_histogram.add(env.progress, top_score - moves[i].rating(), other_weight);
672  }
673  }
674 }
675 
676 void test_record(const FeatureSet& f,
677  const SimpleState& initial,
678  const osl::stl::vector<osl::Move>& moves)
679 {
680  NumEffectState state(initial);
681 
682  RatingEnv env;
683  env.make(state);
684  eval::ProgressEval eval(state);
685  for (size_t i=0; i<moves.size(); ++i) {
686  if (state.inCheck(alt(state.turn())))
687  break; // illegal
688 
689  const Move move = moves[i];
690  assert(state.isValidMove(move));
691  if (i >= first_skip) {
692  test_position(f, moves[i], (i>0 ? moves[i-1] : Move::PASS(alt(moves[i].player()))),
693  env, state, eval);
694  }
695  state.makeMove(move);
696  eval.update(state, move);
697  env.update(state, move);
698  }
699 }
700 
701 void test_file(const FeatureSet& f, const char *filename)
702 {
703  Record rec;
704  try {
705  rec = CsaFile(filename).getRecord();
706  }
707  catch (CsaIOError& e) {
708  std::cerr << "skip " << filename <<"\n";
709  std::cerr << e.what() << "\n";
710  return;
711  }
712  catch (...) {
713  throw;
714  }
715  test_record(f, rec.getInitialState(), rec.getMoves());
716 }
717 
718 /* ------------------------------------------------------------------------- */
719 // ;;; Local Variables:
720 // ;;; mode:c++
721 // ;;; c-basic-offset:2
722 // ;;; End: