Introduce skipping based on the likelihood of the game outcome

this intends to skip (or more correctly weight) data points that are possibly incorrectly evaluated, i.e. retain data that are more likely to be correct.

Was used to train two recent SF nets:
official-stockfish/Stockfish#3816
official-stockfish/Stockfish#3808

--no-wld-fen-skipping option can be used to disable the default

Author: vondele

lib/nnue_training_data_formats.h

@@ -26,11 +26,13 @@ THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #pragma once
 
+#include <algorithm>
 #include <cstdio>
 #include <cassert>
 #include <string>
 #include <string_view>
 #include <vector>
+#include <cmath>
 #include <memory>
 #include <fstream>
 #include <cstring>
@@ -6850,6 +6852,41 @@ namespace binpack
                    pos.pieceAt(move.to).color() != pos.pieceAt(move.from).color(); // Exclude castling
         }
 
+        // The win rate model returns the probability (per mille) of winning given an eval
+        // and a game-ply. The model fits rather accurately the LTC fishtest statistics.
+        std::tuple<double, double, double> win_rate_model() const {
+
+           // The model captures only up to 240 plies, so limit input (and rescale)
+           double m = std::min(240, int(ply)) / 64.0;
+
+           // Coefficients of a 3rd order polynomial fit based on fishtest data
+           // for two parameters needed to transform eval to the argument of a
+           // logistic function.
+           double as[] = {-3.68389304,  30.07065921, -60.52878723, 149.53378557};
+           double bs[] = {-2.0181857,   15.85685038, -29.83452023,  47.59078827};
+           double a = (((as[0] * m + as[1]) * m + as[2]) * m) + as[3];
+           double b = (((bs[0] * m + bs[1]) * m + bs[2]) * m) + bs[3];
+
+           // Transform eval to centipawns with limited range
+           double x = std::clamp(double(100 * score) / 208, -2000.0, 2000.0);
+           double w = 1.0 / (1 + std::exp((a - x) / b));
+           double l = 1.0 / (1 + std::exp((a + x) / b));
+           double d = 1.0 - w - l;
+
+           // Return win, loss, draw rate in per mille (rounded to nearest)
+           return std::make_tuple(w, l, d);
+        }
+
+        // how likely is end-game result with the current score?
+        double score_result_prob() const {
+           auto [w, l, d] = win_rate_model();
+           if (result > 0)
+               return w;
+           if (result < 0)
+               return l;
+           return d;
+        }
+
         [[nodiscard]] bool isInCheck() const
         {
             return pos.isCheck();

nnue_dataset.py

@@ -67,10 +67,10 @@ def get_fens(self):
         return strings
 
 FenBatchPtr = ctypes.POINTER(FenBatch)
-
+# EXPORT FenBatchStream* CDECL create_fen_batch_stream(int concurrency, const char* filename, int batch_size, bool cyclic, bool filtered, int random_fen_skipping, bool wld_filtered)
 create_fen_batch_stream = dll.create_fen_batch_stream
 create_fen_batch_stream.restype = ctypes.c_void_p
-create_fen_batch_stream.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_bool, ctypes.c_bool, ctypes.c_int]
+create_fen_batch_stream.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_bool, ctypes.c_bool, ctypes.c_int, ctypes.c_bool]
 destroy_fen_batch_stream = dll.destroy_fen_batch_stream
 destroy_fen_batch_stream.argtypes = [ctypes.c_void_p]
 
@@ -87,19 +87,21 @@ def __init__(
         num_workers,
         batch_size=None,
         filtered=False,
-        random_fen_skipping=0):
+        random_fen_skipping=0,
+        wld_filtered=False):
 
         self.filename = filename.encode('utf-8')
         self.cyclic = cyclic
         self.num_workers = num_workers
         self.batch_size = batch_size
         self.filtered = filtered
+        self.wld_filtered = wld_filtered
         self.random_fen_skipping = random_fen_skipping
 
         if batch_size:
-            self.stream = create_fen_batch_stream(self.num_workers, self.filename, batch_size, cyclic, filtered, random_fen_skipping)
+            self.stream = create_fen_batch_stream(self.num_workers, self.filename, batch_size, cyclic, filtered, random_fen_skipping, wld_filtered)
         else:
-            self.stream = create_fen_batch_stream(self.num_workers, self.filename, cyclic, filtered, random_fen_skipping)
+            self.stream = create_fen_batch_stream(self.num_workers, self.filename, cyclic, filtered, random_fen_skipping, wld_filtered)
 
     def __iter__(self):
         return self
@@ -131,6 +133,7 @@ def __init__(
         batch_size=None,
         filtered=False,
         random_fen_skipping=0,
+        wld_filtered=False,
         device='cpu'):
 
         self.feature_set = feature_set.encode('utf-8')
@@ -143,13 +146,14 @@ def __init__(
         self.num_workers = num_workers
         self.batch_size = batch_size
         self.filtered = filtered
+        self.wld_filtered = wld_filtered
         self.random_fen_skipping = random_fen_skipping
         self.device = device
 
         if batch_size:
-            self.stream = self.create_stream(self.feature_set, self.num_workers, self.filename, batch_size, cyclic, filtered, random_fen_skipping)
+            self.stream = self.create_stream(self.feature_set, self.num_workers, self.filename, batch_size, cyclic, filtered, random_fen_skipping, wld_filtered)
         else:
-            self.stream = self.create_stream(self.feature_set, self.num_workers, self.filename, cyclic, filtered, random_fen_skipping)
+            self.stream = self.create_stream(self.feature_set, self.num_workers, self.filename, cyclic, filtered, random_fen_skipping, wld_filtered)
 
     def __iter__(self):
         return self
@@ -167,9 +171,11 @@ def __next__(self):
     def __del__(self):
         self.destroy_stream(self.stream)
 
+#     EXPORT Stream<SparseBatch>* CDECL create_sparse_batch_stream(const char* feature_set_c, int concurrency, const char* filename, int batch_size, bool cyclic,
+#                                                                  bool filtered, int random_fen_skipping, bool wld_filtered)
 create_sparse_batch_stream = dll.create_sparse_batch_stream
 create_sparse_batch_stream.restype = ctypes.c_void_p
-create_sparse_batch_stream.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_bool, ctypes.c_bool, ctypes.c_int]
+create_sparse_batch_stream.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_bool, ctypes.c_bool, ctypes.c_int, ctypes.c_bool]
 destroy_sparse_batch_stream = dll.destroy_sparse_batch_stream
 destroy_sparse_batch_stream.argtypes = [ctypes.c_void_p]
 
@@ -198,7 +204,7 @@ def make_sparse_batch_from_fens(feature_set, fens, scores, plies, results):
     return b
 
 class SparseBatchProvider(TrainingDataProvider):
-    def __init__(self, feature_set, filename, batch_size, cyclic=True, num_workers=1, filtered=False, random_fen_skipping=0, device='cpu'):
+    def __init__(self, feature_set, filename, batch_size, cyclic=True, num_workers=1, filtered=False, random_fen_skipping=0, wld_filtered=False, device='cpu'):
         super(SparseBatchProvider, self).__init__(
             feature_set,
             create_sparse_batch_stream,
@@ -211,10 +217,11 @@ def __init__(self, feature_set, filename, batch_size, cyclic=True, num_workers=1
             batch_size,
             filtered,
             random_fen_skipping,
+            wld_filtered,
             device)
 
 class SparseBatchDataset(torch.utils.data.IterableDataset):
-  def __init__(self, feature_set, filename, batch_size, cyclic=True, num_workers=1, filtered=False, random_fen_skipping=0, device='cpu'):
+  def __init__(self, feature_set, filename, batch_size, cyclic=True, num_workers=1, filtered=False, random_fen_skipping=0, wld_filtered=False, device='cpu'):
     super(SparseBatchDataset).__init__()
     self.feature_set = feature_set
     self.filename = filename
@@ -223,10 +230,12 @@ def __init__(self, feature_set, filename, batch_size, cyclic=True, num_workers=1
     self.num_workers = num_workers
     self.filtered = filtered
     self.random_fen_skipping = random_fen_skipping
+    self.wld_filtered = wld_filtered
     self.device = device
 
   def __iter__(self):
-    return SparseBatchProvider(self.feature_set, self.filename, self.batch_size, cyclic=self.cyclic, num_workers=self.num_workers, filtered=self.filtered, random_fen_skipping=self.random_fen_skipping, device=self.device)
+    return SparseBatchProvider(self.feature_set, self.filename, self.batch_size, cyclic=self.cyclic, num_workers=self.num_workers,
+                               filtered=self.filtered, random_fen_skipping=self.random_fen_skipping, wld_filtered=self.wld_filtered, device=self.device)
 
 class FixedNumBatchesDataset(Dataset):
   def __init__(self, dataset, num_batches):

train.py

@@ -9,15 +9,15 @@
 from pytorch_lightning import loggers as pl_loggers
 from torch.utils.data import DataLoader, Dataset
 
-def make_data_loaders(train_filename, val_filename, feature_set, num_workers, batch_size, filtered, random_fen_skipping, main_device):
+def make_data_loaders(train_filename, val_filename, feature_set, num_workers, batch_size, filtered, random_fen_skipping, wld_filtered, main_device):
   # Epoch and validation sizes are arbitrary
   epoch_size = 100000000
   val_size = 1000000
   features_name = feature_set.name
   train_infinite = nnue_dataset.SparseBatchDataset(features_name, train_filename, batch_size, num_workers=num_workers,
-                                                   filtered=filtered, random_fen_skipping=random_fen_skipping, device=main_device)
+                                                   filtered=filtered, random_fen_skipping=random_fen_skipping, wld_filtered=wld_filtered, device=main_device)
   val_infinite = nnue_dataset.SparseBatchDataset(features_name, val_filename, batch_size, filtered=filtered,
-                                                   random_fen_skipping=random_fen_skipping, device=main_device)
+                                                   random_fen_skipping=random_fen_skipping, wld_filtered=wld_filtered, device=main_device)
   # num_workers has to be 0 for sparse, and 1 for dense
   # it currently cannot work in parallel mode but it shouldn't need to
   train = DataLoader(nnue_dataset.FixedNumBatchesDataset(train_infinite, (epoch_size + batch_size - 1) // batch_size), batch_size=None, batch_sampler=None)
@@ -36,6 +36,7 @@ def main():
   parser.add_argument("--seed", default=42, type=int, dest='seed', help="torch seed to use.")
   parser.add_argument("--smart-fen-skipping", action='store_true', dest='smart_fen_skipping_deprecated', help="If enabled positions that are bad training targets will be skipped during loading. Default: True, kept for backwards compatibility. This option is ignored")
   parser.add_argument("--no-smart-fen-skipping", action='store_true', dest='no_smart_fen_skipping', help="If used then no smart fen skipping will be done. By default smart fen skipping is done.")
+  parser.add_argument("--no-wld-fen-skipping", action='store_true', dest='no_wld_fen_skipping', help="If used then no wld fen skipping will be done. By default wld fen skipping is done.")
   parser.add_argument("--random-fen-skipping", default=3, type=int, dest='random_fen_skipping', help="skip fens randomly on average random_fen_skipping before using one.")
   parser.add_argument("--resume-from-model", dest='resume_from_model', help="Initializes training using the weights from the given .pt model")
   features.add_argparse_args(parser)
@@ -71,6 +72,7 @@ def main():
   print('Using batch size {}'.format(batch_size))
 
   print('Smart fen skipping: {}'.format(not args.no_smart_fen_skipping))
+  print('WLD fen skipping: {}'.format(not args.no_wld_fen_skipping))
   print('Random fen skipping: {}'.format(args.random_fen_skipping))
 
   if args.threads > 0:
@@ -89,7 +91,7 @@ def main():
   nnue.to(device=main_device)
 
   print('Using c++ data loader')
-  train, val = make_data_loaders(args.train, args.val, feature_set, args.num_workers, batch_size, not args.no_smart_fen_skipping, args.random_fen_skipping, main_device)
+  train, val = make_data_loaders(args.train, args.val, feature_set, args.num_workers, batch_size, not args.no_smart_fen_skipping, args.random_fen_skipping, not args.no_wld_fen_skipping, main_device)
 
   trainer.fit(nnue, train, val)
 

training_data_loader.cpp

@@ -808,16 +808,23 @@ struct FenBatchStream : Stream<FenBatch>
     std::vector<std::thread> m_workers;
 };
 
-std::function<bool(const TrainingDataEntry&)> make_skip_predicate(bool filtered, int random_fen_skipping)
+std::function<bool(const TrainingDataEntry&)> make_skip_predicate(bool filtered, int random_fen_skipping, bool wld_filtered)
 {
-    if (filtered || random_fen_skipping)
+    if (filtered || random_fen_skipping || wld_filtered)
     {
         return [
             random_fen_skipping,
             prob = double(random_fen_skipping) / (random_fen_skipping + 1),
-            filtered
+            filtered,
+            wld_filtered
             ](const TrainingDataEntry& e){
 
+            auto do_wld_skip = [&]() {
+                std::bernoulli_distribution distrib(1.0 - e.score_result_prob());
+                auto& prng = rng::get_thread_local_rng();
+                return distrib(prng);
+            };
+
             auto do_skip = [&]() {
                 std::bernoulli_distribution distrib(prob);
                 auto& prng = rng::get_thread_local_rng();
@@ -829,7 +836,7 @@ std::function<bool(const TrainingDataEntry&)> make_skip_predicate(bool filtered,
             };
 
             static thread_local std::mt19937 gen(std::random_device{}());
-            return (random_fen_skipping && do_skip()) || (filtered && do_filter());
+            return (random_fen_skipping && do_skip()) || (filtered && do_filter()) || (wld_filtered && do_wld_skip());
         };
     }
 
@@ -896,9 +903,10 @@ extern "C" {
         return nullptr;
     }
 
-    EXPORT FenBatchStream* CDECL create_fen_batch_stream(int concurrency, const char* filename, int batch_size, bool cyclic, bool filtered, int random_fen_skipping)
+    // changing the signature needs matching changes in nnue_dataset.py
+    EXPORT FenBatchStream* CDECL create_fen_batch_stream(int concurrency, const char* filename, int batch_size, bool cyclic, bool filtered, int random_fen_skipping, bool wld_filtered)
     {
-        auto skipPredicate = make_skip_predicate(filtered, random_fen_skipping);
+        auto skipPredicate = make_skip_predicate(filtered, random_fen_skipping, wld_filtered);
 
         return new FenBatchStream(concurrency, filename, batch_size, cyclic, skipPredicate);
     }
@@ -908,9 +916,11 @@ extern "C" {
         delete stream;
     }
 
-    EXPORT Stream<SparseBatch>* CDECL create_sparse_batch_stream(const char* feature_set_c, int concurrency, const char* filename, int batch_size, bool cyclic, bool filtered, int random_fen_skipping)
+    // changing the signature needs matching changes in nnue_dataset.py
+    EXPORT Stream<SparseBatch>* CDECL create_sparse_batch_stream(const char* feature_set_c, int concurrency, const char* filename, int batch_size, bool cyclic,
+                                                                 bool filtered, int random_fen_skipping, bool wld_filtered)
     {
-        auto skipPredicate = make_skip_predicate(filtered, random_fen_skipping);
+        auto skipPredicate = make_skip_predicate(filtered, random_fen_skipping, wld_filtered);
 
         std::string_view feature_set(feature_set_c);
         if (feature_set == "HalfKP")
@@ -981,7 +991,7 @@ extern "C" {
 
 int main()
 {
-    auto stream = create_sparse_batch_stream("HalfKP", 4, "10m_d3_q_2.binpack", 8192, true, false, 0);
+    auto stream = create_sparse_batch_stream("HalfKP", 4, "10m_d3_q_2.binpack", 8192, true, false, 0, false);
     auto t0 = std::chrono::high_resolution_clock::now();
     for (int i = 0; i < 1000; ++i)
     {