initial tests

test/build

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+dir="$(dirname "$0")"
+. "$dir"/../cflags
+
+$cc $cflags ${@:--DOPPAI_IMPLEMENTATION} \
+  test.c $ldflags -o oppai_test
+

test/build.bat

@@ -0,0 +1,16 @@
+@echo off
+
+set flags="%*"
+IF "%1"=="" (
+  set flags=-DOPPAI_IMPLEMENTATION
+)
+
+del oppai_test.exe >nul 2>&1
+del test.obj >nul 2>&1
+cl  -D_CRT_SECURE_NO_WARNINGS=1 ^
+  -DNOMINMAX=1 ^
+  -O2 -nologo -MT -Gm- -GR- -EHsc -W4 ^
+  %flags% ^
+  test.c ^
+  -Feoppai_test.exe ^
+  || EXIT /B 1

test/download_suite

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+dir="$(dirname "$0")"
+if command -v realpath 2>&1 >/dev/null; then
+  wdir="$(realpath "$dir")"
+else
+  wdir="$dir"
+fi
+olddir="$(pwd)"
+cd "$wdir" || exit $?
+
+url="$(cat ./suite_url)"
+
+if [ $(find test_suite 2>/dev/null | tail -n +2 | wc -l) = "0" ]; then
+  curl -LO "$url" || exit $?
+  tar xf "$(basename $url)" || exit $?
+else
+  echo "using existing test_suite"
+fi
+
+cd "$olddir"

test/download_suite.ps1

@@ -0,0 +1,19 @@
+# you must allow script execution by running
+# 'Set-ExecutionPolicy RemoteSigned' in an admin powershell
+# 7zip is also required (choco install 7zip and add it to path)
+
+$url = Get-Content .\suite_url -Raw
+$dir = Split-Path -Parent $MyInvocation.MyCommand.Definition
+Push-Location "$dir"
+
+if ((Test-Path .\test_suite) -and (Get-ChildItem .\test_suite | Measure-Object).Count -gt 0) {
+  Write-Host "using existing test_suite"
+} else {
+  # my windows 7 install doesn't support Tls3
+  [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+  (New-Object System.Net.WebClient).DownloadFile($url, "$dir\test_suite.tar.gz")
+  &7z x .\test_suite.tar.gz
+  &7z x .\test_suite.tar
+}
+
+Pop-Location

test/download_suite.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# very rough script that downloads unique maps from test_suite.json that
+# gensuite.py generates
+
+import sys
+import json
+
+try:
+    import httplib
+except ImportError:
+    import http.client as httplib
+
+try:
+    import urllib
+except ImportError:
+    import urllib.parse as urllib
+
+osu = httplib.HTTPSConnection('osu.ppy.sh')
+
+def osu_get(path):
+    while True:
+        try:
+            osu.request('GET', path)
+            r = osu.getresponse()
+
+            raw = bytes()
+
+            while True:
+                try:
+                    raw += r.read()
+                    break
+                except httplib.IncompleteRead as e:
+                    raw += e.partial
+
+            return raw
+
+        except (httplib.HTTPException, ValueError) as e:
+            sys.stderr.write('%s\n' % (traceback.format_exc()))
+
+            # prevents exceptions on next request if the
+            # response wasn't previously read due to errors
+            try:
+                osu.getresponse().read()
+            except httplib.HTTPException:
+                pass
+
+            time.sleep(5)
+
+
+if len(sys.argv) != 2:
+    sys.stderr.write('usage: %s test_suite.json\n' % sys.argv[0])
+    sys.exit(1)
+
+with open(sys.argv[1], 'r') as f:
+    scores = json.loads(f.read())
+
+unique_maps = set([s['beatmap_id'] for m in [0, 1] for s in scores[m]])
+i = 1
+
+for b in unique_maps:
+    sys.stderr.write(
+        "[%.02f%% - %d/%d] %s" % (i / float(len(unique_maps)) * 100, i,
+            len(unique_maps), b)
+    )
+    i += 1
+
+    # TODO: tmp file and rename
+    try:
+        with open(b + '.osu', 'r') as f:
+            sys.stderr.write(' (already exists)\n')
+            continue
+    except FileNotFoundError:
+        pass
+
+    sys.stderr.write('\n')
+
+    with open(b + '.osu', 'wb') as f:
+        f.write(osu_get('/osu/' + b))

test/gentest.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import time
+import json
+import traceback
+import argparse
+import hashlib
+
+if sys.version_info[0] < 3:
+  # hack to force utf-8
+  reload(sys)
+  sys.setdefaultencoding('utf-8')
+
+try:
+  import httplib
+except ImportError:
+  import http.client as httplib
+
+try:
+  import urllib
+except ImportError:
+  import urllib.parse as urllib
+
+# -------------------------------------------------------------------------
+
+parser = argparse.ArgumentParser(
+  description = (
+    'generates the oppai test suite. outputs c++ code to ' +
+    'stdout and the json dump to a file.'
+  )
+)
+
+parser.add_argument(
+  '-key',
+  default = None,
+  help = (
+    'osu! api key. required if -input-file is not present. ' +
+    'can also be specified through the OSU_API_KEY ' +
+    'environment variable'
+  )
+)
+
+parser.add_argument(
+  '-output-file',
+  default = 'test_suite.json',
+  help = 'dumps json to this file'
+)
+
+parser.add_argument(
+  '-input-file',
+  default = None,
+  help = (
+    'loads test suite from this json file instead of '
+    'fetching it from osu api. if set to "-", json will be '
+    'read from standard input'
+  )
+)
+
+args = parser.parse_args()
+
+if args.key == None and 'OSU_API_KEY' in os.environ:
+  args.key = os.environ['OSU_API_KEY']
+
+# -------------------------------------------------------------------------
+
+osu_treset = time.time() + 60
+osu_ncalls = 0
+
+def osu_get(conn, endpoint, paramsdict=None):
+  # GETs /api/endpoint?paramsdict&k=args.key from conn
+  # return json object, exits process on api errors
+  global osu_treset, osu_ncalls, args
+
+  sys.stderr.write('%s %s\n' % (endpoint, str(paramsdict)))
+
+  paramsdict['k'] = args.key
+  path = '/api/%s?%s' % (endpoint, urllib.urlencode(paramsdict))
+
+  while True:
+    while True:
+      if time.time() >= osu_treset:
+        osu_ncalls = 0
+        osu_treset = time.time() + 60
+        sys.stderr.write('\napi ready\n')
+
+      if osu_ncalls < 60:
+        break
+      else:
+        sys.stderr.write('waiting for api cooldown...\r')
+        time.sleep(1)
+
+
+    try:
+      conn.request('GET', path)
+      osu_ncalls += 1
+      r = conn.getresponse()
+
+      raw = ''
+
+      while True:
+        try:
+          raw += r.read()
+          break
+        except httplib.IncompleteRead as e:
+          raw += e.partial
+
+      j = json.loads(raw)
+
+      if 'error' in j:
+        sys.stderr.write('%s\n' % j['error'])
+        sys.exit(1)
+
+      return j
+
+    except (httplib.HTTPException, ValueError) as e:
+      sys.stderr.write('%s\n' % (traceback.format_exc()))
+
+      try:
+        # prevents exceptions on next request if the
+        # response wasn't previously read due to errors
+        conn.getresponse().read()
+
+      except httplib.HTTPException:
+        pass
+
+      time.sleep(5)
+
+
+def gen_modstr(bitmask):
+  # generates c++ code for a mod combination's bitmask
+  mods = []
+
+  allmods = {
+    (1<< 0, 'nf'), (1<< 1, 'ez'), (1<< 2, 'td'), (1<< 3, 'hd'),
+    (1<< 4, 'hr'), (1<< 6, 'dt'), (1<< 8, 'ht'),
+    (1<< 9, 'nc'), (1<<10, 'fl'), (1<<12, 'so')
+  }
+
+  for bit, string in allmods:
+    if bitmask & bit != 0:
+      mods.append(string)
+
+  if len(mods) == 0:
+    return 'nomod'
+
+  return ' | '.join(mods)
+
+# -------------------------------------------------------------------------
+
+if args.key == None:
+  sys.stderr.write(
+    'please set OSU_API_KEY or pass it as a parameter\n'
+  )
+  sys.exit(1)
+
+
+scores = []
+
+top_players = [
+  [ 124493, 4787150, 2558286, 1777162, 2831793, 50265 ],
+  [ 3174184, 8276884, 5991961, 2774767 ]
+]
+
+if args.input_file == None:
+  # fetch a fresh test suite from osu api
+  osu = httplib.HTTPSConnection('osu.ppy.sh')
+
+  for m in [0, 1]:
+    for u in top_players[m]:
+      params = { 'u': u, 'limit': 100, 'type': 'id', 'm': m }
+      batch = osu_get(osu, 'get_user_best', params)
+      for s in batch:
+        s['mode'] = m
+      scores += batch
+
+  # temporarily removed, not all std scores are recalc'd
+  #params = { 'm': 0, 'since': '2015-11-26' }
+  #maps = osu_get(osu, 'get_beatmaps', params)
+
+  # no taiko converts here because as explained below, tiny float
+  # errors can lead to completely broken conversions
+  for mode in [1]:
+    params = { 'm': mode, 'since': '2015-11-26' }
+    maps = osu_get(osu, 'get_beatmaps', params)
+
+    for m in maps:
+      params = { 'b': m['beatmap_id'], 'm': mode }
+      map_scores = osu_get(osu, 'get_scores', params)
+
+      if len(map_scores) == 0:
+        sys.stderr.write('W: map has no scores???\n')
+        continue
+
+      # note: api also returns qualified and loved, so ignore
+      # maps that don't have pp in rankings
+      if not 'pp' in map_scores[0] or map_scores[0]['pp'] is None:
+        sys.stderr.write('W: ignoring loved/qualified map\n')
+        continue
+
+      for s in map_scores:
+        s['beatmap_id'] = m['beatmap_id']
+        s['mode'] = mode
+
+      scores += map_scores
+
+
+  with open(args.output_file, 'w+') as f:
+    f.write(json.dumps(scores))
+
+else:
+  # load existing test suite from json file
+  with open(args.input_file, 'r') as f:
+    scores = json.loads(f.read())
+  # sort by mode by map
+  scores.sort(key=lambda x: int(x['beatmap_id'])<<32|x['mode'],
+    reverse=True)
+
+
+print('/* this code was automatically generated by gentest.py */')
+print('')
+
+# make code a little nicer by shortening mods
+allmods = {
+  'nf', 'ez', 'td', 'hd', 'hr', 'dt', 'ht', 'nc', 'fl', 'so', 'nomod'
+}
+
+for mod in allmods:
+  print('#define %s MODS_%s' % (mod, mod.upper()))
+
+print('''
+typedef struct {
+  int mode;
+  int id;
+  int max_combo;
+  int n300, n100, n50, nmiss;
+  int mods;
+  double pp;
+} score_t;
+
+score_t suite[] = {''')
+
+seen_hashes = []
+
+for s in scores:
+  # due to tiny floating point errors, maps can even double
+  # in combo and not even lazer gets this right, taiko converts are hell
+  # so I'm just gonna exclude them
+  if s['mode'] == 1:
+    is_convert = False
+    with open('test_suite/'+s['beatmap_id']+'.osu') as f:
+      for line in f:
+        split = line.split(':')
+        if len(split) >= 2 and split[0] == 'Mode' and int(split[1]) == 0:
+          is_convert = True
+          break
+
+    if is_convert:
+        continue
+
+    # some taiko maps ignore combo scaling for no apparent reason
+    # so i will only include full combos for now
+    if int(s['countmiss']) != 0:
+        continue
+
+  if s['pp'] is None:
+    continue
+
+  # why is every value returned by osu api a string?
+  line = (
+    '  { %d, %s, %s, %s, %s, %s, %s, %s, %s },' %
+    (
+      s['mode'], s['beatmap_id'], s['maxcombo'], s['count300'],
+      s['count100'], s['count50'], s['countmiss'],
+      gen_modstr(int(s['enabled_mods'])), s['pp']
+    )
+  )
+
+  # don't include identical scores by different people
+  s = hashlib.sha1(line).digest()
+  if s in seen_hashes:
+    continue
+
+  print(line)
+  seen_hashes.append(s)
+
+print('};\n')
+
+for mod in allmods:
+  print('#undef %s' % (mod))
+

test/pack_suite

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+dir="$(dirname "$0")"
+wdir="$(realpath "$dir")"
+
+olddir="$(pwd)"
+cd "$wdir" || exit $?
+tar -zcvf test_suite_$(date +%F).tar.gz test_suite/*.osu
+res=$?
+cd "$olddir"
+exit $res

test/suite_url

@@ -0,0 +1 @@
+https://github.com/Francesco149/oppai-ng/releases/download/2.3.2/test_suite_2019-02-19.tar.gz

test/test.c

@@ -0,0 +1,119 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../oppai.c"
+#include "test_suite.c" /* defines suite */
+
+#define ERROR_MARGIN 0.02f /* pp can be off by +- 2% */
+
+/*
+ * margin is actually
+ * - 3x for < 100pp
+ * - 2x for 100-200pp
+ * - 1.5x for 200-300pp
+ */
+
+void print_score(score_t* s) {
+  char mods_str_buf[20];
+  char* mods_str = mods_str_buf;
+
+  strcpy(mods_str, "nomod");
+
+#define m(mod) \
+  if (s->mods & MODS_##mod) { \
+    mods_str += sprintf(mods_str, #mod); \
+  } \
+
+  m(HR) m(NC) m(HT) m(SO) m(NF) m(EZ) m(DT) m(FL) m(HD)
+#undef m
+
+  printf("m=%d %u +%s %dx %dx300 %dx100 %dx50 %dxmiss %g pp\n",
+    s->mode, s->id, mods_str_buf, s->max_combo, s->n300, s->n100,
+    s->n50, s->nmiss, s->pp);
+}
+
+int main() {
+  char fname_buf[4096];
+  char* fname = fname_buf;
+  int i, n = (int)(sizeof(suite) / sizeof(suite[0]));
+  float max_err[2] = { 0, 0 };
+  int max_err_map[2] = { 0, 0 };
+  float avg_err[2] = { 0, 0 };
+  int count[2] = { 0, 0 };
+  float error = 0;
+  float error_percent = 0;
+  ezpp_t ez = ezpp_new();
+  int err;
+  int last_id = 0;
+
+  fname += sprintf(fname, "test_suite/");
+
+  for (i = 0; i < n; ++i) {
+    score_t* s = &suite[i];
+    float margin;
+    float pptotal;
+
+    print_score(s);
+    if (s->id != last_id) {
+      sprintf(fname, "%u.osu", s->id);
+      last_id = s->id;
+      /* force reparse and don't reuse prev map ar/cs/od/hp */
+      ezpp_set_base_cs(ez, -1);
+      ezpp_set_base_ar(ez, -1);
+      ezpp_set_base_od(ez, -1);
+      ezpp_set_base_hp(ez, -1);
+    }
+    ezpp_set_mods(ez, s->mods);
+    ezpp_set_accuracy(ez, s->n100, s->n50);
+    ezpp_set_nmiss(ez, s->nmiss);
+    ezpp_set_combo(ez, s->max_combo);
+    ezpp_set_mode_override(ez, s->mode);
+    err = ezpp(ez, fname_buf);
+    if (err < 0) {
+      printf("%s\n", errstr(err));
+      exit(1);
+    }
+    pptotal = ezpp_pp(ez);
+    ++count[s->mode];
+
+    margin = (float)s->pp * ERROR_MARGIN;
+    if (s->pp < 100) {
+      margin *= 3;
+    } else if (s->pp < 200) {
+      margin *= 2;
+    } else if (s->pp < 300) {
+      margin *= 1.5f;
+    }
+
+    error = (float)fabs(pptotal - (float)s->pp);
+    error_percent = error / (float)s->pp;
+    avg_err[s->mode] += error_percent;
+    if (error_percent > max_err[s->mode]) {
+      max_err[s->mode] = error_percent;
+      max_err_map[s->mode] = s->id;
+    }
+
+    if (error >= margin) {
+      printf("failed test: got %g pp, expected %g\n", pptotal, s->pp);
+      exit(1);
+    }
+  }
+
+  for (i = 0; i < 2; ++i) {
+    switch (i) {
+      case MODE_STD: puts("osu"); break;
+      case MODE_TAIKO: puts("taiko"); break;
+    }
+    avg_err[i] /= count[i];
+    printf("%d scores\n", count[i]);
+    printf("avg err %f\n", avg_err[i]);
+    printf("max err %f on %d\n", max_err[i], max_err_map[i]);
+  }
+
+  ezpp_free(ez);
+
+  return 0;
+}
+