adjusted in- and output of pp-gen

.gitignore

@@ -4,5 +4,4 @@ Cargo.lock
 output
 /pp-gen/.env
 /pp-gen/target
-/pp-gen/input/*
+/pp-gen/output.json
-/pp-gen/output/*

pp-gen/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2021"
 [dependencies]
 dotenv = { version = "0.15", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["std"] }
+pbr = { version = "1.0" }
+rosu-pp = { path = "..", features = ["async_tokio"] }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0", default-features = false, features = ["std"] }
 tokio = { version = "1.0", default-features = false, features = ["fs", "io-util", "process", "rt-multi-thread"] }

pp-gen/README.md

@@ -4,16 +4,15 @@ Small script to calculate difficulty and performance points related values for a
 
 ### How to use
 
-- Create 4 files in `/input/`: `osu.csv`, `taiko.csv`, `catch.csv`, and `mania.csv`. For all beatmaps that you want to calculate the values for, add the map's id into the file that corresponds to the map's mode. The ids must be seperated either by comma (",") or by whitespace (" ", new line, tab).
 - Rename `.env.example` to `.env` and put proper values for both of its variables:
   - `PERF_CALC_PATH` is going to be something like `C:/osu-tools/PerformanceCalculator/bin/Release/net5.0/PerformanceCalculator.dll`
-  - `MAP_PATH` is the path to the folder containing all relevant `.osu` files that were specified in the `.csv` files
+  - `MAP_PATH` is the path to the folder containing a bunch of `{map_id}.osu` files 
 - Since `PerformanceCalculator.dll` won't print the stars for a map simulation by default, you will need to make a tiny adjustment to a file in the osu-tools repo. In `osu-tools/PerformanceCalculator/Simulate/SimulateCommand.cs`, add the following line to the other lines that look similar:
     ```cs
     o["Stars"] = difficultyAttributes.StarRating;
     ```
     Don't forget to recompile osu-tools via `dotnet build -c Release` after making the change.
-- Run `cargo run --release`. The program will use the `PerformanceCalculator.dll` to calculate the values and then store them in `/output/{mode}.json`.
+- Run `cargo run --release`. The program will use the `PerformanceCalculator.dll` to calculate the values and then store them in `output.json`.
 
 ### Output format
 
@@ -23,15 +22,15 @@ The calculated values for each map for multiple different mod combinations will
     {
         "mode": 0,
         "map_id": integer,
-        "aim": float,
+        "aim": double,
-        "speed": float,
+        "speed": double,
-        "accuracy": float,
+        "accuracy": double,
-        "flashlight": float,
+        "flashlight": double,
-        "od": float,
+        "od": double,
-        "ar": float,
+        "ar": double,
         "mods": "HD, HR", // comma + whitespace separated string of mod abbreviations ("None" for nomod)
-        "stars": float,
+        "stars": double,
-        "pp": float,
+        "pp": double,
     }
   ```
 - taiko: 
@@ -39,11 +38,11 @@ The calculated values for each map for multiple different mod combinations will
     {
         "mode": 1,
         "map_id": integer,
-        "accuracy": float,
+        "accuracy": double,
-        "strain": float,
+        "strain": double,
         "mods": "HD, HR", // comma + whitespace separated string of mod abbreviations ("None" for nomod)
-        "stars": float,
+        "stars": double,
-        "pp": float,
+        "pp": double,
     }
   ```
 - catch: 
@@ -52,8 +51,8 @@ The calculated values for each map for multiple different mod combinations will
         "mode": 2,
         "map_id": integer,
         "mods": "HD, HR", // comma + whitespace separated string of mod abbreviations ("None" for nomod)
-        "stars": float,
+        "stars": double,
-        "pp": float,
+        "pp": double,
     }
   ```
 - mania: 
@@ -61,10 +60,10 @@ The calculated values for each map for multiple different mod combinations will
     {
         "mode": 3,
         "map_id": integer,
-        "accuracy": float,
+        "accuracy": double,
-        "strain": float,
+        "strain": double,
         "mods": "HD, HR", // comma + whitespace separated string of mod abbreviations ("None" for nomod)
-        "stars": float,
+        "stars": double,
-        "pp": float,
+        "pp": double,
     }
   ```

pp-gen/src/main.rs

@@ -1,20 +1,23 @@
-use std::{env, fs::File as StdFile};
+use std::{
+    env, fmt,
+    fs::File as StdFile,
+    io,
+    path::PathBuf,
+    pin::Pin,
+    sync::Mutex,
+    task::{Context, Poll},
+    time::Instant,
+};
 
-use futures::{stream::FuturesUnordered, StreamExt};
+use futures::{future, stream::FuturesUnordered, Stream, StreamExt, TryStreamExt};
+use pbr::ProgressBar;
+use rosu_pp::{Beatmap, GameMode};
 use serde::{Deserialize, Serialize};
-use tokio::{fs::File, io::AsyncReadExt, process::Command, runtime::Runtime};
+use tokio::{
-
+    fs::{self, DirEntry, File, ReadDir},
-macro_rules! info {
+    process::Command,
-    ($($args:tt)*) => {
+    runtime::Runtime,
-        println!("[INFO] {}", format_args!($($args)*))
+};
-    }
-}
-
-macro_rules! error {
-    ($($args:tt)*) => {
-        eprintln!("[ERROR] {}", format_args!($($args)*))
-    }
-}
 
 const OSU_MODS: &[&[&str]] = &[
     &[""],
@@ -67,144 +70,224 @@ const MANIA: &str = "mania";
 
 fn main() {
     dotenv::dotenv().expect("failed to read .env file");
-    let runtime = Runtime::new().expect("failed to create runtiem");
 
-    for mode in [OSU, TAIKO, CATCH, MANIA] {
+    Runtime::new()
-        runtime.block_on(handle_mode(mode));
+        .expect("failed to create runtiem")
-    }
+        .block_on(async_main());
 }
 
-async fn handle_mode(mode: &'static str) {
+async fn async_main() {
+    let start = Instant::now();
+    let map_path = env::var("MAP_PATH").expect("missing `MAP_PATH` environment variable");
+
     let perf_calc_path_ =
         env::var("PERF_CALC_PATH").expect("missing `PERF_CALC_PATH` environment variable");
     let perf_calc_path = perf_calc_path_.as_str();
 
-    let map_path_ = env::var("MAP_PATH").expect("missing `MAP_PATH` environment variable");
+    let mut output = StdFile::create("./output_.json").expect("failed to create output file");
-    let map_path = map_path_.as_str();
 
-    let input_filename = format!("./input/{}.csv", mode);
+    let take = 50;
 
-    let mut file = match File::open(&input_filename).await {
+    let pbr = Mutex::new(ProgressBar::new(take));
-        Ok(file) => file,
+    println!("[INFO] Calculating...");
-        Err(err) => {
-            return error!(
-                "skipping file `{}` because it failed to open: {}",
-                input_filename, err
-            )
-        }
-    };
 
-    let mut csv_data = String::new();
+    let result = fs::read_dir(map_path)
+        .await
+        .map(ReadDirStream::new)
+        .expect("failed to open directory at `MAP_PATH`")
+        .scan(0, |idx, entry| {
+            *idx += 1;
 
-    if let Err(err) = file.read_to_string(&mut csv_data).await {
+            future::ready(Some((*idx % 75 == 0).then(|| entry)))
-        return error!(
+        })
-            "skipping file `{}` because it could not be read: {}",
+        .filter_map(|opt| future::ready(opt))
-            input_filename, err
+        .take(take as usize)
-        );
+        .map(|dir_entry| async {
-    }
+            let dir_entry = dir_entry?;
-
+            let file_name = dir_entry.file_name();
-    let output_filename = format!("./output/{}.json", mode);
+            let file_path = dir_entry.path();
-
+
-    let mut output = match StdFile::create(&output_filename) {
+            let map_id = match file_name
-        Ok(file) => file,
+                .to_string_lossy()
-        Err(err) => {
+                .split('.')
-            return error!(
+                .next()
-                "skipping file `{}` because its output file `{}` could not be created: {}",
+                .map(str::parse)
-                input_filename, output_filename, err
+                .transpose()
-            )
+                .map_err(|_| Error::ParseId)?
-        }
+            {
-    };
+                Some(id) => id,
-
+                None => return Err(Error::EmptyFileName),
-    let (mods, mode_int) = match mode {
+            };
-        OSU => (OSU_MODS, 0),
+
-        TAIKO => (TAIKO_MODS, 1),
+            let file = File::open(&file_path).await?;
-        CATCH => (CATCH_MODS, 2),
+            let mode = Beatmap::parse(file).await?.mode;
-        MANIA => (MANIA_MODS, 3),
+
-        _ => unreachable!(),
+            let result = handle_map(mode, map_id, file_path, perf_calc_path).await;
-    };
+
-
+            if let Ok(mut progress) = pbr.lock() {
-    info!(
+                progress.inc();
-        "Starting to calculate {} data, each map with {} different mod combinations...",
+            }
-        mode,
+
-        mods.len()
+            result
-    );
+        })
-
+        .collect::<FuturesUnordered<_>>()
-    let data: Vec<Data> = csv_data
+        .await
-        .split(|c: char| c == ',' || c.is_whitespace())
+        .try_collect::<Vec<Vec<Data>>>()
-        .filter(|id| !id.is_empty())
+        .await;
-        .filter_map(|id| match id.parse::<u32>() {
+
-            Ok(id) => Some(id),
+    pbr.lock().unwrap().finish_println(&format!(
-            Err(_) => {
+        "[INFO] [{:?}] Finished calculating, now flattening...\n",
-                error!("could not parse `{}` as u32", id);
+        start.elapsed(),
-
+    ));
-                return None;
+
-            }
+    let mut mode_counts = [0; 4];
+
+    let data: Vec<Data> = match result {
+        Ok(data) => data
+            .into_iter()
+            .map(|d| {
+                if let Some(data) = d.get(0) {
+                    mode_counts[data.mode as usize] += 1;
+                }
+
+                d.into_iter()
             })
-        .map(|id| mods.iter().map(move |m| (m, id)))
             .flatten()
-        .map(|(mods, map_id)| async move {
+            .collect(),
-            let map_path = format!("{}/{}.osu", map_path, map_id);
+        Err(err) => return print_err(err.into()),
+    };
+
+    println!(
+        "[INFO] [{:?}] Flattened {} calculations, now writing to file...",
+        start.elapsed(),
+        data.len()
+    );
+
+    match serde_json::to_writer(&mut output, &data) {
+        Ok(_) => println!(
+            "[INFO] [{:?}] Finished writing data into `output.json`\n\
+            Maps:\n  - osu: {}\n  - taiko: {}\n  - catch: {}\n  - mania: {}",
+            start.elapsed(),
+            mode_counts[0],
+            mode_counts[1],
+            mode_counts[2],
+            mode_counts[3]
+        ),
+        Err(err) => print_err(err.into()),
+    }
+}
+
+fn print_err(err: Error) {
+    let mut e: &dyn std::error::Error = &err;
+    eprintln!("[ERROR] {}", err);
+
+    while let Some(src) = e.source() {
+        eprintln!("[ERROR]  - caused by: {}", src);
+        e = src;
+    }
+}
+
+async fn handle_map(
+    mode: GameMode,
+    map_id: u32,
+    path: PathBuf,
+    perf_calc_path: &str,
+) -> Result<Vec<Data>, Error> {
+    let (mods, mode_str) = match mode {
+        GameMode::STD => (OSU_MODS, OSU),
+        GameMode::TKO => (TAIKO_MODS, TAIKO),
+        GameMode::CTB => (CATCH_MODS, CATCH),
+        GameMode::MNA => (MANIA_MODS, MANIA),
+    };
+
+    let mut result = Vec::with_capacity(mods.len());
+
+    for mods_ in mods {
         let mut command = Command::new("dotnet");
 
         command
             .arg(perf_calc_path)
             .arg("simulate")
-                .arg(mode)
+            .arg(mode_str)
-                .arg(map_path)
+            .arg(&path)
             .arg("--json");
 
-            if !mods[0].is_empty() {
+        if !mods_[0].is_empty() {
-                for &m in mods.iter() {
+            for &m in mods_.iter() {
                 command.arg("-m").arg(m);
             }
         }
 
-            let output = match command.output().await {
+        let output = command.output().await?;
-                Ok(output) => output,
+
-                Err(err) => {
+        let data = match serde_json::from_slice(&output.stdout) {
-                    error!(
+            Ok(data) => data,
-                        "failed to calculate values for map {} on {:?}: {}",
+            Err(_) => {
-                        map_id, mods, err
+                let content = String::from_utf8_lossy(&output.stderr);
+                eprintln!(
+                    "[ERROR] mods={:?} | mode={:?} | map={}\n{}",
+                    mods_, mode, map_id, content
                 );
 
-                    return None;
+                continue;
             }
         };
 
-            match serde_json::from_slice(&output.stdout) {
+        let data = Data::new(mode as u32, map_id, data);
-                Ok(data) => Some(Data::new(mode_int, map_id, data)),
+        result.push(data);
-                Err(err) => {
+    }
-                    error!(
-                        "failed to deserialize output for map {} on {:?}: {}\n \
-                        >stdout: {}\n >stderr: {}",
-                        map_id,
-                        mods,
-                        err,
-                        String::from_utf8_lossy(&output.stdout),
-                        String::from_utf8_lossy(&output.stderr),
-                    );
 
-                    None
+    Ok(result)
+}
+
+#[derive(Debug)]
+enum Error {
+    EmptyFileName,
+    Io(std::io::Error),
+    ParseId,
+    ParseMap(rosu_pp::ParseError),
+    Serde(serde_json::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::EmptyFileName => f.write_str("empty file name"),
+            Error::Io(_) => f.write_str("io error"),
+            Error::ParseId => f.write_str("failed to parse map id"),
+            Error::ParseMap(_) => f.write_str("failed to parse map"),
+            Error::Serde(_) => f.write_str("failed to deserialize"),
         }
     }
-        })
+}
-        .collect::<FuturesUnordered<_>>()
-        .filter_map(|data| async { data })
-        .collect::<Vec<Data>>()
-        .await;
 
-    info!(
+impl std::error::Error for Error {
-        "Calculated data for {} map-mod pairs, storing in file `{}`...",
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        data.len(),
+        match self {
-        output_filename
+            Self::EmptyFileName => None,
-    );
+            Self::Io(src) => Some(src),
+            Self::ParseId => None,
+            Self::ParseMap(src) => Some(src),
+            Self::Serde(src) => Some(src),
+        }
+    }
+}
 
-    match serde_json::to_writer(&mut output, &data) {
+impl From<std::io::Error> for Error {
-        Ok(_) => info!("Finished calculating {} data", mode),
+    fn from(e: std::io::Error) -> Self {
-        Err(err) => error!(
+        Self::Io(e)
-            "failed to serialize data into file `{}`: {}",
+    }
-            output_filename, err
+}
-        ),
+
+impl From<rosu_pp::ParseError> for Error {
+    fn from(e: rosu_pp::ParseError) -> Self {
+        Self::ParseMap(e)
+    }
+}
+
+impl From<serde_json::Error> for Error {
+    fn from(e: serde_json::Error) -> Self {
+        Self::Serde(e)
     }
 }
 
@@ -248,3 +331,21 @@ struct GenericData {
     stars: f64,
     pp: f64,
 }
+
+struct ReadDirStream {
+    inner: ReadDir,
+}
+
+impl ReadDirStream {
+    fn new(read_dir: ReadDir) -> Self {
+        Self { inner: read_dir }
+    }
+}
+
+impl Stream for ReadDirStream {
+    type Item = io::Result<DirEntry>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.inner.poll_next_entry(cx).map(Result::transpose)
+    }
+}