diff --git a/Tutorials/ML/benchmarkOnnxModel.cxx b/Tutorials/ML/benchmarkOnnxModel.cxx index 67677940346..521f3604d8f 100644 --- a/Tutorials/ML/benchmarkOnnxModel.cxx +++ b/Tutorials/ML/benchmarkOnnxModel.cxx @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -116,6 +118,8 @@ struct BenchmarkOnnxModel { Configurable iterations{"iterations", 100, "Timed inference calls"}; Configurable cpuThreads{"cpu-threads", 1, "ORT intra-op CPU threads"}; Configurable outputCsv{"output-csv", "onnx-benchmark.csv", "CSV output (overwritten)"}; + Configurable inputFile{"input-file", "", "Optional whitespace-separated feature matrix [batch-size, features]; empty uses synthetic inputs"}; + Configurable predictionsFile{"predictions-file", "", "Optional first-output matrix as text (overwritten); extra inference outside timing"}; void run(ProcessingContext& pc) { @@ -189,17 +193,55 @@ struct BenchmarkOnnxModel { if (count > std::numeric_limits::max() / sizeof(float)) { throw std::runtime_error("Input size overflow"); } - // Replace this feature filling with AO2D extraction and training preprocessing later. + // File loading and FP16 conversion are outside the timing loop. Without a + // file, retain deterministic synthetic inputs for performance-only runs. std::vector floats(fp16.value ? 0 : count); std::vector halves(fp16.value ? count : 0); static_assert(sizeof(o2::OrtDataType::Float16_t) == 2); - for (size_t i = 0; i < count; ++i) { - float value = static_cast(i % 101) / 100.f - 0.5f; + auto store = [&](size_t i, float value) { + if (!std::isfinite(value)) { + throw std::runtime_error("Input features must be finite"); + } if (fp16.value) { halves[i] = o2::OrtDataType::Float16_t(value); + if (!std::isfinite(halves[i].ToFloat())) { + throw std::runtime_error("Input feature overflows FP16"); + } } else { floats[i] = value; } + }; + if (inputFile.value.empty()) { + for (size_t i = 0; i < count; ++i) { + store(i, static_cast(i % 101) / 100.f - 0.5f); + } + } else { + std::ifstream file(inputFile.value); + if (!file) { + throw std::runtime_error("Cannot open input-file: " + inputFile.value); + } + std::string line; + for (int row = 0; row < batchSize.value; ++row) { + if (!std::getline(file, line)) { + throw std::runtime_error("input-file has fewer rows than batch-size"); + } + std::istringstream values(line); + for (int column = 0; column < features.value; ++column) { + float value = 0; + if (!(values >> value)) { + throw std::runtime_error("input-file has a missing or invalid feature at row " + std::to_string(row + 1)); + } + store(static_cast(row) * features.value + column, value); + } + values >> std::ws; + if (!values.eof()) { + throw std::runtime_error("input-file has extra columns at row " + std::to_string(row + 1)); + } + } + file >> std::ws; + if (!file.eof() || file.bad()) { + throw std::runtime_error("input-file has extra rows or a read error"); + } } auto host = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); void* data = fp16.value ? static_cast(halves.data()) : static_cast(floats.data()); @@ -238,6 +280,50 @@ struct BenchmarkOnnxModel { if (memory) { afterRun = memory->used(); } + if (!predictionsFile.value.empty()) { + // A separate, untimed run leaves benchmark allocation/destruction intact. + auto outputs = session.Run(runOptions, inputNames, &input, 1, outputNames.data(), outputNames.size()); + auto const& output = outputs.front(); + if (!output.IsTensor()) { + throw std::runtime_error("Prediction export requires a dense tensor as the first output"); + } + auto info = output.GetTensorTypeAndShapeInfo(); + auto outputShape = info.GetShape(); + auto outputType = info.GetElementType(); + if (outputShape.size() != 2 || outputShape[0] != batchSize.value || outputShape[1] <= 0 || + (outputType != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT && outputType != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16)) { + throw std::runtime_error("Prediction export requires an FP32 or FP16 first output [batch-size, classes]"); + } + const float* fp32 = outputType == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT ? output.GetTensorData() : nullptr; + const auto* fp16Data = outputType == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 ? output.GetTensorData() : nullptr; + const size_t columns = static_cast(outputShape[1]); + // Validate before opening the destination, so invalid predictions do not + // replace an existing file with a partial matrix. + auto valueAt = [&](size_t i) { return fp32 ? fp32[i] : fp16Data[i].ToFloat(); }; + for (size_t i = 0; i < info.GetElementCount(); ++i) { + if (!std::isfinite(valueAt(i))) { + throw std::runtime_error("Prediction export found a nonfinite value"); + } + } + std::ofstream predictions(predictionsFile.value, std::ios::trunc); + if (!predictions) { + throw std::runtime_error("Cannot open predictions-file: " + predictionsFile.value); + } + predictions << std::setprecision(std::numeric_limits::max_digits10); + for (int row = 0; row < batchSize.value; ++row) { + for (size_t column = 0; column < columns; ++column) { + if (column) { + predictions << ' '; + } + predictions << valueAt(static_cast(row) * columns + column); + } + predictions << '\n'; + } + predictions.close(); + if (!predictions) { + throw std::runtime_error("Failed writing predictions-file"); + } + } std::ofstream csv(outputCsv.value, std::ios::trunc); if (!csv) { throw std::runtime_error("Cannot open output-csv");