From e098c933f04f5498dbf11dfb9461a353d03b8fe7 Mon Sep 17 00:00:00 2001 From: Kris Austin Date: Tue, 6 Oct 2026 05:50:08 -0500 Subject: [PATCH] perf: write post-processed G-code without per-line copies to speed up export by up to 6% (#16167) * perf: write post-processed G-code without a per-line copy * perf: size the post-process line map from the first pass * test: line ends of the exported G-code * test: include the headers the line-ends test and gcode() helper use --- src/libslic3r/GCode/GCodeProcessor.cpp | 188 +++++++++++------------- src/libslic3r/GCode/GCodeProcessor.hpp | 3 + tests/fff_print/test_gcodeprocessor.cpp | 44 ++++++ tests/fff_print/test_helpers.cpp | 7 +- tests/fff_print/test_helpers.hpp | 8 +- 5 files changed, 139 insertions(+), 111 deletions(-) diff --git a/src/libslic3r/GCode/GCodeProcessor.cpp b/src/libslic3r/GCode/GCodeProcessor.cpp index 10a6f9dc88..17f55905f9 100644 --- a/src/libslic3r/GCode/GCodeProcessor.cpp +++ b/src/libslic3r/GCode/GCodeProcessor.cpp @@ -751,6 +751,50 @@ template } } +namespace { +// Writes G-code to a file in blocks and records in lines_ends the file offset after every '\n' +class GCodeFileWriter +{ +public: + GCodeFileWriter(FilePtr& out, const std::string& out_path, std::vector& lines_ends, const char* error_message) + : m_out(out), m_out_path(out_path), m_lines_ends(lines_ends), m_error_message(error_message) + {} + ~GCodeFileWriter() { assert(m_buffer.empty() || std::uncaught_exceptions() > 0); } + + void append(std::string_view text) + { + const size_t text_pos = m_file_pos + m_buffer.size(); + for (size_t i = text.find('\n'); i != std::string_view::npos; i = text.find('\n', i + 1)) + m_lines_ends.emplace_back(text_pos + i + 1); + m_buffer += text; + if (m_buffer.size() >= GCodeProcessor::Output_Block_Size) + flush(); + } + + void flush() + { + if (m_buffer.empty()) + return; + fwrite(m_buffer.data(), 1, m_buffer.size(), m_out.f); + if (ferror(m_out.f)) { + m_out.close(); + boost::nowide::remove(m_out_path.c_str()); + throw Slic3r::RuntimeError(m_error_message); + } + m_file_pos += m_buffer.size(); + m_buffer.clear(); + } + +private: + FilePtr& m_out; + const std::string& m_out_path; + std::vector& m_lines_ends; + const char* m_error_message; + std::string m_buffer; + size_t m_file_pos{0}; +}; +} // namespace + // Helper class to modify and export gcode to file class ExportLines { @@ -765,16 +809,6 @@ public: enum class EWriteType { BySize, ByTime }; private: - static void update_lines_ends_and_out_file_pos(const std::string& out_string, std::vector& lines_ends, size_t* out_file_pos) - { - for (size_t i = 0; i < out_string.size(); ++i) { - if (out_string[i] == '\n') - lines_ends.emplace_back((out_file_pos != nullptr) ? *out_file_pos + i + 1 : i + 1); - } - if (out_file_pos != nullptr) - *out_file_pos += out_string.size(); - } - struct LineData { std::string line; @@ -814,12 +848,14 @@ private: EWriteType m_write_type{EWriteType::BySize}; // Time machines containing g1 times cache const std::array(PrintEstimatedStatistics::ETimeMode::Count)>& m_machines; + // Output file writer + GCodeFileWriter& m_writer; // Current time std::array(PrintEstimatedStatistics::ETimeMode::Count)> m_times{0.0f, 0.0f}; - // Current size in bytes + // Current size of the cache in bytes size_t m_size{0}; - // gcode lines cache + // gcode lines cache, used only when writing by time std::deque m_lines; size_t m_added_lines_counter{0}; // map of gcode line ids from original to final @@ -827,16 +863,16 @@ private: std::vector> m_gcode_lines_map; size_t m_times_cache_id{0}; - size_t m_out_file_pos{0}; public: - ExportLines(EWriteType type, const std::array(PrintEstimatedStatistics::ETimeMode::Count)>& machines) + ExportLines(EWriteType type, const std::array(PrintEstimatedStatistics::ETimeMode::Count)>& machines, GCodeFileWriter& writer) #ifndef NDEBUG : m_statistics(*this) , m_write_type(type) - , m_machines(machines){} + , m_machines(machines) + , m_writer(writer){} #else - : m_write_type(type), m_machines(machines) + : m_write_type(type), m_machines(machines), m_writer(writer) {} #endif // NDEBUG @@ -890,11 +926,14 @@ public: if (line.empty()) return; - m_lines.push_back({line, m_times}); + if (m_write_type == EWriteType::ByTime) { + m_lines.push_back({line, m_times}); #ifndef NDEBUG - m_statistics.add_line(line.length()); + m_statistics.add_line(line.length()); #endif // NDEBUG - m_size += line.length(); + m_size += line.length(); + } else + m_writer.append(line); ++m_added_lines_counter; if (!ignore_from_move) { assert(!m_gcode_lines_map.empty()); @@ -960,65 +999,35 @@ public: } } - // write to file: - // m_write_type == EWriteType::ByTime - all lines older than m_time - backtrace_time - // m_write_type == EWriteType::BySize - all lines if current size is greater than 65535 bytes - void write(FilePtr& out, float backtrace_time, GCodeProcessorResult& result, const std::string& out_path) + // when writing by time, pass the cached lines older than m_times[Normal] - backtrace_time to the writer + void write(float backtrace_time) { - if (m_lines.empty()) + if (m_write_type != EWriteType::ByTime) return; - // collect lines to write into a single string - std::string out_string; - if (!m_lines.empty()) { - if (m_write_type == EWriteType::ByTime) { - while (m_lines.front().times[Normal] < m_times[Normal] - backtrace_time) { - const LineData& data = m_lines.front(); - out_string += data.line; - m_size -= data.line.length(); - m_lines.pop_front(); + while (!m_lines.empty() && m_lines.front().times[Normal] < m_times[Normal] - backtrace_time) { + const LineData& data = m_lines.front(); + m_writer.append(data.line); + m_size -= data.line.length(); + m_lines.pop_front(); #ifndef NDEBUG - m_statistics.remove_line(); + m_statistics.remove_line(); #endif // NDEBUG - } - } else { - if (m_size > 65535) { - while (!m_lines.empty()) { - out_string += m_lines.front().line; - m_lines.pop_front(); - } - m_size = 0; -#ifndef NDEBUG - m_statistics.remove_all_lines(); -#endif // NDEBUG - } - } - } - - { - write_to_file(out, out_string, result, out_path); - update_lines_ends_and_out_file_pos(out_string, result.lines_ends, &m_out_file_pos); } } - // flush the current content of the cache to file - void flush(FilePtr& out, GCodeProcessorResult& result, const std::string& out_path) + // flush the current content of the cache and the writer to file + void flush() { - // collect lines to flush into a single string - std::string out_string; while (!m_lines.empty()) { - out_string += m_lines.front().line; + m_writer.append(m_lines.front().line); m_lines.pop_front(); } m_size = 0; #ifndef NDEBUG m_statistics.remove_all_lines(); #endif // NDEBUG - - { - write_to_file(out, out_string, result, out_path); - update_lines_ends_and_out_file_pos(out_string, result.lines_ends, &m_out_file_pos); - } + m_writer.flush(); } void synchronize_moves(GCodeProcessorResult& result) const @@ -1051,20 +1060,7 @@ public: size_t get_size() const { return m_size; } -private: - void write_to_file(FilePtr& out, const std::string& out_string, GCodeProcessorResult& result, const std::string& out_path) - { - if (!out_string.empty()) { - if (true) { - fwrite((const void*) out_string.c_str(), 1, out_string.length(), out.f); - if (ferror(out.f)) { - out.close(); - boost::nowide::remove(out_path.c_str()); - throw Slic3r::RuntimeError("GCode processor post process export failed.\nIs the disk full?"); - } - } - } - } + void reserve(size_t lines_count) { m_gcode_lines_map.reserve(lines_count); } }; void GCodeProcessor::run_post_process() @@ -1160,8 +1156,13 @@ void GCodeProcessor::run_post_process() last_exported_stop[i] = time_in_minutes(m_time_processor.machines[i].time); } + m_result.lines_ends.clear(); + // m_result.lines_ends.emplace_back(std::vector()); + GCodeFileWriter writer(out, out_path, m_result.lines_ends, "GCode processor post process export failed.\nIs the disk full?"); ExportLines export_line(m_result.backtrace_enabled ? ExportLines::EWriteType::ByTime : ExportLines::EWriteType::BySize, - m_time_processor.machines); + m_time_processor.machines, writer); + // The line map holds an entry for each line of the file, and the first pass counted them + export_line.reserve(m_line_id); // replace placeholder lines with the proper final value // gcode_line is in/out parameter, to reduce expensive memory allocation @@ -1535,9 +1536,6 @@ void GCodeProcessor::run_post_process() } }; - m_result.lines_ends.clear(); - // m_result.lines_ends.emplace_back(std::vector()); - // Orca: freshly collect SKIPPABLE ranges each post-process pass. The ranges are stored on the // member (rather than a local) so the injection pass can consume them, hence the clear here to // avoid stale ranges on re-invocation. @@ -1802,7 +1800,7 @@ void GCodeProcessor::run_post_process() if (!gcode_line.empty()) export_line.append_line(gcode_line); - export_line.write(out, 1.1f * max_backtrace_time, m_result, out_path); + export_line.write(1.1f * max_backtrace_time); gcode_line.clear(); } } @@ -1844,7 +1842,7 @@ void GCodeProcessor::run_post_process() } } - export_line.flush(out, m_result, out_path); + export_line.flush(); out.close(); in.close(); @@ -1992,31 +1990,13 @@ void GCodeProcessor::run_second_pass_injection() // The rewrite may shift byte positions (once the injector inserts lines), so rebuild lines_ends from scratch. // With an empty map the scanned '\n' offsets reproduce the current lines_ends exactly. m_result.lines_ends.clear(); - size_t out_file_pos = 0; - - auto write_out = [&out, &out_path, this, &out_file_pos](std::string& str) { - if (str.empty()) - return; - fwrite((const void*) str.c_str(), 1, str.length(), out.f); - if (ferror(out.f)) { - out.close(); - boost::nowide::remove(out_path.c_str()); - throw Slic3r::RuntimeError(std::string("GCode processor pre-heat injection pass failed.\nIs the disk full?\n")); - } - for (size_t i = 0; i < str.size(); ++i) { - if (str[i] == '\n') - m_result.lines_ends.emplace_back(out_file_pos + i + 1); - } - out_file_pos += str.size(); - str.clear(); - }; + GCodeFileWriter writer(out, out_path, m_result.lines_ends, "GCode processor pre-heat injection pass failed.\nIs the disk full?\n"); // Orca: read/split lines with EOL-preserving semantics (keep the original \r and \n bytes, and // synthesize no trailing newline). This is required for the empty-map identity: normalizing every // line ending to "\n" would not be byte-identical if the finished file used \r\n or lacked a // final newline. std::string gcode_line; - std::string export_buffer; unsigned int line_id = 0; auto op_it = inserted_operation_lines.begin(); std::vector buffer(65536 * 10, 0); @@ -2058,16 +2038,14 @@ void GCodeProcessor::run_second_pass_injection() } ++op_it; } - export_buffer += gcode_line; + writer.append(gcode_line); gcode_line.clear(); - if (export_buffer.length() >= 65536) - write_out(export_buffer); } } if (eof) break; } - write_out(export_buffer); + writer.flush(); out.close(); in.close(); diff --git a/src/libslic3r/GCode/GCodeProcessor.hpp b/src/libslic3r/GCode/GCodeProcessor.hpp index 819b019703..919315d00c 100644 --- a/src/libslic3r/GCode/GCodeProcessor.hpp +++ b/src/libslic3r/GCode/GCodeProcessor.hpp @@ -491,6 +491,9 @@ class Print; static const std::string VFlush_End_Tag; static const std::string External_Purge_Tag; public: + // Size of the blocks the post-processing passes write the G-code in + static constexpr size_t Output_Block_Size = 65536; + // Orca: SKIPPABLE region tags, stored as static strings (the FLUSH idiom above) rather than // a CustomETags/CustomTags array. Public so the emission sites (WipeTower / change_filament // path) can reference them single-sourced. diff --git a/tests/fff_print/test_gcodeprocessor.cpp b/tests/fff_print/test_gcodeprocessor.cpp index b512753c01..2d6bafd0a9 100644 --- a/tests/fff_print/test_gcodeprocessor.cpp +++ b/tests/fff_print/test_gcodeprocessor.cpp @@ -1,15 +1,20 @@ #include +#include #include #include #include #include #include "libslic3r/libslic3r.h" +#include "libslic3r/Config.hpp" #include "libslic3r/GCode/GCodeProcessor.hpp" +#include "libslic3r/Model.hpp" #include "libslic3r/Utils.hpp" +#include "test_helpers.hpp" #include "test_utils.hpp" +#include #include #include #include "libslic3r/PrintConfig.hpp" @@ -177,3 +182,42 @@ TEST_CASE("A seam takes the actual speed of the move it follows", "[GCodeProcess } REQUIRE(seams > 0); } + +TEST_CASE("Line ends of the exported G-code mark every newline in the file", "[GCodeProcessor]") +{ + struct Case + { + const char* name; + bool preheat_backtrace; + bool pre_heating; + }; + const auto test_case = GENERATE(values({ + { "written by size", false, false }, + { "written by time for the preheat backtrace", true, false }, + { "rewritten by the pre-heating pass", false, true }, + })); + INFO(test_case.name); + DynamicPrintConfig config = Test::multifilament_config(2, { + { "single_extruder_multi_material", 0 }, + { "ooze_prevention", test_case.preheat_backtrace }, + { "preheat_time", 30 }, + { "enable_pre_heating", test_case.pre_heating }, + }); + Print print; + Model model; + const std::vector> overrides{ { { "extruder", 1 } }, { { "extruder", 2 } } }; + Test::init_print({ Test::cube(20), Test::cube(20) }, print, model, config, &overrides); + GCodeProcessorResult result; + const std::string gcode = Test::gcode(print, &result); + REQUIRE((gcode.find("preheat T") != std::string::npos) == test_case.preheat_backtrace); + REQUIRE((gcode.find(GCodeProcessor::Machine_Start_GCode_End_Tag) != std::string::npos) == test_case.pre_heating); + REQUIRE(gcode.size() > GCodeProcessor::Output_Block_Size); + + std::vector newline_ends; + for (size_t i = gcode.find('\n'); i != std::string::npos; i = gcode.find('\n', i + 1)) + newline_ends.push_back(i + 1); + REQUIRE(result.lines_ends.size() == newline_ends.size()); + const auto difference = std::mismatch(result.lines_ends.begin(), result.lines_ends.end(), newline_ends.begin()); + INFO("first difference at line " << difference.first - result.lines_ends.begin() + 1); + CHECK(difference.first == result.lines_ends.end()); +} diff --git a/tests/fff_print/test_helpers.cpp b/tests/fff_print/test_helpers.cpp index 38935f33a7..e32f79944e 100644 --- a/tests/fff_print/test_helpers.cpp +++ b/tests/fff_print/test_helpers.cpp @@ -14,6 +14,7 @@ #include #include "libslic3r/Point.hpp" #include +#include #include #include #include @@ -330,13 +331,13 @@ void init_and_process_print(std::initializer_list meshes, Slic3r:: print.process(); } -std::string gcode(Print & print) +std::string gcode(Print & print, GCodeProcessorResult* result) { ScopedTemporaryFile temp(".gcode"); print.set_status_silent(); print.process(); - print.export_gcode(temp.string(), nullptr, nullptr); - std::ifstream t(temp.string()); + print.export_gcode(temp.string(), result, nullptr); + std::ifstream t(temp.string(), std::ios::binary); std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); return str; } diff --git a/tests/fff_print/test_helpers.hpp b/tests/fff_print/test_helpers.hpp index 1ffc08c64e..c603007a7c 100644 --- a/tests/fff_print/test_helpers.hpp +++ b/tests/fff_print/test_helpers.hpp @@ -17,7 +17,9 @@ #include #include -namespace Slic3r { namespace Test { +namespace Slic3r { +struct GCodeProcessorResult; +namespace Test { constexpr double MM_PER_MIN = 60.0; @@ -92,8 +94,8 @@ void init_and_process_print(std::initializer_list meshes, Slic3r:: void init_and_process_print(std::initializer_list meshes, Slic3r::Print &print, std::initializer_list config_items); void init_and_process_print(std::initializer_list meshes, Slic3r::Print &print, std::initializer_list config_items); -// Process `print` and return its exported G-code. -std::string gcode(Print& print); +// Process `print` and return its exported G-code, filling `result` when one is given. +std::string gcode(Print& print, GCodeProcessorResult* result = nullptr); // Build, slice, and return the G-code for `meshes` under the given config. std::string slice(std::initializer_list meshes, const DynamicPrintConfig &config);