Load profile includes so synced Bambu Lab presets get their G-code back

This commit is contained in:
SoftFever
2026-09-24 13:22:37 +08:00
parent d51ad889fd
commit 2b602943a5
14 changed files with 768 additions and 350 deletions
+6
View File
@@ -28,6 +28,12 @@ if (ORCA_TOOLS)
target_link_libraries(generate_system_cache libslic3r boost_headeronly)
target_compile_definitions(generate_system_cache PRIVATE ${_DEV_DEFS})
# profile_include_dump: prints what included templates contribute to a vendor's presets,
# to diff against the same tool built in BambuStudio. Built only on request.
add_executable(profile_include_dump EXCLUDE_FROM_ALL profile_include_dump.cpp)
target_link_libraries(profile_include_dump libslic3r boost_headeronly)
target_compile_definitions(profile_include_dump PRIVATE ${_DEV_DEFS})
endif()
# Function that adds source file encoding check to a target
+215
View File
@@ -0,0 +1,215 @@
// Prints what the profile `include` key contributes to one vendor's system
// presets, so that the dump from OrcaSlicer and the one from BambuStudio can be
// diffed: the two copies of this file differ only in how they load the vendor.
// For every preset whose values pass through an included template, from its
// own file or an ancestor's, it prints the templates those files name, in the
// order they apply, then the loaded value of each key the templates set. Every
// line starts with the preset, so a plain diff names the preset of each
// difference.
#include "libslic3r/PresetBundle.hpp"
#include "libslic3r/Preset.hpp"
#include "libslic3r/Utils.hpp"
#include <boost/filesystem.hpp>
#include <boost/nowide/fstream.hpp>
#include "nlohmann/json.hpp"
#include <algorithm>
#include <iostream>
#include <map>
#include <set>
#include <string>
#include <vector>
using namespace Slic3r;
namespace fs = boost::filesystem;
using nlohmann::json;
namespace {
// What a sub-file states about where its values come from, and what it sets.
struct Entry
{
std::string inherits;
std::vector<std::string> includes;
std::vector<std::string> keys;
};
// Sub-files by name, per section of the vendor index: `inherits` and `include`
// both resolve within a section.
using Sections = std::map<std::string, std::map<std::string, Entry>>;
json read_json(const fs::path &path)
{
boost::nowide::ifstream in(path.string());
if (!in)
throw std::runtime_error("Cannot read " + path.string());
return json::parse(in);
}
Sections read_vendor(const fs::path &dir, const std::string &vendor)
{
// Keys that name and place a sub-file rather than set a value.
static const std::set<std::string> metadata = {"name", "type", "from", "instantiation", "inherits", "include"};
Sections sections;
const json index = read_json(dir / (vendor + ".json"));
for (const char *section : {"process_list", "filament_list", "machine_list"}) {
const auto list = index.find(section);
if (list == index.end())
continue;
for (const json &item : *list) {
const json file = read_json(dir / vendor / item.at("sub_path").get<std::string>());
// A file without a name goes by its name in the index.
Entry &entry = sections[section][file.value("name", item.at("name").get<std::string>())];
entry.inherits = file.value("inherits", "");
if (const auto include = file.find("include"); include != file.end()) {
if (include->is_string())
entry.includes.push_back(include->get<std::string>());
else
for (const json &name : *include)
entry.includes.push_back(name.get<std::string>());
}
for (auto it = file.begin(); it != file.end(); ++it)
if (metadata.count(it.key()) == 0)
entry.keys.push_back(it.key());
}
}
return sections;
}
const PresetCollection &presets_of(const PresetBundle &bundle, const std::string &section)
{
if (section == "process_list")
return bundle.prints;
if (section == "filament_list")
return bundle.filaments;
return bundle.printers;
}
size_t dump(const PresetBundle &bundle, const Sections &sections, std::ostream &out)
{
size_t dumped = 0;
for (const auto &[section, entries] : sections) {
std::vector<const Preset *> presets;
for (const Preset &preset : presets_of(bundle, section).get_presets())
if (preset.is_system)
presets.push_back(&preset);
std::sort(presets.begin(), presets.end(), [](const Preset *a, const Preset *b) { return a->name < b->name; });
for (const Preset *preset : presets) {
// The preset and its ancestors, root first: the order their includes apply in.
std::vector<const Entry *> lineage;
for (auto it = entries.find(preset->name); it != entries.end() && lineage.size() <= entries.size();
it = entries.find(it->second.inherits))
lineage.insert(lineage.begin(), &it->second);
std::string templates;
std::set<std::string> keys;
for (const Entry *entry : lineage)
for (const std::string &name : entry->includes) {
templates += (templates.empty() ? "" : "; ") + name;
if (const auto it = entries.find(name); it != entries.end())
keys.insert(it->second.keys.begin(), it->second.keys.end());
}
if (templates.empty())
continue;
const std::string prefix = section.substr(0, section.find('_')) + " | " + preset->name + " | ";
out << prefix << "include = " << templates << "\n";
for (const std::string &key : keys)
out << prefix << key << " = " << (preset->config.has(key) ? preset->config.opt_serialize(key) : "<absent>") << "\n";
++dumped;
}
}
return dumped;
}
// Orca: load the vendor as the app does, against the filament library when the
// directory has one: parsed from its JSON files or, with from_cache, from the
// preset cache the app reads on every launch after the first.
void load_vendor(PresetBundle &bundle, const std::string &dir, const std::string &vendor, bool from_cache)
{
const auto rule = ForwardCompatibilitySubstitutionRule::EnableSilent;
PresetBundle library;
const PresetBundle *base = nullptr;
if (fs::is_regular_file(fs::path(dir) / (std::string(PresetBundle::ORCA_FILAMENT_LIBRARY) + ".json"))) {
library.load_vendor_configs_from_json(dir, PresetBundle::ORCA_FILAMENT_LIBRARY, PresetBundle::LoadSystem, rule, nullptr, false);
base = &library;
}
if (!from_cache) {
bundle.load_vendor_configs_from_json(dir, vendor, PresetBundle::LoadSystem, rule, base, false);
return;
}
// Parse once to write <vendor>.opc beside the profile, then load that
// cache into a clean bundle. Any cache already there goes first, so that a
// failed write cannot leave it to be loaded; one that was not there before
// goes again afterwards.
const fs::path cache = fs::path(dir) / (vendor + ".opc");
const bool had_cache = fs::exists(cache);
fs::remove(cache);
PresetBundle parsed;
parsed.set_is_validation_mode(true); // parse the JSON: validation never serves a cache
parsed.set_generate_vendor_caches(true);
parsed.load_vendor_configs_from_json(dir, vendor, PresetBundle::LoadSystem, rule, base);
const bool loaded = bundle.load_vendor_cache(cache.string(), vendor, parsed.vendors.at(vendor).config_version, base);
if (!had_cache)
fs::remove(cache);
if (!loaded)
throw std::runtime_error("The preset cache " + cache.string() + " was not written or was rejected");
}
} // namespace
int main(int argc, char *argv[])
{
const char *usage = "Usage: profile_include_dump -p <profiles dir> -o <dump file> [-v <vendor>] [-l <log level>] [-c]\n"
" -v vendor to dump, BBL if omitted\n"
" -l log level, 0 (fatal) to 5 (trace); 1 if omitted\n"
" -c load the presets from the vendor's preset cache, generated first, not the JSON\n";
std::string dir, output, vendor = "BBL";
unsigned log_level = 1;
bool from_cache = false;
for (int i = 1; i < argc; ++i) {
const std::string arg = argv[i];
if (arg == "-c") {
from_cache = true;
continue;
}
if (i + 1 == argc) {
std::cerr << usage;
return 1;
}
const std::string value = argv[++i];
if (arg == "-p")
dir = value;
else if (arg == "-o")
output = value;
else if (arg == "-v")
vendor = value;
else if (arg == "-l" && value.size() == 1 && value[0] >= '0' && value[0] <= '5')
log_level = unsigned(value[0] - '0');
else {
std::cerr << usage;
return 1;
}
}
if (dir.empty() || output.empty() || !fs::is_directory(dir)) {
std::cerr << usage;
return 1;
}
set_logging_level(log_level);
try {
PresetBundle bundle;
load_vendor(bundle, dir, vendor, from_cache);
boost::nowide::ofstream out(output);
if (!out)
throw std::runtime_error("Cannot write " + output);
const size_t dumped = dump(bundle, read_vendor(dir, vendor), out);
out.close();
if (!out)
throw std::runtime_error("Failed writing " + output);
std::cerr << "Dumped " << dumped << " " << vendor << " presets that include a template\n";
} catch (const std::exception &ex) {
std::cerr << ex.what() << "\n";
return 1;
}
return 0;
}
+3
View File
@@ -950,6 +950,9 @@ int ConfigBase::load_from_json(const std::string &file, ConfigSubstitutionContex
}
else if (!load_inherits_to_config && boost::iequals(it.key(), BBL_JSON_KEY_INHERITS)) {
key_values.emplace(BBL_JSON_KEY_INHERITS, it.value());
}
else if (!load_inherits_to_config && boost::iequals(it.key(), BBL_JSON_KEY_INCLUDES)) {
key_values.emplace(BBL_JSON_KEY_INCLUDES, it.value().dump());
} else if (boost::iequals(it.key(), ORCA_JSON_KEY_RENAMED_FROM)) {
key_values.emplace(ORCA_JSON_KEY_RENAMED_FROM, it.value());
} else {
+1
View File
@@ -62,6 +62,7 @@
#define ORCA_JSON_KEY_UPDATE_TIME "updated_time"
#define ORCA_JSON_KEY_CREATED_TIME "created_time"
#define BBL_JSON_KEY_INHERITS "inherits"
#define BBL_JSON_KEY_INCLUDES "include"
#define BBL_JSON_KEY_INSTANTIATION "instantiation"
#define BBL_JSON_KEY_NOZZLE_DIAMETER "nozzle_diameter"
#define BBL_JSON_KEY_PRINTER_TECH "machine_tech"
+65 -20
View File
@@ -6598,7 +6598,8 @@ void PresetBundle::load_config_file_config(const std::string &name_or_path, bool
// now, or deserialized from the vendor's cache; the code is shared so a
// cache-loaded bundle cannot come out different from a JSON-loaded one.
// Resolves `inherits` against the presets loaded before this one
// (config_maps) or against base_bundle's filament library, flattens, validates
// (config_maps) or against base_bundle's filament library, layers each
// `include` (include_maps) under the preset's own keys, flattens, validates
// and registers the preset. Returns the reason loading failed, empty on
// success.
std::string PresetBundle::load_vendor_preset(
@@ -6607,9 +6608,10 @@ std::string PresetBundle::load_vendor_preset(
const PresetBundle* base_bundle,
LoadConfigBundleAttributes flags,
ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions,
std::map<std::string, DynamicPrintConfig>& config_maps, std::map<std::string, std::string>& filament_id_maps,
std::map<std::string, DynamicPrintConfig>& config_maps, std::map<std::string, DynamicPrintConfig>& include_maps,
std::map<std::string, std::string>& filament_id_maps,
PresetCollection* presets_collection, size_t& count, bool is_from_lib,
const std::set<std::string>* retain_configs)
const std::set<std::string>* retain_configs, const std::set<std::string>* retain_includes)
{
const VendorProfile* current_vendor_profile = &this->vendors.at(vendor_name);
const std::string subfile = path + "/" + vendor_name + "/" + entry.sub_path;
@@ -6652,14 +6654,30 @@ std::string PresetBundle::load_vendor_preset(
return reason;
}
}
else {
if (presets_collection->type() == Preset::TYPE_PRINTER)
default_config = &presets_collection->default_preset_for(entry.config_src).config;
else
default_config = &presets_collection->default_preset().config;
}
else
default_config = &presets_collection->default_preset_for(entry.config_src).config;
config = *default_config;
// Layer each included preset's own keys over the parent, in the order listed;
// this preset's own keys go on top.
for (const std::string& name : entry.includes) {
auto it = include_maps.find(name);
if (it == include_maps.end()) {
++m_errors;
BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": can not find include " << name << " for " << preset_name;
continue;
}
config.apply(it->second);
}
config.apply(entry.config_src);
// Record what a base states, its diff against the default, for the presets
// that include it. It is taken before extend_default_config_length pads every
// per-variant key to the base's variant count: the padded defaults would
// otherwise override the values each includer inherits.
if (entry.instantiation == "false" && (retain_includes == nullptr || retain_includes->count(preset_name) != 0)) {
DynamicPrintConfig included;
included.apply_only(config, config.diff(presets_collection->default_preset_for(config).config));
include_maps.emplace(preset_name, std::move(included));
}
extend_default_config_length(config, true, *default_config);
if (entry.instantiation == "false" && "Template" != vendor_name) {
// Report configuration fields, which are misplaced into a wrong group.
@@ -7118,6 +7136,15 @@ std::pair<PresetsConfigSubstitutions, size_t> PresetBundle::load_vendor_configs_
}
entry.name = key_values[BBL_JSON_KEY_NAME];
entry.description = key_values[BBL_JSON_KEY_DESCRIPTION];
// A file that states no instantiation and is named as G-code, or has no
// name, is a template that is only there to be included. A nameless one
// goes by its name in the vendor index.
if (auto it = key_values.find(BBL_JSON_KEY_INSTANTIATION);
(it == key_values.end() || it->second.empty()) && (entry.name.empty() || entry.name.find("gcode") != std::string::npos)) {
key_values[BBL_JSON_KEY_INSTANTIATION] = "false";
if (entry.name.empty())
entry.name = subfile_iter.first;
}
if(key_values.find(BBL_JSON_KEY_INSTANTIATION) == key_values.end())
{
BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": Missing instantiation attribute for " << entry.name;
@@ -7146,6 +7173,20 @@ std::pair<PresetsConfigSubstitutions, size_t> PresetBundle::load_vendor_configs_
return reason;
}
}
if (auto it = key_values.find(BBL_JSON_KEY_INCLUDES); it != key_values.end()) {
// An array of names, or one bare name; load_from_json kept the JSON text.
nlohmann::json includes = nlohmann::json::parse(it->second);
if (!includes.is_array())
includes = nlohmann::json::array({std::move(includes)});
for (const auto& name : includes) {
if (name.is_string())
entry.includes.push_back(name.get<std::string>());
else {
++m_errors;
BOOST_LOG_TRIVIAL(error) << __FUNCTION__ << ": invalid include " << name.dump() << " for " << entry.name;
}
}
}
if (key_values.find(ORCA_JSON_KEY_RENAMED_FROM) != key_values.end()) {
if (!unescape_strings_cstyle(key_values[ORCA_JSON_KEY_RENAMED_FROM], entry.renamed_from)) {
BOOST_LOG_TRIVIAL(error) << "Error in a Config \"" << dir << "\": The preset \"" << entry.name
@@ -7162,7 +7203,7 @@ std::pair<PresetsConfigSubstitutions, size_t> PresetBundle::load_vendor_configs_
return reason;
};
std::map<std::string, DynamicPrintConfig> configs;
std::map<std::string, DynamicPrintConfig> configs, include_maps;
std::map<std::string, std::string> filament_id_maps;
// Orca: whether to (re)write the vendor's cache after this parse, leaving it
// in step with the profile so the next run reads it instead. It is written
@@ -7179,6 +7220,7 @@ std::pair<PresetsConfigSubstitutions, size_t> PresetBundle::load_vendor_configs_
auto load_subfiles = [&](std::vector<std::pair<std::string, std::string>>& subfiles,
std::vector<CachedPreset>& entries, const char* kind, bool is_from_lib = false) {
configs.clear();
include_maps.clear();
filament_id_maps.clear();
for (auto& subfile : subfiles) {
CachedPreset entry;
@@ -7186,8 +7228,8 @@ std::pair<PresetsConfigSubstitutions, size_t> PresetBundle::load_vendor_configs_
if (reason.empty()) {
const int errors_before_install = m_errors;
reason = load_vendor_preset(entry, dir, vendor_name, base_bundle, flags,
substitution_context, substitutions, configs, filament_id_maps, presets,
presets_loaded, is_from_lib);
substitution_context, substitutions, configs, include_maps, filament_id_maps,
presets, presets_loaded, is_from_lib);
install_errors += m_errors - errors_before_install;
}
if (!reason.empty()) {
@@ -7982,26 +8024,29 @@ bool PresetBundle::load_vendor_cache(const std::string& cache_path, const std::s
// parsed), so no substitutions are reported, as before.
ConfigSubstitutionContext substitution_context { ForwardCompatibilitySubstitutionRule::EnableSilent };
PresetsConfigSubstitutions substitutions;
std::map<std::string, DynamicPrintConfig> configs;
std::map<std::string, DynamicPrintConfig> configs, include_maps;
std::map<std::string, std::string> filament_id_maps;
const std::string path = boost::filesystem::path(cache_path).parent_path().string();
size_t count = 0;
auto install_entries = [&](const std::vector<CachedPreset>& entries, PresetCollection* presets, bool is_from_lib) {
configs.clear();
include_maps.clear();
filament_id_maps.clear();
// Only configs of presets that other entries inherit are ever looked
// up again; registering just those skips one full config copy for
// every leaf preset. The library's filaments are all retained — they
// become the m_config_maps other vendors resolve against.
std::set<std::string> inherited;
for (const CachedPreset& entry : entries)
// Only configs of presets that other entries inherit or include are
// ever looked up again; registering just those skips one full config
// copy for every leaf preset. The library's filaments are all retained
// — they become the m_config_maps other vendors resolve against.
std::set<std::string> inherited, included;
for (const CachedPreset& entry : entries) {
if (! entry.inherits.empty())
inherited.insert(entry.inherits);
included.insert(entry.includes.begin(), entry.includes.end());
}
const std::set<std::string>* retain_configs = is_from_lib ? nullptr : &inherited;
for (const CachedPreset& entry : entries) {
const std::string reason = load_vendor_preset(entry, path, vendor_name,
base_bundle, LoadConfigBundleAttribute::LoadSystem, substitution_context, substitutions,
configs, filament_id_maps, presets, count, is_from_lib, retain_configs);
configs, include_maps, filament_id_maps, presets, count, is_from_lib, retain_configs, &included);
if (! reason.empty())
throw std::runtime_error("entry " + entry.name + " failed to install: " + reason);
}
+12 -10
View File
@@ -634,22 +634,24 @@ private:
// load_vendor_configs_from_json reads a cache.
bool load_vendor_cache(const boost::filesystem::path& dir, const std::string& vendor_name, const PresetBundle* base_bundle);
// Load one source-form preset entry into this bundle: resolve `inherits`,
// flatten, validate and register the preset. Returns the reason loading
// failed, empty on success. See the definition for the sharing contract
// between the JSON parse and the cache load.
// retain_configs, when non-null, names the only presets registered into
// config_maps (a full config copy each). The cache load passes the names its
// entries inherit — the only ones ever looked up again; the JSON parse
// retains all, not knowing what later subfiles inherit.
// Load one source-form preset entry into this bundle: resolve `inherits`
// and `include`, flatten, validate and register the preset. Returns the
// reason loading failed, empty on success. See the definition for the
// sharing contract between the JSON parse and the cache load.
// retain_configs / retain_includes, when non-null, name the only presets
// registered into config_maps / include_maps (a config copy each). The
// cache load passes the names its entries inherit / include — the only
// ones ever looked up again; the JSON parse retains all, not knowing what
// later subfiles name.
std::string load_vendor_preset(const CachedPreset& entry,
const std::string& path, const std::string& vendor_name,
const PresetBundle* base_bundle,
LoadConfigBundleAttributes flags,
ConfigSubstitutionContext& substitution_context, PresetsConfigSubstitutions& substitutions,
std::map<std::string, DynamicPrintConfig>& config_maps, std::map<std::string, std::string>& filament_id_maps,
std::map<std::string, DynamicPrintConfig>& config_maps, std::map<std::string, DynamicPrintConfig>& include_maps,
std::map<std::string, std::string>& filament_id_maps,
PresetCollection* presets_collection, size_t& count, bool is_from_lib,
const std::set<std::string>* retain_configs = nullptr);
const std::set<std::string>* retain_configs = nullptr, const std::set<std::string>* retain_includes = nullptr);
// Clear every collection's m_printer_hold_alias, which reset() leaves alone.
void clear_printer_hold_aliases();
+2 -2
View File
@@ -254,7 +254,7 @@ constexpr uint32_t CACHE_MAGIC = 0x4F52435A; // "ORCZ"
// save_entries below), or a change to the cache's own layout or the
// meaning of its stamps. Option-schema drift is NOT such a change — the
// dictionary handles it, which is why this no longer moves every release.
constexpr uint32_t CACHE_VERSION = 1;
constexpr uint32_t CACHE_VERSION = 2;
// A stamp-string read that refuses an absurd length before allocating anything.
// The stamps are read from files named from the outside (peek_version is
@@ -325,7 +325,7 @@ void visit_entry(Archive& ar, Entry& e, ConfigFn&& config)
{
ar(e.name, e.sub_path);
config();
ar(e.inherits, e.description, e.instantiation, e.setting_id, e.filament_id, e.renamed_from);
ar(e.inherits, e.includes, e.description, e.instantiation, e.setting_id, e.filament_id, e.renamed_from);
}
// The count comes from a file that has already passed magic and CRC, but a
+8 -6
View File
@@ -107,12 +107,13 @@ void load_config(cereal::BinaryInputArchive& ar, DynamicPrintConfig& config, con
// comes after.
void skip_config(cereal::BinaryInputArchive& ar, const CacheDictionary& dict);
// One preset as its JSON subfile states it: the config diff, the name of the
// preset it inherits, and the parse metadata — everything the parse phase of
// load_vendor_configs_from_json extracts and nothing it derives. Inheritance
// is resolved when the entry is installed, against whatever filament library
// is loaded then, so a cache carries no other vendor's values and no other
// vendor's update can make it stale.
// One preset as its JSON subfile states it: the config diff, the names of the
// preset it inherits and the presets it includes, and the parse metadata —
// everything the parse phase of load_vendor_configs_from_json extracts and
// nothing it derives. Inheritance and includes are resolved when the entry is
// installed, against whatever filament library is loaded then, so a cache
// carries no other vendor's values and no other vendor's update can make it
// stale.
// Written and read by visit_entry in PresetCacheFormat.cpp, which lists every
// field below in this order — once, for the save, the load and the name peek alike.
struct CachedPreset
@@ -121,6 +122,7 @@ struct CachedPreset
std::string sub_path; // path under the vendor's directory
DynamicPrintConfig config_src; // the preset's own diff, nothing inherited
std::string inherits;
std::vector<std::string> includes; // layered under config_src, in this order
std::string description;
std::string instantiation; // "true"/"false" as stated; anything else was already counted as a parse error
std::string setting_id;