Lay on Face: flip a face pointing straight up about X, not an arbitrary axis

Laying a face that already points up down needs a half turn, and every
horizontal axis does the job. setFromTwoVectors() resolved the choice through an
SVD whose null space is the whole XY plane, so the axis followed rounding noise:
the object landed at an arbitrary angle in the plate plane, and a part that fit
the plate no longer did. Turn such a face over about X, which keeps the
footprint, and share the rotation between the gizmo and the --ground-* options
instead of duplicating it.

src/libslic3r/LayOnFace.cpp

@@ -213,13 +213,29 @@ int find_plane_at_point(const std::vector<LayOnFacePlane> &planes, const Transfo
     return -1;
 }
 
+Transform3d lay_on_face_rotation(const Vec3d &normal, const Transform3d &instance_matrix_no_offset)
+{
+    // Normal in instance coordinates: a normal transforms by the inverse transpose.
+    const Vec3d tnormal = (instance_matrix_no_offset.matrix().block(0, 0, 3, 3).inverse().transpose() * normal).normalized();
+    const Vec3d down    = -Vec3d::UnitZ();
+
+    // A face already pointing straight up needs a half turn, and every horizontal axis does
+    // that equally well. setFromTwoVectors() resolves the choice through an SVD whose null
+    // space is the whole XY plane, so the axis follows rounding noise and the object lands at
+    // an arbitrary angle in the plate plane. Turn it over about X instead, which keeps the
+    // footprint the object already had.
+    if (tnormal.dot(down) < -1. + 1e-9)
+        return Transform3d(Eigen::AngleAxisd(PI, Vec3d::UnitX()));
+
+    return Transform3d(Eigen::Quaterniond().setFromTwoVectors(tnormal, down));
+}
+
 void lay_on_face(ModelObject &object, size_t instance_idx, const Vec3d &normal)
 {
     ModelInstance                  &instance = *object.instances[instance_idx];
     const Geometry::Transformation &trafo    = instance.get_transformation();
-    // Same rotation as Selection::flattening_rotate(): turn the transformed normal to point down.
-    const Vec3d       tnormal  = trafo.get_matrix().matrix().block(0, 0, 3, 3).inverse().transpose() * normal;
-    const Transform3d rotation = Transform3d(Eigen::Quaterniond().setFromTwoVectors(tnormal, -Vec3d::UnitZ()));
+    // Same rotation as Selection::flattening_rotate(), which the gizmo uses.
+    const Transform3d rotation = lay_on_face_rotation(normal, trafo.get_matrix_no_offset());
     instance.set_transformation(Geometry::Transformation(trafo.get_offset_matrix() * rotation * trafo.get_matrix_no_offset()));
     // Drop this instance only: ensure_on_bed() skips instances without auto_drop and measures the first instance.
     object.translate_instance(instance_idx, -object.instance_bounding_box(instance_idx).min.z() * Vec3d::UnitZ());

src/libslic3r/LayOnFace.hpp

@@ -46,4 +46,10 @@ int find_plane_at_point(const std::vector<LayOnFacePlane> &planes, const Transfo
 // the gizmo applies, then drops the instance so its lowest point is at z = 0.
 void lay_on_face(ModelObject &object, size_t instance_idx, const Vec3d &normal);
 
+// The rotation that turns `normal` (object coordinates) to point down, given the instance's
+// rotation, scale and mirror. A face that already points straight up is turned over about X:
+// the axis setFromTwoVectors() picks there is arbitrary, and it lands the object at a random
+// angle in the plate plane.
+Transform3d lay_on_face_rotation(const Vec3d &normal, const Transform3d &instance_matrix_no_offset);
+
 } // namespace Slic3r

src/slic3r/GUI/Selection.cpp

@@ -17,6 +17,7 @@
 #include "libslic3r/Color.hpp"
 #include "libslic3r/Point.hpp"
 #include "libslic3r/Geometry.hpp"
+#include "libslic3r/LayOnFace.hpp"
 #include "slic3r/GUI/GLModel.hpp"
 #include <string>
 #include <vector>
@@ -1472,11 +1473,9 @@ void Selection::flattening_rotate(const Vec3d& normal)
 
     for (unsigned int i : m_list) {
         GLVolume& v = *(*m_volumes)[i];
-        // Normal transformed from the object coordinate space to the world coordinate space.
+        // Additional rotation to align the face normal with the down vector in the world coordinate space.
         const Geometry::Transformation& old_inst_trafo = v.get_instance_transformation();
-        const Vec3d tnormal = old_inst_trafo.get_matrix().matrix().block(0, 0, 3, 3).inverse().transpose() * normal;
-        // Additional rotation to align tnormal with the down vector in the world coordinate space.
-        const Transform3d rotation_matrix = Transform3d(Eigen::Quaterniond().setFromTwoVectors(tnormal, -Vec3d::UnitZ()));
+        const Transform3d rotation_matrix = lay_on_face_rotation(normal, old_inst_trafo.get_matrix_no_offset());
         v.set_instance_transformation(old_inst_trafo.get_offset_matrix() * rotation_matrix * old_inst_trafo.get_matrix_no_offset());
     }
 

tests/libslic3r/test_lay_on_face.cpp

@@ -187,6 +187,23 @@ TEST_CASE("A part rests on its largest hull face even when parallel inner faces
     check_on_bed(plate);
 }
 
+TEST_CASE("A box laid on its top face is turned over about X", "[LayOnFace]")
+{
+    // A face pointing straight up leaves the half turn's axis open to every horizontal
+    // direction. It must be the fixed X flip both times, keep the footprint and not follow
+    // rounding noise to an arbitrary angle.
+    Model        model;
+    ModelObject &box = add_box_object(model, { 40, 20, 10 });
+    lay_on_face(box, 0, { 0, 0, 1 });
+
+    check_size(box, { 40, 20, 10 });
+    check_on_bed(box);
+    const Matrix3d rotation = box.instances.front()->get_matrix_no_offset().linear();
+    CHECK_THAT(rotation(0, 0), WithinAbs(1., 1e-9));
+    CHECK_THAT(rotation(1, 1), WithinAbs(-1., 1e-9));
+    CHECK_THAT(rotation(2, 2), WithinAbs(-1., 1e-9));
+}
+
 TEST_CASE("Faces are selected in object coordinates whatever the instance rotation", "[LayOnFace]")
 {
     Model        model;