MDEV-26161 crash in Gis_point::calculate_haversine

return an error on invalid gis data

4 files changed, 27 insertions, 12 deletions

diff --git a/mysql-test/main/gis.result b/mysql-test/main/gis.result
index bfe1d3f40a5..3d8c64b0ce8 100644
--- a/mysql-test/main/gis.result
+++ b/mysql-test/main/gis.result
@@ -4977,5 +4977,10 @@ ERROR HY000: Illegal parameter data type geometry for operation 'is_free_lock'
 SELECT IS_USED_LOCK(POINT(1,1));
 ERROR HY000: Illegal parameter data type geometry for operation 'is_used_lock'
 #
+# MDEV-26161 crash in Gis_point::calculate_haversine
+#
+select st_distance_sphere(x'01030000000400000004000000000000', multipoint(point(124,204)), 10);
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
+#
 # End of 10.3 tests
 #
diff --git a/mysql-test/main/gis.test b/mysql-test/main/gis.test
index c7bdb366124..1d202e9be08 100644
--- a/mysql-test/main/gis.test
+++ b/mysql-test/main/gis.test
@@ -3090,6 +3090,11 @@ SELECT IS_FREE_LOCK(POINT(1,1));
 --error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION
 SELECT IS_USED_LOCK(POINT(1,1));
 
+--echo #
+--echo # MDEV-26161 crash in Gis_point::calculate_haversine
+--echo #
+--error ER_CANT_CREATE_GEOMETRY_OBJECT
+select st_distance_sphere(x'01030000000400000004000000000000', multipoint(point(124,204)), 10);
 
 --echo #
 --echo # End of 10.3 tests
diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc
index b40105aaf36..87c736ab94b 100644
--- a/sql/item_geofunc.cc
+++ b/sql/item_geofunc.cc
@@ -2596,7 +2596,7 @@ double Item_func_sphere_distance::spherical_distance_points(Geometry *g1,
   double res= 0.0;
    // Length for the single point (25 Bytes)
   uint32 len= SRID_SIZE + POINT_DATA_SIZE + WKB_HEADER_SIZE;
-  int error= 0;
+  int err_hv= 0, err_sph= 0;
 
   switch (g2->get_class_info()->m_type_id)
   {
@@ -2606,21 +2606,21 @@ double Item_func_sphere_distance::spherical_distance_points(Geometry *g1,
       // Optimization for point-point case
       if (g1->get_class_info()->m_type_id == Geometry::wkb_point)
       {
-        res= g2p->calculate_haversine(g1, r, &error);
+        res= g2p->calculate_haversine(g1, r, &err_hv);
       }
       else
       {
         // Optimization for single point in Multipoint
         if (g1->get_data_size() == len)
         {
-          res= g2p->calculate_haversine(g1, r, &error);
+          res= g2p->calculate_haversine(g1, r, &err_hv);
         }
         else
         {
           // There are multipoints in g1
           // g1 is MultiPoint and calculate MP.sphericaldistance from g2 Point
           if (g1->get_data_size() != GET_SIZE_ERROR)
-            g2p->spherical_distance_multipoints(g1, r, &res, &error);
+            err_sph= g2p->spherical_distance_multipoints(g1, r, &res, &err_hv);
         }
       }
       break;
@@ -2634,20 +2634,20 @@ double Item_func_sphere_distance::spherical_distance_points(Geometry *g1,
          // Optimization for single point in Multipoint g2
         if (g2->get_data_size() == len)
         {
-          res= g1p->calculate_haversine(g2, r, &error);
+          res= g1p->calculate_haversine(g2, r, &err_hv);
         }
         else
         {
           if (g2->get_data_size() != GET_SIZE_ERROR)
           // g1 is a point (casted to multi_point) and g2 multipoint
-            g1p->spherical_distance_multipoints(g2, r, &res, &error);
+            err_sph= g1p->spherical_distance_multipoints(g2, r, &res, &err_hv);
         }
       }
       else
       {
         Gis_multi_point *g1mp= static_cast<Gis_multi_point *>(g1);
         // Multipoints in g1 and g2 - no optimization
-        g1mp->spherical_distance_multipoints(g2, r, &res, &error);
+        err_sph= g1mp->spherical_distance_multipoints(g2, r, &res, &err_hv);
       }
       break;
 
@@ -2656,12 +2656,14 @@ double Item_func_sphere_distance::spherical_distance_points(Geometry *g1,
       break;
   }
 
-  if (error > 0)
+  if (err_hv > 0)
     my_error(ER_STD_OUT_OF_RANGE_ERROR, MYF(0),
              "Longitude should be [-180,180]", "ST_Distance_Sphere");
-  else if(error < 0)
+  else if(err_hv < 0)
     my_error(ER_STD_OUT_OF_RANGE_ERROR, MYF(0),
              "Latitude should be [-90,90]", "ST_Distance_Sphere");
+  else if (err_sph)
+    my_error(ER_CANT_CREATE_GEOMETRY_OBJECT, MYF(0));
   return res;
 }
 
diff --git a/sql/spatial.cc b/sql/spatial.cc
index 53e8c4c8bdd..6e044a67161 100644
--- a/sql/spatial.cc
+++ b/sql/spatial.cc
@@ -1151,7 +1151,8 @@ int Gis_point::spherical_distance_multipoints(Geometry *g, const double r,
                                     POINT_DATA_SIZE*(i-1), POINT_DATA_SIZE);
     s[len-1]= '\0';
     temp= Geometry::construct(&buff_temp, s, len);
-    DBUG_ASSERT(temp);
+    if (!temp)
+      return 1;
     temp_res= this->calculate_haversine(temp, r, err);
     if (res > temp_res)
       res= temp_res;
@@ -2335,7 +2336,8 @@ int Gis_multi_point::spherical_distance_multipoints(Geometry *g, const double r,
                                     POINT_DATA_SIZE*(i-1), POINT_DATA_SIZE);
     s[len-1]= '\0';
     temp= Geometry::construct(&buff_temp, s, len);
-    DBUG_ASSERT(temp);
+    if (!temp)
+      return 1;
     // Optimization for single Multipoint
     if (num_of_points2 == 1)
     {
@@ -2354,7 +2356,8 @@ int Gis_multi_point::spherical_distance_multipoints(Geometry *g, const double r,
                                        POINT_DATA_SIZE*(j-1), POINT_DATA_SIZE);
       s2[len-1]= '\0';
       temp2= Geometry::construct(&buff_temp2, s2, len);
-      DBUG_ASSERT(temp2);
+      if (!temp2)
+        return 1;
       temp_res= static_cast<Gis_point *>(temp)->calculate_haversine(temp2, r, err);
       if (res > temp_res)
         res= temp_res;