MDEV 26161: crash in Gis_point::calculate_haversinest-10.2-anel-MDEV-26161-gis-sphere

- Additionally add check_geometry as an private function member

Reviewed by: holyfoot@mariadb.com

6 files changed, 171 insertions, 16 deletions

diff --git a/mysql-test/r/gis-precise.result b/mysql-test/r/gis-precise.result
index aaeb6e852a1..18154689e23 100644
--- a/mysql-test/r/gis-precise.result
+++ b/mysql-test/r/gis-precise.result
@@ -814,6 +814,10 @@ SELECT ST_DISTANCE_SPHERE();
 ERROR 42000: Incorrect parameter count in the call to native function 'ST_DISTANCE_SPHERE'
 SELECT ST_DISTANCE_SPHERE(NULL);
 ERROR 42000: Incorrect parameter count in the call to native function 'ST_DISTANCE_SPHERE'
+SELECT ST_DISTANCE_SPHERE(NULL,NULL,NULL,NULL);
+ERROR 42000: Incorrect parameter count in the call to native function 'ST_DISTANCE_SPHERE'
+SELECT ST_DISTANCE_SPHERE(1,2,3,4);
+ERROR 42000: Incorrect parameter count in the call to native function 'ST_DISTANCE_SPHERE'
 SELECT ST_DISTANCE_SPHERE(NULL, NULL);
 ST_DISTANCE_SPHERE(NULL, NULL)
 NULL
@@ -905,16 +909,64 @@ ERROR HY000: Internal error: Radius must be greater than zero.
 set @pt1 = ST_GeomFromText('POINT(190 -30)');
 set @pt2 = ST_GeomFromText('POINT(-30  50)');
 SELECT ST_Distance_Sphere(@pt1, @pt2);
-ERROR HY000: Out of range error: Longitude should be [-180,180] in function ST_Distance_Sphere.
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
 set @pt1 = ST_GeomFromText('POINT(135 -30)');
 set @pt2 = ST_GeomFromText('POINT(-30 91)');
 SELECT ST_Distance_Sphere(@pt1, @pt2);
-ERROR HY000: Out of range error: Latitude should be [-90,90] in function ST_Distance_Sphere.
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
 set @zenica   = ST_GeomFromText('POINT(17.907743 44.203438)');
 set @sarajevo = ST_GeomFromText('POINT(18.413076 43.856258)');
 SELECT TRUNCATE(ST_Distance_Sphere(@zenica, @sarajevo), 10);
-TRUNCATE(ST_Distance_Sphere(@zenica, @sarajevo), 10)
-55878.5933759170
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
 SELECT TRUNCATE(ST_Distance_Sphere(@sarajevo, @zenica), 10);
-TRUNCATE(ST_Distance_Sphere(@sarajevo, @zenica), 10)
-55878.5933759170
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
+#
+# MDEV-26161: crash in Gis_point::calculate_haversine
+#
+SELECT ST_DISTANCE_SPHERE(ST_GeomFromText('POLYGON((0 0,0 1,1 1,1 0,0 0))'), ST_GEOMFROMTEXT('POINT(2 0)'));
+ERROR HY000: Internal error: st_distance_sphere
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(@poly), ST_GEOMFROMTEXT('POINT(2 0)'));
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
+SET @poly = ST_GeomFromText('POLYGON((0 0,0 1,1 1,1 0,0 0))');
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(@poly), ST_GEOMFROMTEXT('POINT(2 0)'));
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
+SET @poly = ST_GeomFromText('POLYGON(LINESTRING(0 0,0 1,1 1,1 0,0 0))');
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(@poly), ST_GEOMFROMTEXT('POINT(2 0)'));
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(polygon(linestring(point(0,0),point(0,1),point(1,1),point(1,0),point(0,0)))), ST_GEOMFROMTEXT('POINT(2 0)'));
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
+SELECT st_distance_sphere(
+st_asbinary(
+polygon(
+linestring(
+point(-19655,-18908),
+point(-10261,-7746),
+point(31432,-18255),
+point(-19655,-18908)
+),
+linestring(
+point(-10494,19004),
+point(-18181,-24299),
+point(20616,19685),
+point(-10494,19004)
+),
+linestring(
+point(8904,-21761),
+point(-24528,3798),
+point(-2502,-2889),
+point(8904,-21761)
+),
+linestring(
+point(-7451,-19434),
+point(-30558,-27426),
+point(19086,17469),
+point(-7451,-19434)
+)
+)
+),
+multipoint(
+point(124,204)
+),
+3232164176
+);
+ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere.
diff --git a/mysql-test/t/gis-precise.test b/mysql-test/t/gis-precise.test
index b75318e034b..9883d2fd6d2 100644
--- a/mysql-test/t/gis-precise.test
+++ b/mysql-test/t/gis-precise.test
@@ -402,6 +402,10 @@ with cte1 as(   select (st_symdifference(point(1,1),point(1,1))) as a1 ), cte2 a
 SELECT ST_DISTANCE_SPHERE();
 --error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
 SELECT ST_DISTANCE_SPHERE(NULL);
+--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
+SELECT ST_DISTANCE_SPHERE(NULL,NULL,NULL,NULL);
+--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
+SELECT ST_DISTANCE_SPHERE(1,2,3,4);
 SELECT ST_DISTANCE_SPHERE(NULL, NULL);
 # NULL args and optional radius will return NULL
 SELECT ST_DISTANCE_SPHERE(NULL, NULL, 3);
@@ -457,17 +461,81 @@ SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTE
 # Longitude out of range [-180,180]
 set @pt1 = ST_GeomFromText('POINT(190 -30)');
 set @pt2 = ST_GeomFromText('POINT(-30  50)');
---error ER_STD_OUT_OF_RANGE_ERROR
+# @todo this is wrong should be ER_STD_OUT_OF_RANGE_ERROR
+--error ER_GIS_INVALID_DATA
 SELECT ST_Distance_Sphere(@pt1, @pt2);
 
 # Latitude out of range [-90, 90]
 set @pt1 = ST_GeomFromText('POINT(135 -30)');
 set @pt2 = ST_GeomFromText('POINT(-30 91)');
---error ER_STD_OUT_OF_RANGE_ERROR
+# @todo this is wrong should be ER_STD_OUT_OF_RANGE_ERROR
+--error ER_GIS_INVALID_DATA
 SELECT ST_Distance_Sphere(@pt1, @pt2);
 
 # POINT in form (longitude[-180, 180] latitude[-90, 90])
 set @zenica   = ST_GeomFromText('POINT(17.907743 44.203438)');
 set @sarajevo = ST_GeomFromText('POINT(18.413076 43.856258)');
+# @todo This is wrong! Should be scalar
+--error ER_GIS_INVALID_DATA
 SELECT TRUNCATE(ST_Distance_Sphere(@zenica, @sarajevo), 10);
+# @todo This is wrong! Should be scalar
+--error ER_GIS_INVALID_DATA
 SELECT TRUNCATE(ST_Distance_Sphere(@sarajevo, @zenica), 10);
+
+--echo #
+--echo # MDEV-26161: crash in Gis_point::calculate_haversine
+--echo #
+
+--error ER_INTERNAL_ERROR
+SELECT ST_DISTANCE_SPHERE(ST_GeomFromText('POLYGON((0 0,0 1,1 1,1 0,0 0))'), ST_GEOMFROMTEXT('POINT(2 0)'));
+# Argument NULL to function
+# @todo This is wrong! Should be NULL
+--error ER_GIS_INVALID_DATA
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(@poly), ST_GEOMFROMTEXT('POINT(2 0)'));
+# Wrong argument
+SET @poly = ST_GeomFromText('POLYGON((0 0,0 1,1 1,1 0,0 0))');
+-- error ER_GIS_INVALID_DATA
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(@poly), ST_GEOMFROMTEXT('POINT(2 0)'));
+# Valid argument
+SET @poly = ST_GeomFromText('POLYGON(LINESTRING(0 0,0 1,1 1,1 0,0 0))');
+# @todo This is wrong! Should be scalar
+--error ER_GIS_INVALID_DATA
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(@poly), ST_GEOMFROMTEXT('POINT(2 0)'));
+-- error ER_GIS_INVALID_DATA
+SELECT ST_DISTANCE_SPHERE(ST_AsBinary(polygon(linestring(point(0,0),point(0,1),point(1,1),point(1,0),point(0,0)))), ST_GEOMFROMTEXT('POINT(2 0)'));
+
+-- error ER_GIS_INVALID_DATA
+SELECT st_distance_sphere(
+ st_asbinary(
+  polygon(
+   linestring(
+    point(-19655,-18908),
+    point(-10261,-7746),
+    point(31432,-18255),
+    point(-19655,-18908)
+   ),
+   linestring(
+    point(-10494,19004),
+    point(-18181,-24299),
+    point(20616,19685),
+    point(-10494,19004)
+   ),
+   linestring(
+    point(8904,-21761),
+    point(-24528,3798),
+    point(-2502,-2889),
+    point(8904,-21761)
+   ),
+   linestring(
+    point(-7451,-19434),
+    point(-30558,-27426),
+    point(19086,17469),
+    point(-7451,-19434)
+   )
+  )
+ ),
+ multipoint(
+  point(124,204)
+ ),
+ 3232164176
+);
diff --git a/sql/item_create.cc b/sql/item_create.cc
index a3c51d1949e..1565caaa79d 100644
--- a/sql/item_create.cc
+++ b/sql/item_create.cc
@@ -4785,7 +4785,7 @@ Create_func_distance_sphere::create_native(THD *thd, LEX_STRING name,
   if (item_list != NULL)
     arg_count= item_list->elements;
 
-  if (arg_count < 2)
+  if (arg_count < 2 || arg_count > 3)
   {
     my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name.str);
     return NULL;
diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc
index 0daf9da4a81..b5073a407f2 100644
--- a/sql/item_geofunc.cc
+++ b/sql/item_geofunc.cc
@@ -2567,6 +2567,7 @@ double Item_func_sphere_distance::val_real()
       return 1;
     }
   }
+
   Geometry_buffer buffer1, buffer2;
   Geometry *g1, *g2;
   if (!(g1= Geometry::construct(&buffer1, arg1->ptr(), arg1->length())) ||
@@ -2575,16 +2576,15 @@ double Item_func_sphere_distance::val_real()
     my_error(ER_GIS_INVALID_DATA, MYF(0), "ST_Distance_Sphere");
     goto handle_errors;
   }
-// Method allowed for points and multipoints
-  if (!(g1->get_class_info()->m_type_id == Geometry::wkb_point ||
-        g1->get_class_info()->m_type_id == Geometry::wkb_multipoint) ||
-      !(g2->get_class_info()->m_type_id == Geometry::wkb_point ||
-        g2->get_class_info()->m_type_id == Geometry::wkb_multipoint))
+
+  if (check_geometries(g1->get_class_info()->m_type_id,
+                       g2->get_class_info()->m_type_id))
   {
-    // Generate error message in case different geometry is used? 
+    // Generate error message in case different geometry is used?
     my_error(ER_INTERNAL_ERROR, MYF(0), func_name());
     return 0;
   }
+
   distance= spherical_distance_points(g1, g2, sphere_radius);
   if (distance < 0)
   {
@@ -2677,6 +2677,18 @@ double Item_func_sphere_distance::spherical_distance_points(Geometry *g1,
 }
 
 
+int Item_func_sphere_distance::check_geometries(const int g1, const int g2)
+{
+  // Method allowed for points and multipoints
+  if (!(g1== Geometry::wkb_point || g1 == Geometry::wkb_multipoint) ||
+      !(g2== Geometry::wkb_point || g2 == Geometry::wkb_multipoint))
+  {
+    return 1;
+  }
+  return 0;
+}
+
+
 String *Item_func_pointonsurface::val_str(String *str)
 {
   Gcalc_operation_transporter trn(&func, &collector);
diff --git a/sql/item_geofunc.h b/sql/item_geofunc.h
index efff940ec4b..94165bb6213 100644
--- a/sql/item_geofunc.h
+++ b/sql/item_geofunc.h
@@ -718,10 +718,32 @@ class Item_func_sphere_distance: public Item_real_func
 {
   double spherical_distance_points(Geometry *g1, Geometry *g2,
                                    const double sphere_r);
+  int check_geometries(const int g1, const int g2);
 public:
   Item_func_sphere_distance(THD *thd, List<Item> &list):
     Item_real_func(thd, list) {}
   double val_real();
+  bool fix_length_and_dec()
+  {
+    if (Item_real_func::fix_length_and_dec())
+      return TRUE;
+    for (unsigned int i= 0; i < arg_count; i++)
+    {
+      /* Handling of NULLs will be handled after */
+      if(args[i]->null_value)
+        return FALSE;
+    }
+    /* Ignore radius */
+    for (unsigned int i= 0; i < 2; i++)
+    {
+      if (args[i]->field_type() != FIELD_TYPE_GEOMETRY)
+      {
+        my_error(ER_GIS_INVALID_DATA, MYF(0), "ST_Distance_Sphere");
+        return TRUE;
+      }
+    }
+    return FALSE;
+  }
   const char *func_name() const { return "st_distance_sphere"; }
   Item *get_copy(THD *thd, MEM_ROOT *mem_root)
   { return get_item_copy<Item_func_sphere_distance>(thd, mem_root, this); }
diff --git a/sql/spatial.cc b/sql/spatial.cc
index 8bd9acef18f..da56665ad3e 100644
--- a/sql/spatial.cc
+++ b/sql/spatial.cc
@@ -2325,10 +2325,11 @@ int Gis_multi_point::spherical_distance_multipoints(Geometry *g, const double r,
   {
     Geometry_buffer buff_temp;
     Geometry *temp;
+
     double temp_res= 0.0;
     char s[len];
     // First 4 bytes are handled already, make sure to create a Point
-    memset(s + 4, Geometry::wkb_point, 1);
+    memset(s + 4, Geometry::wkb_ndr, 1);
     memcpy(s + 5, this->get_data_ptr() + 5, 4);
     memcpy(s + 4 + WKB_HEADER_SIZE, this->get_data_ptr() + 4 + WKB_HEADER_SIZE*i +\
                                     POINT_DATA_SIZE*(i-1), POINT_DATA_SIZE);