Merge branch 'wip/carlosg/ttl-parser-improvements' into 'master'

Wip/carlosg/ttl parser improvements

Closes #260

See merge request GNOME/tracker!323

8 files changed, 56 insertions, 28 deletions

diff --git a/docs/tools/ttl_loader.c b/docs/tools/ttl_loader.c
index dfa9ce5db..08b634588 100644
--- a/docs/tools/ttl_loader.c
+++ b/docs/tools/ttl_loader.c
@@ -390,7 +390,7 @@ ttl_loader_load_ontology (Ontology    *ontology,
 	while (inner_error == NULL &&
 	       tracker_turtle_reader_next (reader,
 	                                   &subject, &predicate, &object,
-	                                   NULL, &inner_error)) {
+	                                   NULL, NULL, &inner_error)) {
 		load_in_memory (ontology, subject, predicate, object);
 	}
 
@@ -429,7 +429,7 @@ ttl_loader_load_description (GFile *file)
 	while (error == NULL &&
 	       tracker_turtle_reader_next (reader,
 	                                   &subject, &predicate, &object,
-	                                   NULL, &error)) {
+	                                   NULL, NULL, &error)) {
 		load_description (desc, subject, predicate, object);
 	}
 
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index a5e077041..bbb4d2e0e 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -1781,7 +1781,7 @@ load_ontology_file (TrackerDataManager  *manager,
 
 	while (tracker_turtle_reader_next (reader,
 	                                   &subject, &predicate, &object,
-	                                   NULL, &ttl_error)) {
+	                                   NULL, NULL, &ttl_error)) {
 		GError *ontology_error = NULL;
 
 		tracker_data_ontology_load_statement (manager, ontology_uri,
@@ -1829,7 +1829,7 @@ get_ontology_from_file (TrackerDataManager *manager,
 
 	while (tracker_turtle_reader_next (reader,
 	                                   &subject, &predicate, &object,
-	                                   NULL, &error)) {
+	                                   NULL, NULL, &error)) {
 		if (g_strcmp0 (predicate, RDF_TYPE) == 0) {
 			if (g_strcmp0 (object, TRACKER_PREFIX_NRL "Ontology") == 0) {
 				TrackerOntology *ontology;
@@ -2007,7 +2007,7 @@ import_ontology_file (TrackerDataManager *manager,
 
 	while (tracker_turtle_reader_next (reader,
 	                                   &subject, &predicate, &object,
-	                                   NULL, &error)) {
+	                                   NULL, NULL, &error)) {
 		tracker_data_ontology_process_statement (manager,
 		                                         subject, predicate, object,
 		                                         in_update);
diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c
index 86a707dde..3fe39842f 100644
--- a/src/libtracker-data/tracker-data-update.c
+++ b/src/libtracker-data/tracker-data-update.c
@@ -2674,7 +2674,7 @@ tracker_data_load_turtle_file (TrackerData  *data,
 {
 	TrackerTurtleReader *reader = NULL;
 	GError *inner_error = NULL;
-	const gchar *subject, *predicate, *object_str;
+	const gchar *subject, *predicate, *object_str, *langtag;
 	gboolean object_is_uri;
 
 	reader = tracker_turtle_reader_new_for_file (file, &inner_error);
@@ -2685,11 +2685,12 @@ tracker_data_load_turtle_file (TrackerData  *data,
 	                                   &subject,
 	                                   &predicate,
 	                                   &object_str,
+	                                   &langtag,
 	                                   &object_is_uri,
 	                                   &inner_error)) {
 		GBytes *object;
 
-		object = g_bytes_new (object_str, strlen (object_str) + 1);
+		object = tracker_sparql_make_langstring (object_str, langtag);
 
 		if (object_is_uri) {
 			tracker_data_insert_statement_with_uri (data, graph,
diff --git a/src/libtracker-data/tracker-sparql.c b/src/libtracker-data/tracker-sparql.c
index 2ff43d045..3bee2b34f 100644
--- a/src/libtracker-data/tracker-sparql.c
+++ b/src/libtracker-data/tracker-sparql.c
@@ -8746,20 +8746,9 @@ translate_RDFLiteral (TrackerSparql  *sparql,
 	if (is_parameter) {
 		binding = tracker_parameter_binding_new (str, NULL);
 	} else {
-		GString *langstr;
 		GBytes *bytes;
 
-		langstr = g_string_new (str);
-
-		if (langtag) {
-			g_string_append_c (langstr, '\0');
-			g_string_append_printf (langstr, "%s", &langtag[1]);
-		}
-
-		bytes = g_bytes_new_take (langstr->str,
-		                          langstr->len + 1);
-		g_string_free (langstr, FALSE);
-
+		bytes = tracker_sparql_make_langstring (str, langtag);
 		binding = tracker_literal_binding_new (bytes, NULL);
 		g_bytes_unref (bytes);
 	}
@@ -9566,3 +9555,24 @@ tracker_sparql_execute_update (TrackerSparql  *sparql,
 
 	return NULL;
 }
+
+GBytes *
+tracker_sparql_make_langstring (const gchar *str,
+                                const gchar *langtag)
+{
+	GString *langstr;
+	GBytes *bytes;
+
+	langstr = g_string_new (str);
+
+	if (langtag) {
+		g_string_append_c (langstr, '\0');
+		g_string_append_printf (langstr, "%s", &langtag[1]);
+	}
+
+	bytes = g_bytes_new_take (langstr->str,
+	                          langstr->len + 1);
+	g_string_free (langstr, FALSE);
+
+	return bytes;
+}
diff --git a/src/libtracker-data/tracker-sparql.h b/src/libtracker-data/tracker-sparql.h
index e86665c51..57651de1a 100644
--- a/src/libtracker-data/tracker-sparql.h
+++ b/src/libtracker-data/tracker-sparql.h
@@ -45,4 +45,7 @@ GVariant *      tracker_sparql_execute_update (TrackerSparql  *sparql,
 					       gboolean        blank,
 					       GError        **error);
 
+GBytes * tracker_sparql_make_langstring (const gchar *str,
+                                         const gchar *langtag);
+
 #endif /* __TRACKER_SPARQL_H__ */
diff --git a/src/libtracker-data/tracker-turtle-reader.c b/src/libtracker-data/tracker-turtle-reader.c
index 545d95475..50cde79b4 100644
--- a/src/libtracker-data/tracker-turtle-reader.c
+++ b/src/libtracker-data/tracker-turtle-reader.c
@@ -55,6 +55,7 @@ struct _TrackerTurtleReader {
 	gchar *subject;
 	gchar *predicate;
 	gchar *object;
+	gchar *object_lang;
 	gboolean object_is_uri;
 	ParserState state;
 };
@@ -229,6 +230,7 @@ pop_stack (TrackerTurtleReader *reader)
 		/* Restore the old subject as current object */
 		reader->object = s;
 		reader->object_is_uri = TRUE;
+		g_clear_pointer (&reader->object_lang, g_free);
 		s = NULL;
 	} else if (reader->state == STATE_SUBJECT) {
 		g_clear_pointer (&reader->subject, g_free);
@@ -294,6 +296,8 @@ parse_terminal (TrackerTurtleReader  *reader,
 
 	if (out)
 		*out = str;
+	else
+		g_free (str);
 
 	return TRUE;
 }
@@ -484,7 +488,7 @@ tracker_turtle_reader_iterate_next (TrackerTurtleReader  *reader,
                                     GError              **error)
 {
 	while (TRUE) {
-		gchar *str;
+		gchar *str, *lang;
 
 		advance_whitespace (reader);
 
@@ -563,6 +567,7 @@ tracker_turtle_reader_iterate_next (TrackerTurtleReader  *reader,
 			break;
 		case STATE_OBJECT:
 			g_clear_pointer (&reader->object, g_free);
+			g_clear_pointer (&reader->object_lang, g_free);
 			reader->object_is_uri = FALSE;
 
 			if (parse_token (reader, "[")) {
@@ -585,18 +590,24 @@ tracker_turtle_reader_iterate_next (TrackerTurtleReader  *reader,
 				reader->object = generate_bnode (reader, str);
 				reader->object_is_uri = TRUE;
 				g_free (str);
-			} else if (parse_terminal (reader, terminal_STRING_LITERAL1, 1, &str) ||
-			           parse_terminal (reader, terminal_STRING_LITERAL2, 1, &str)) {
-				reader->object = g_strcompress (str);
-				g_free (str);
-				if (!handle_type_cast (reader, error))
-					return FALSE;
 			} else if (parse_terminal (reader, terminal_STRING_LITERAL_LONG1, 3, &str) ||
 			           parse_terminal (reader, terminal_STRING_LITERAL_LONG2, 3, &str)) {
 				reader->object = g_strcompress (str);
 				g_free (str);
-				if (!handle_type_cast (reader, error))
-					return FALSE;
+				if (parse_terminal (reader, terminal_LANGTAG, 0, &lang)) {
+					reader->object_lang = lang;
+				} else if (!handle_type_cast (reader, error)) {
+						return FALSE;
+				}
+			} else if (parse_terminal (reader, terminal_STRING_LITERAL1, 1, &str) ||
+			           parse_terminal (reader, terminal_STRING_LITERAL2, 1, &str)) {
+				reader->object = g_strcompress (str);
+				g_free (str);
+				if (parse_terminal (reader, terminal_LANGTAG, 0, &lang)) {
+					reader->object_lang = lang;
+				} else if (!handle_type_cast (reader, error)) {
+						return FALSE;
+				}
 			} else if (parse_terminal (reader, terminal_DOUBLE, 0, &str) ||
 			           parse_terminal (reader, terminal_INTEGER, 0, &str)) {
 				reader->object = str;
@@ -658,6 +669,7 @@ tracker_turtle_reader_next (TrackerTurtleReader  *reader,
                             const gchar         **subject,
                             const gchar         **predicate,
                             const gchar         **object,
+                            const gchar         **object_lang,
                             gboolean             *object_is_uri,
                             GError              **error)
 {
diff --git a/src/libtracker-data/tracker-turtle-reader.h b/src/libtracker-data/tracker-turtle-reader.h
index 892d2c46a..2d7846bf0 100644
--- a/src/libtracker-data/tracker-turtle-reader.h
+++ b/src/libtracker-data/tracker-turtle-reader.h
@@ -38,6 +38,7 @@ gboolean tracker_turtle_reader_next (TrackerTurtleReader  *reader,
                                      const gchar         **subject,
                                      const gchar         **predicate,
                                      const gchar         **object,
+                                     const gchar         **object_lang,
                                      gboolean             *object_is_uri,
                                      GError              **error);
 
diff --git a/tests/functional-tests/test-data/test-movie.ttl b/tests/functional-tests/test-data/test-movie.ttl
index 6e2483341..e7a3a3303 100644
--- a/tests/functional-tests/test-data/test-movie.ttl
+++ b/tests/functional-tests/test-data/test-movie.ttl
@@ -7,6 +7,7 @@ _:3 a nfo:FileHash ;
   nfo:hashAlgorithm "gibest" .
 
 <file:///home/sam/Videos/My%20Movie.avi> a nmm:Video , nfo:FileDataObject ;
+  nie:title """It"""@en ;
   nfo:frameRate 25 ;
   nfo:channels 2 ;
   nfo:hasHash _:3 ;