diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/ephy-prefs.h | 4 | ||||
-rw-r--r-- | lib/ephy-profile-utils.h | 4 | ||||
-rw-r--r-- | lib/meson.build | 4 | ||||
-rw-r--r-- | lib/safe-browsing/ephy-gsb-service.c | 767 | ||||
-rw-r--r-- | lib/safe-browsing/ephy-gsb-service.h | 41 | ||||
-rw-r--r-- | lib/safe-browsing/ephy-gsb-storage.c | 1710 | ||||
-rw-r--r-- | lib/safe-browsing/ephy-gsb-storage.h | 68 | ||||
-rw-r--r-- | lib/safe-browsing/ephy-gsb-utils.c | 919 | ||||
-rw-r--r-- | lib/safe-browsing/ephy-gsb-utils.h | 98 |
9 files changed, 1 insertions, 3614 deletions
diff --git a/lib/ephy-prefs.h b/lib/ephy-prefs.h index 05bbb2399..e2e81edc2 100644 --- a/lib/ephy-prefs.h +++ b/lib/ephy-prefs.h @@ -114,9 +114,7 @@ static const char * const ephy_prefs_state_schema[] = { #define EPHY_PREFS_WEB_ENABLE_ADBLOCK "enable-adblock" #define EPHY_PREFS_WEB_REMEMBER_PASSWORDS "remember-passwords" #define EPHY_PREFS_WEB_ENABLE_SITE_SPECIFIC_QUIRKS "enable-site-specific-quirks" -#define EPHY_PREFS_WEB_ENABLE_SAFE_BROWSING "enable-safe-browsing" #define EPHY_PREFS_WEB_ENABLE_ITP "enable-itp" -#define EPHY_PREFS_WEB_GSB_API_KEY "gsb-api-key" #define EPHY_PREFS_WEB_DEFAULT_ZOOM_LEVEL "default-zoom-level" #define EPHY_PREFS_WEB_ENABLE_AUTOSEARCH "enable-autosearch" #define EPHY_PREFS_WEB_ENABLE_MOUSE_GESTURES "enable-mouse-gestures" @@ -144,9 +142,7 @@ static const char * const ephy_prefs_web_schema[] = { EPHY_PREFS_WEB_ENABLE_ADBLOCK, EPHY_PREFS_WEB_REMEMBER_PASSWORDS, EPHY_PREFS_WEB_ENABLE_SITE_SPECIFIC_QUIRKS, - EPHY_PREFS_WEB_ENABLE_SAFE_BROWSING, EPHY_PREFS_WEB_ENABLE_ITP, - EPHY_PREFS_WEB_GSB_API_KEY, EPHY_PREFS_WEB_DEFAULT_ZOOM_LEVEL, EPHY_PREFS_WEB_ENABLE_AUTOSEARCH, EPHY_PREFS_WEB_ENABLE_MOUSE_GESTURES, diff --git a/lib/ephy-profile-utils.h b/lib/ephy-profile-utils.h index c97910063..caa6bbea6 100644 --- a/lib/ephy-profile-utils.h +++ b/lib/ephy-profile-utils.h @@ -24,7 +24,7 @@ G_BEGIN_DECLS -#define EPHY_PROFILE_MIGRATION_VERSION 35 +#define EPHY_PROFILE_MIGRATION_VERSION 36 #define EPHY_INSECURE_PASSWORDS_MIGRATION_VERSION 11 #define EPHY_FIREFOX_SYNC_PASSWORDS_MIGRATION_VERSION 19 #define EPHY_TARGET_ORIGIN_MIGRATION_VERSION 21 @@ -33,8 +33,6 @@ G_BEGIN_DECLS #define EPHY_BOOKMARKS_FILE "bookmarks.gvdb" #define EPHY_HISTORY_FILE "ephy-history.db" -/* Threat list database for Google Safe Browsing. */ -#define EPHY_GSB_FILE "gsb-threats.db" int ephy_profile_utils_get_migration_version (void); int ephy_profile_utils_get_migration_version_for_profile_dir (const char *profile_directory); diff --git a/lib/meson.build b/lib/meson.build index 616e7da0a..04b5bbf8a 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -44,9 +44,6 @@ libephymisc_sources = [ 'history/ephy-history-service-urls-table.c', 'history/ephy-history-service-visits-table.c', 'history/ephy-history-types.c', - 'safe-browsing/ephy-gsb-service.c', - 'safe-browsing/ephy-gsb-storage.c', - 'safe-browsing/ephy-gsb-utils.c', enums ] @@ -74,7 +71,6 @@ libephymisc_includes = include_directories( '..', 'contrib', 'history', - 'safe-browsing' ) libephymisc = shared_library('ephymisc', diff --git a/lib/safe-browsing/ephy-gsb-service.c b/lib/safe-browsing/ephy-gsb-service.c deleted file mode 100644 index 98d23d3fa..000000000 --- a/lib/safe-browsing/ephy-gsb-service.c +++ /dev/null @@ -1,767 +0,0 @@ -/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* - * Copyright © 2017 Gabriel Ivascu <gabrielivascu@gnome.org> - * - * This file is part of Epiphany. - * - * Epiphany is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Epiphany is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Epiphany. If not, see <http://www.gnu.org/licenses/>. - */ - -#include "config.h" -#include "ephy-gsb-service.h" - -#include "ephy-debug.h" -#include "ephy-gsb-storage.h" -#include "ephy-user-agent.h" - -#include <libsoup/soup.h> -#include <math.h> -#include <stdio.h> -#include <string.h> - -#define API_PREFIX "https://safebrowsing.googleapis.com/v4/" - -/* See comment in ephy_gsb_service_schedule_update(). */ -#define JITTER 2 /* seconds */ -#define CURRENT_TIME (g_get_real_time () / 1000000) /* seconds */ -#define DEFAULT_WAIT_TIME (30 * 60) /* seconds */ - -struct _EphyGSBService { - GObject parent_instance; - - char *api_key; - EphyGSBStorage *storage; - - gboolean is_updating; - guint source_id; - - gint64 next_full_hashes_time; - gint64 next_list_updates_time; - gint64 back_off_exit_time; - gint64 back_off_num_fails; - - SoupSession *session; -}; - -G_DEFINE_TYPE (EphyGSBService, ephy_gsb_service, G_TYPE_OBJECT); - -enum { - PROP_0, - PROP_API_KEY, - PROP_GSB_STORAGE, - LAST_PROP -}; - -static GParamSpec *obj_properties[LAST_PROP]; - -enum { - UPDATE_FINISHED, - LAST_SIGNAL -}; - -static guint signals[LAST_SIGNAL]; - -static gboolean ephy_gsb_service_update (EphyGSBService *self); - -static inline gboolean -json_object_has_non_null_string_member (JsonObject *object, - const char *member) -{ - JsonNode *node; - - node = json_object_get_member (object, member); - if (!node || !JSON_NODE_HOLDS_VALUE (node)) - return FALSE; - - return json_node_get_string (node) != NULL; -} - -static inline gboolean -json_object_has_non_null_array_member (JsonObject *object, - const char *member) -{ - JsonNode *node; - - node = json_object_get_member (object, member); - if (!node) - return FALSE; - - return JSON_NODE_HOLDS_ARRAY (node); -} - -/* - * https://developers.google.com/safe-browsing/v4/request-frequency#back-off-mode - */ -static inline void -ephy_gsb_service_update_back_off_mode (EphyGSBService *self) -{ - gint64 duration; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - - duration = (1 << self->back_off_num_fails++) * 15 * 60 * (g_random_double () + 1); - self->back_off_exit_time = CURRENT_TIME + MIN (duration, 24 * 60 * 60); - - ephy_gsb_storage_set_metadata (self->storage, "back_off_exit_time", self->back_off_exit_time); - ephy_gsb_storage_set_metadata (self->storage, "back_off_num_fails", self->back_off_num_fails); - - LOG ("Set back-off mode for %ld seconds", duration); -} - -static inline void -ephy_gsb_service_reset_back_off_mode (EphyGSBService *self) -{ - g_assert (EPHY_IS_GSB_SERVICE (self)); - - self->back_off_num_fails = self->back_off_exit_time = 0; -} - -static inline gboolean -ephy_gsb_service_is_back_off_mode (EphyGSBService *self) -{ - g_assert (EPHY_IS_GSB_SERVICE (self)); - - return self->back_off_num_fails > 0 && self->back_off_exit_time > CURRENT_TIME; -} - -static void -ephy_gsb_service_schedule_update (EphyGSBService *self) -{ - gint64 interval; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - g_assert (ephy_gsb_storage_is_operable (self->storage)); - - /* This function should only be called when self->next_list_updates_time is - * greater than CURRENT_TIME. However, asserting (self->next_list_updates_time - * - CURRENT_TIME) to be greater than 0 can be faulty in the (very rare, but - * not impossible) case when the value returned by CURRENT_TIME changes while - * calling this function to become equal to self->next_list_updates_time, i.e. - * when opening Epiphany at the exact same second as next_list_updates_time - * value read from disk. To prevent a crash in that situation, add a jitter - * value to the difference between next_list_updates_time and CURRENT_TIME. - */ - interval = self->next_list_updates_time - CURRENT_TIME + JITTER; - g_assert (interval > 0); - - self->source_id = g_timeout_add_seconds (interval, - (GSourceFunc)ephy_gsb_service_update, - self); - g_source_set_name_by_id (self->source_id, "[epiphany] gsb_service_update"); - - LOG ("Next update scheduled in %ld seconds", interval); -} - -static void -ephy_gsb_service_update_thread (GTask *task, - EphyGSBService *self, - gpointer task_data, - GCancellable *cancellable) -{ - JsonNode *body_node = NULL; - JsonObject *body_obj; - JsonArray *responses; - SoupMessage *msg = NULL; - GList *threat_lists = NULL; - char *url = NULL; - char *body; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - g_assert (ephy_gsb_storage_is_operable (self->storage)); - - /* Set up a default next update time in case of failure or non-existent - * minimum wait duration. - */ - self->next_list_updates_time = CURRENT_TIME + DEFAULT_WAIT_TIME; - - ephy_gsb_storage_delete_old_full_hashes (self->storage); - - threat_lists = ephy_gsb_storage_get_threat_lists (self->storage); - if (!threat_lists) { - LOG ("No threat lists to update"); - goto out; - } - - body = ephy_gsb_utils_make_list_updates_request (threat_lists); - url = g_strdup_printf ("%sthreatListUpdates:fetch?key=%s", API_PREFIX, self->api_key); - msg = soup_message_new (SOUP_METHOD_POST, url); - soup_message_set_request (msg, "application/json", SOUP_MEMORY_TAKE, body, strlen (body)); - soup_session_send_message (self->session, msg); - - /* Handle unsuccessful responses. */ - if (msg->status_code != 200) { - LOG ("Cannot update threat lists, got: %u, %s", msg->status_code, msg->response_body->data); - ephy_gsb_service_update_back_off_mode (self); - self->next_list_updates_time = self->back_off_exit_time; - goto out; - } - - /* Successful response, reset back-off mode. */ - ephy_gsb_service_reset_back_off_mode (self); - - body_node = json_from_string (msg->response_body->data, NULL); - if (!body_node || !JSON_NODE_HOLDS_OBJECT (body_node)) { - g_warning ("Response is not a valid JSON object"); - goto out; - } - - body_obj = json_node_get_object (body_node); - responses = json_object_get_array_member (body_obj, "listUpdateResponses"); - - for (guint i = 0; i < json_array_get_length (responses); i++) { - EphyGSBThreatList *list; - JsonObject *lur = json_array_get_object_element (responses, i); - const char *type = json_object_get_string_member (lur, "responseType"); - JsonObject *checksum = json_object_get_object_member (lur, "checksum"); - const char *remote_checksum = json_object_get_string_member (checksum, "sha256"); - char *local_checksum; - - list = ephy_gsb_threat_list_new (json_object_get_string_member (lur, "threatType"), - json_object_get_string_member (lur, "platformType"), - json_object_get_string_member (lur, "threatEntryType"), - json_object_get_string_member (lur, "newClientState")); - LOG ("Updating list %s/%s/%s", list->threat_type, list->platform_type, list->threat_entry_type); - - /* If full update, clear all previous hash prefixes for the given list. */ - if (!g_strcmp0 (type, "FULL_UPDATE")) { - LOG ("FULL UPDATE, clearing all previous hash prefixes..."); - ephy_gsb_storage_clear_hash_prefixes (self->storage, list); - } - - /* Removals need to be handled before additions. */ - if (json_object_has_non_null_array_member (lur, "removals")) { - JsonArray *removals = json_object_get_array_member (lur, "removals"); - for (guint k = 0; k < json_array_get_length (removals); k++) { - JsonObject *tes = json_array_get_object_element (removals, k); - ephy_gsb_storage_delete_hash_prefixes (self->storage, list, tes); - } - } - - /* Handle additions. */ - if (json_object_has_non_null_array_member (lur, "additions")) { - JsonArray *additions = json_object_get_array_member (lur, "additions"); - for (guint k = 0; k < json_array_get_length (additions); k++) { - JsonObject *tes = json_array_get_object_element (additions, k); - ephy_gsb_storage_insert_hash_prefixes (self->storage, list, tes); - } - } - - /* Verify checksum. */ - local_checksum = ephy_gsb_storage_compute_checksum (self->storage, list); - if (!g_strcmp0 (local_checksum, remote_checksum)) { - LOG ("Local checksum matches the remote checksum, updating client state..."); - ephy_gsb_storage_update_client_state (self->storage, list, FALSE); - } else { - LOG ("Local checksum does NOT match the remote checksum, clearing list..."); - ephy_gsb_storage_clear_hash_prefixes (self->storage, list); - ephy_gsb_storage_update_client_state (self->storage, list, TRUE); - } - - g_free (local_checksum); - ephy_gsb_threat_list_free (list); - } - - /* Update next update time. */ - if (json_object_has_non_null_string_member (body_obj, "minimumWaitDuration")) { - const char *duration_str; - double duration; - - duration_str = json_object_get_string_member (body_obj, "minimumWaitDuration"); - /* g_ascii_strtod() ignores trailing characters, i.e. 's' character. */ - duration = g_ascii_strtod (duration_str, NULL); - self->next_list_updates_time = CURRENT_TIME + (gint64)ceil (duration); - } - -out: - g_free (url); - if (msg) - g_object_unref (msg); - if (body_node) - json_node_unref (body_node); - g_list_free_full (threat_lists, (GDestroyNotify)ephy_gsb_threat_list_free); - - ephy_gsb_storage_set_metadata (self->storage, "next_list_updates_time", self->next_list_updates_time); - - g_object_unref (self); -} - -static void -ephy_gsb_service_update_finished_cb (EphyGSBService *self, - GAsyncResult *result, - gpointer user_data) -{ - g_atomic_int_set (&self->is_updating, FALSE); - g_signal_emit (self, signals[UPDATE_FINISHED], 0); - ephy_gsb_service_schedule_update (self); -} - -static gboolean -ephy_gsb_service_update (EphyGSBService *self) -{ - GTask *task; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - g_assert (ephy_gsb_storage_is_operable (self->storage)); - - g_atomic_int_set (&self->is_updating, TRUE); - task = g_task_new (g_object_ref (self), NULL, - (GAsyncReadyCallback)ephy_gsb_service_update_finished_cb, - NULL); - g_task_run_in_thread (task, (GTaskThreadFunc)ephy_gsb_service_update_thread); - g_object_unref (task); - - return G_SOURCE_REMOVE; -} - -static void -ephy_gsb_service_set_property (GObject *object, - guint prop_id, - const GValue *value, - GParamSpec *pspec) -{ - EphyGSBService *self = EPHY_GSB_SERVICE (object); - - switch (prop_id) { - case PROP_API_KEY: - g_free (self->api_key); - self->api_key = g_value_dup_string (value); - break; - case PROP_GSB_STORAGE: - if (self->storage) - g_object_unref (self->storage); - self->storage = g_value_dup_object (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - } -} - -static void -ephy_gsb_service_get_property (GObject *object, - guint prop_id, - GValue *value, - GParamSpec *pspec) -{ - EphyGSBService *self = EPHY_GSB_SERVICE (object); - - switch (prop_id) { - case PROP_API_KEY: - g_value_set_string (value, self->api_key); - break; - case PROP_GSB_STORAGE: - g_value_set_object (value, self->storage); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - } -} - -static void -ephy_gsb_service_finalize (GObject *object) -{ - EphyGSBService *self = EPHY_GSB_SERVICE (object); - - g_free (self->api_key); - - G_OBJECT_CLASS (ephy_gsb_service_parent_class)->finalize (object); -} - -static void -ephy_gsb_service_dispose (GObject *object) -{ - EphyGSBService *self = EPHY_GSB_SERVICE (object); - - g_clear_object (&self->storage); - g_clear_object (&self->session); - - g_clear_handle_id (&self->source_id, g_source_remove); - - G_OBJECT_CLASS (ephy_gsb_service_parent_class)->dispose (object); -} - -static void -ephy_gsb_service_constructed (GObject *object) -{ - EphyGSBService *self = EPHY_GSB_SERVICE (object); - - G_OBJECT_CLASS (ephy_gsb_service_parent_class)->constructed (object); - - if (!ephy_gsb_storage_is_operable (self->storage)) - return; - - /* Restore back-off parameters. */ - self->back_off_exit_time = ephy_gsb_storage_get_metadata (self->storage, - "back_off_exit_time", - CURRENT_TIME); - self->back_off_num_fails = ephy_gsb_storage_get_metadata (self->storage, - "back_off_num_fails", - 0); - - /* Restore next fullHashes:find request time. */ - self->next_full_hashes_time = ephy_gsb_storage_get_metadata (self->storage, - "next_full_hashes_time", - CURRENT_TIME); - - /* Restore next threatListUpdates:fetch request time. */ - self->next_list_updates_time = ephy_gsb_storage_get_metadata (self->storage, - "next_list_updates_time", - CURRENT_TIME); - - if (ephy_gsb_service_is_back_off_mode (self)) - self->next_list_updates_time = self->back_off_exit_time; - else - ephy_gsb_service_reset_back_off_mode (self); - - if (self->next_list_updates_time > CURRENT_TIME) - ephy_gsb_service_schedule_update (self); - else - ephy_gsb_service_update (self); -} - -static void -ephy_gsb_service_init (EphyGSBService *self) -{ - self->session = soup_session_new (); - g_object_set (self->session, "user-agent", ephy_user_agent_get (), NULL); -} - -static void -ephy_gsb_service_class_init (EphyGSBServiceClass *klass) -{ - GObjectClass *object_class = G_OBJECT_CLASS (klass); - - object_class->set_property = ephy_gsb_service_set_property; - object_class->get_property = ephy_gsb_service_get_property; - object_class->constructed = ephy_gsb_service_constructed; - object_class->dispose = ephy_gsb_service_dispose; - object_class->finalize = ephy_gsb_service_finalize; - - obj_properties[PROP_API_KEY] = - g_param_spec_string ("api-key", - "API key", - "The API key to access the Google Safe Browsing API", - NULL, - G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); - - obj_properties[PROP_GSB_STORAGE] = - g_param_spec_object ("gsb-storage", - "GSB filename", - "Handler object for the Google Safe Browsing database", - EPHY_TYPE_GSB_STORAGE, - G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); - - g_object_class_install_properties (object_class, LAST_PROP, obj_properties); - - signals[UPDATE_FINISHED] = - g_signal_new ("update-finished", - EPHY_TYPE_GSB_SERVICE, - G_SIGNAL_RUN_LAST, - 0, NULL, NULL, NULL, - G_TYPE_NONE, 0); -} - -EphyGSBService * -ephy_gsb_service_new (const char *api_key, - const char *db_path) -{ - EphyGSBService *service; - EphyGSBStorage *storage; - - storage = ephy_gsb_storage_new (db_path); - service = g_object_new (EPHY_TYPE_GSB_SERVICE, - "api-key", api_key, - "gsb-storage", storage, - NULL); - g_object_unref (storage); - - return service; -} - -static void -ephy_gsb_service_update_full_hashes_sync (EphyGSBService *self, - GList *prefixes) -{ - SoupMessage *msg; - GList *threat_lists; - JsonNode *body_node; - JsonObject *body_obj; - JsonArray *matches; - const char *duration_str; - char *url; - char *body; - double duration; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - g_assert (ephy_gsb_storage_is_operable (self->storage)); - g_assert (prefixes); - - if (self->next_full_hashes_time > CURRENT_TIME) { - LOG ("Cannot send fullHashes:find request. Requests are restricted for %ld seconds", - self->next_full_hashes_time - CURRENT_TIME); - return; - } - - if (ephy_gsb_service_is_back_off_mode (self)) { - LOG ("Cannot send fullHashes:find request. Back-off mode is enabled for %ld seconds", - self->back_off_exit_time - CURRENT_TIME); - return; - } - - threat_lists = ephy_gsb_storage_get_threat_lists (self->storage); - if (!threat_lists) - return; - - body = ephy_gsb_utils_make_full_hashes_request (threat_lists, prefixes); - url = g_strdup_printf ("%sfullHashes:find?key=%s", API_PREFIX, self->api_key); - msg = soup_message_new (SOUP_METHOD_POST, url); - soup_message_set_request (msg, "application/json", SOUP_MEMORY_TAKE, body, strlen (body)); - soup_session_send_message (self->session, msg); - - /* Handle unsuccessful responses. */ - if (msg->status_code != 200) { - LOG ("Cannot update full hashes, got: %u, %s", msg->status_code, msg->response_body->data); - ephy_gsb_service_update_back_off_mode (self); - goto out; - } - - /* Successful response, reset back-off mode. */ - ephy_gsb_service_reset_back_off_mode (self); - - body_node = json_from_string (msg->response_body->data, NULL); - if (!body_node || !JSON_NODE_HOLDS_OBJECT (body_node)) { - g_warning ("Response is not a valid JSON object"); - goto out; - } - - body_obj = json_node_get_object (body_node); - - if (json_object_has_non_null_array_member (body_obj, "matches")) { - matches = json_object_get_array_member (body_obj, "matches"); - - /* Update full hashes in database. */ - for (guint i = 0; i < json_array_get_length (matches); i++) { - EphyGSBThreatList *list; - JsonObject *match = json_array_get_object_element (matches, i); - const char *threat_type = json_object_get_string_member (match, "threatType"); - const char *platform_type = json_object_get_string_member (match, "platformType"); - const char *threat_entry_type = json_object_get_string_member (match, "threatEntryType"); - JsonObject *threat = json_object_get_object_member (match, "threat"); - const char *hash_b64 = json_object_get_string_member (threat, "hash"); - const char *positive_duration; - guint8 *hash; - gsize length; - - list = ephy_gsb_threat_list_new (threat_type, platform_type, threat_entry_type, NULL); - hash = g_base64_decode (hash_b64, &length); - positive_duration = json_object_get_string_member (match, "cacheDuration"); - /* g_ascii_strtod() ignores trailing characters, i.e. 's' character. */ - duration = g_ascii_strtod (positive_duration, NULL); - - ephy_gsb_storage_insert_full_hash (self->storage, list, hash, floor (duration)); - - g_free (hash); - ephy_gsb_threat_list_free (list); - } - } - - /* Update negative cache duration. */ - duration_str = json_object_get_string_member (body_obj, "negativeCacheDuration"); - /* g_ascii_strtod() ignores trailing characters, i.e. 's' character. */ - duration = g_ascii_strtod (duration_str, NULL); - for (GList *l = prefixes; l && l->data; l = l->next) - ephy_gsb_storage_update_hash_prefix_expiration (self->storage, l->data, floor (duration)); - - /* Handle minimum wait duration. */ - if (json_object_has_non_null_string_member (body_obj, "minimumWaitDuration")) { - duration_str = json_object_get_string_member (body_obj, "minimumWaitDuration"); - /* g_ascii_strtod() ignores trailing characters, i.e. 's' character. */ - duration = g_ascii_strtod (duration_str, NULL); - self->next_full_hashes_time = CURRENT_TIME + (gint64)ceil (duration); - ephy_gsb_storage_set_metadata (self->storage, "next_full_hashes_time", self->next_full_hashes_time); - } - - json_node_unref (body_node); -out: - g_free (url); - g_list_free_full (threat_lists, (GDestroyNotify)ephy_gsb_threat_list_free); - g_object_unref (msg); -} - -static void -ephy_gsb_service_verify_url_thread (GTask *task, - EphyGSBService *self, - const char *url, - GCancellable *cancellable) -{ - GList *hashes = NULL; - GList *cues = NULL; - GList *prefixes_lookup = NULL; - GList *hashes_lookup = NULL; - GList *matching_prefixes = NULL; - GList *matching_hashes = NULL; - GHashTable *matching_prefixes_set = NULL; - GHashTable *matching_hashes_set = NULL; - GHashTableIter iter; - gpointer value; - gboolean has_matching_expired_hashes = FALSE; - gboolean has_matching_expired_prefixes = FALSE; - GList *threats = NULL; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - g_assert (G_IS_TASK (task)); - g_assert (url); - - /* If the local database is broken or an update is in course, we cannot - * really verify the URL, so we have no choice other than to consider it safe. - */ - if (g_atomic_int_get (&self->is_updating)) { - LOG ("Local GSB database is being updated, cannot verify URL"); - goto out; - } - - if (!ephy_gsb_storage_is_operable (self->storage)) { - LOG ("Local GSB database is broken, cannot verify URL"); - goto out; - } - - hashes = ephy_gsb_utils_compute_hashes (url); - if (!hashes) - goto out; - - matching_prefixes_set = g_hash_table_new (g_bytes_hash, g_bytes_equal); - matching_hashes_set = g_hash_table_new (g_bytes_hash, g_bytes_equal); - - /* Check for hash prefixes in database that match any of the full hashes. */ - cues = ephy_gsb_utils_get_hash_cues (hashes); - prefixes_lookup = ephy_gsb_storage_lookup_hash_prefixes (self->storage, cues); - for (GList *p = prefixes_lookup; p && p->data; p = p->next) { - EphyGSBHashPrefixLookup *lookup = (EphyGSBHashPrefixLookup *)p->data; - - for (GList *h = hashes; h && h->data; h = h->next) { - if (ephy_gsb_utils_hash_has_prefix (h->data, lookup->prefix)) { - value = g_hash_table_lookup (matching_prefixes_set, lookup->prefix); - - /* Consider the prefix expired if it's expired in at least one threat list. */ - g_hash_table_replace (matching_prefixes_set, - lookup->prefix, - GINT_TO_POINTER (GPOINTER_TO_INT (value) || lookup->negative_expired)); - g_hash_table_add (matching_hashes_set, h->data); - } - } - } - - /* If there are no database matches, then the URL is safe. */ - if (g_hash_table_size (matching_hashes_set) == 0) { - LOG ("No database match, URL is safe"); - goto out; - } - - /* Check for full hashes matches. - * All unexpired full hash matches are added directly to the result set. - */ - matching_hashes = g_hash_table_get_keys (matching_hashes_set); - hashes_lookup = ephy_gsb_storage_lookup_full_hashes (self->storage, matching_hashes); - for (GList *l = hashes_lookup; l && l->data; l = l->next) { - EphyGSBHashFullLookup *lookup = (EphyGSBHashFullLookup *)l->data; - - if (lookup->expired) - has_matching_expired_hashes = TRUE; - else if (!g_list_find_custom (threats, lookup->threat_type, (GCompareFunc)g_strcmp0)) - threats = g_list_append (threats, g_strdup (lookup->threat_type)); - } - - /* Check for positive cache hit. - * That is, there is at least one unexpired full hash match. - */ - if (threats) { - LOG ("Positive cache hit, URL is not safe"); - goto out; - } - - /* Check for negative cache hit, i.e. there are no expired full hash - * matches and all hash prefix matches are negative-unexpired. - */ - g_hash_table_iter_init (&iter, matching_prefixes_set); - while (g_hash_table_iter_next (&iter, NULL, &value)) { - if (GPOINTER_TO_INT (value) == TRUE) { - has_matching_expired_prefixes = TRUE; - break; - } - } - if (!has_matching_expired_hashes && !has_matching_expired_prefixes) { - LOG ("Negative cache hit, URL is safe"); - goto out; - } - - /* At this point we have either expired full hash matches and/or - * negative-expired hash prefix matches, so we need to find from - * the server whether the URL is safe or not. We do this by updating - * the full hashes of the matching prefixes with fresh values from - * server and re-checking for positive cache hits. - */ - matching_prefixes = g_hash_table_get_keys (matching_prefixes_set); - ephy_gsb_service_update_full_hashes_sync (self, matching_prefixes); - - /* Repeat the full hash verification. */ - g_list_free_full (hashes_lookup, (GDestroyNotify)ephy_gsb_hash_full_lookup_free); - hashes_lookup = ephy_gsb_storage_lookup_full_hashes (self->storage, matching_hashes); - for (GList *l = hashes_lookup; l && l->data; l = l->next) { - EphyGSBHashFullLookup *lookup = (EphyGSBHashFullLookup *)l->data; - - if (!lookup->expired && - !g_list_find_custom (threats, lookup->threat_type, (GCompareFunc)g_strcmp0)) - threats = g_list_append (threats, g_strdup (lookup->threat_type)); - } - -out: - g_task_return_pointer (task, threats, NULL); - - g_list_free (matching_prefixes); - g_list_free (matching_hashes); - g_list_free_full (hashes, (GDestroyNotify)g_bytes_unref); - g_list_free_full (cues, (GDestroyNotify)g_bytes_unref); - g_list_free_full (prefixes_lookup, (GDestroyNotify)ephy_gsb_hash_prefix_lookup_free); - g_list_free_full (hashes_lookup, (GDestroyNotify)ephy_gsb_hash_full_lookup_free); - if (matching_prefixes_set) - g_hash_table_unref (matching_prefixes_set); - if (matching_hashes_set) - g_hash_table_unref (matching_hashes_set); -} - -void -ephy_gsb_service_verify_url (EphyGSBService *self, - const char *url, - GAsyncReadyCallback callback, - gpointer user_data) -{ - GTask *task; - - g_assert (EPHY_IS_GSB_SERVICE (self)); - g_assert (url); - g_assert (callback); - - task = g_task_new (self, NULL, callback, user_data); - g_task_set_task_data (task, g_strdup (url), g_free); - g_task_run_in_thread (task, (GTaskThreadFunc)ephy_gsb_service_verify_url_thread); - g_object_unref (task); -} - -GList * -ephy_gsb_service_verify_url_finish (EphyGSBService *self, - GAsyncResult *result) -{ - g_assert (g_task_is_valid (result, self)); - - return g_task_propagate_pointer (G_TASK (result), NULL); -} diff --git a/lib/safe-browsing/ephy-gsb-service.h b/lib/safe-browsing/ephy-gsb-service.h deleted file mode 100644 index decb73a51..000000000 --- a/lib/safe-browsing/ephy-gsb-service.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* - * Copyright © 2017 Gabriel Ivascu <gabrielivascu@gnome.org> - * - * This file is part of Epiphany. - * - * Epiphany is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Epiphany is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Epiphany. If not, see <http://www.gnu.org/licenses/>. - */ - -#pragma once - -#include <gio/gio.h> -#include <glib-object.h> - -G_BEGIN_DECLS - -#define EPHY_TYPE_GSB_SERVICE (ephy_gsb_service_get_type ()) - -G_DECLARE_FINAL_TYPE (EphyGSBService, ephy_gsb_service, EPHY, GSB_SERVICE, GObject) - -EphyGSBService *ephy_gsb_service_new (const char *api_key, - const char *db_path); -void ephy_gsb_service_verify_url (EphyGSBService *self, - const char *url, - GAsyncReadyCallback callback, - gpointer user_data); -GList *ephy_gsb_service_verify_url_finish (EphyGSBService *self, - GAsyncResult *result); - -G_END_DECLS diff --git a/lib/safe-browsing/ephy-gsb-storage.c b/lib/safe-browsing/ephy-gsb-storage.c deleted file mode 100644 index 501dfaaf2..000000000 --- a/lib/safe-browsing/ephy-gsb-storage.c +++ /dev/null @@ -1,1710 +0,0 @@ -/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* - * Copyright © 2017 Gabriel Ivascu <gabrielivascu@gnome.org> - * - * This file is part of Epiphany. - * - * Epiphany is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Epiphany is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Epiphany. If not, see <http://www.gnu.org/licenses/>. - */ - -#include "config.h" -#include "ephy-gsb-storage.h" - -#include "ephy-debug.h" -#include "ephy-sqlite-connection.h" - -#include <string.h> - -#define EXPIRATION_THRESHOLD (8 * 60 * 60) - -/* Keep this lower than 6533 (SQLITE_MAX_VARIABLE_NUMBER / 5 slots) or else - * you'll get "too many SQL variables" error in ephy_gsb_storage_insert_batch(). - * SQLITE_MAX_VARIABLE_NUMBER is hardcoded in sqlite3 (>= 3.22) as 32766. - */ -#define BATCH_SIZE 6553 - -/* Increment schema version if you: - * 1) Modify the database table structure. - * 2) Modify the threat lists below. - */ -#define SCHEMA_VERSION 3 - -/* The available Linux threat lists of Google Safe Browsing API v4. - * The format is {THREAT_TYPE, PLATFORM_TYPE, THREAT_ENTRY_TYPE}. - */ -static const char * const gsb_linux_threat_lists[][3] = { - {GSB_THREAT_TYPE_MALWARE, "LINUX", "URL"}, - {GSB_THREAT_TYPE_SOCIAL_ENGINEERING, "ANY_PLATFORM", "URL"}, - {GSB_THREAT_TYPE_UNWANTED_SOFTWARE, "LINUX", "URL"}, - {GSB_THREAT_TYPE_MALWARE, "LINUX", "IP_RANGE"}, -}; - -struct _EphyGSBStorage { - GObject parent_instance; - - char *db_path; - EphySQLiteConnection *db; - - gboolean is_operable; -}; - -G_DEFINE_TYPE (EphyGSBStorage, ephy_gsb_storage, G_TYPE_OBJECT); - -enum { - PROP_0, - PROP_DB_PATH, - LAST_PROP -}; - -static GParamSpec *obj_properties[LAST_PROP]; - -static gboolean -bind_threat_list_params (EphySQLiteStatement *statement, - EphyGSBThreatList *list, - int threat_type_col, - int platform_type_col, - int threat_entry_type_col, - int client_state_col) -{ - GError *error = NULL; - - g_assert (statement); - g_assert (list); - - if (list->threat_type && threat_type_col >= 0) { - ephy_sqlite_statement_bind_string (statement, threat_type_col, list->threat_type, &error); - if (error) { - g_warning ("Failed to bind threat type: %s", error->message); - g_error_free (error); - return FALSE; - } - } - if (list->platform_type && platform_type_col >= 0) { - ephy_sqlite_statement_bind_string (statement, platform_type_col, list->platform_type, &error); - if (error) { - g_warning ("Failed to bind platform type: %s", error->message); - g_error_free (error); - return FALSE; - } - } - if (list->threat_entry_type && threat_entry_type_col >= 0) { - ephy_sqlite_statement_bind_string (statement, threat_entry_type_col, list->threat_entry_type, &error); - if (error) { - g_warning ("Failed to bind threat entry type: %s", error->message); - g_error_free (error); - return FALSE; - } - } - if (list->client_state && client_state_col >= 0) { - ephy_sqlite_statement_bind_string (statement, client_state_col, list->client_state, &error); - if (error) { - g_warning ("Failed to bind client state: %s", error->message); - g_error_free (error); - return FALSE; - } - } - - return TRUE; -} - -static void -ephy_gsb_storage_start_transaction (EphyGSBStorage *self) -{ - GError *error = NULL; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (!self->is_operable) - return; - - ephy_sqlite_connection_begin_transaction (self->db, &error); - if (error) { - g_warning ("Failed to begin transaction on GSB database: %s", error->message); - g_error_free (error); - } -} - -static void -ephy_gsb_storage_end_transaction (EphyGSBStorage *self) -{ - GError *error = NULL; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (!self->is_operable) - return; - - ephy_sqlite_connection_commit_transaction (self->db, &error); - if (error) { - g_warning ("Failed to commit transaction on GSB database: %s", error->message); - g_error_free (error); - } -} - -static gboolean -ephy_gsb_storage_init_metadata_table (EphyGSBStorage *self) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (EPHY_IS_SQLITE_CONNECTION (self->db)); - - if (ephy_sqlite_connection_table_exists (self->db, "metadata")) - return TRUE; - - sql = "CREATE TABLE metadata (" - "key VARCHAR NOT NULL PRIMARY KEY," - "value INTEGER NOT NULL" - ")"; - ephy_sqlite_connection_execute (self->db, sql, &error); - if (error) { - g_warning ("Failed to create metadata table: %s", error->message); - g_error_free (error); - return FALSE; - } - - sql = "INSERT INTO metadata (key, value) VALUES" - "('schema_version', ?)," - "('next_list_updates_time', (CAST(strftime('%s', 'now') AS INT)))," - "('next_full_hashes_time', (CAST(strftime('%s', 'now') AS INT)))," - "('back_off_exit_time', 0)," - "('back_off_num_fails', 0)"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create metadata insert statement: %s", error->message); - g_error_free (error); - return FALSE; - } - - ephy_sqlite_statement_bind_int64 (statement, 0, SCHEMA_VERSION, &error); - if (error) { - g_warning ("Failed to bind int64 in metadata insert statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return FALSE; - } - - ephy_sqlite_statement_step (statement, &error); - g_object_unref (statement); - - if (error) { - g_warning ("Failed to insert initial data into metadata table: %s", error->message); - g_error_free (error); - return FALSE; - } - - return TRUE; -} - -static gboolean -ephy_gsb_storage_init_threats_table (EphyGSBStorage *self) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GString *string; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (EPHY_IS_SQLITE_CONNECTION (self->db)); - - if (ephy_sqlite_connection_table_exists (self->db, "threats")) - return TRUE; - - sql = "CREATE TABLE threats (" - "threat_type VARCHAR NOT NULL," - "platform_type VARCHAR NOT NULL," - "threat_entry_type VARCHAR NOT NULL," - "client_state VARCHAR," - "PRIMARY KEY (threat_type, platform_type, threat_entry_type)" - ")"; - ephy_sqlite_connection_execute (self->db, sql, &error); - if (error) { - g_warning ("Failed to create threats table: %s", error->message); - g_error_free (error); - return FALSE; - } - - sql = "INSERT INTO threats (threat_type, platform_type, threat_entry_type) VALUES "; - string = g_string_new (sql); - for (guint i = 0; i < G_N_ELEMENTS (gsb_linux_threat_lists); i++) - g_string_append (string, "(?, ?, ?),"); - /* Remove trailing comma character. */ - g_string_erase (string, string->len - 1, -1); - - statement = ephy_sqlite_connection_create_statement (self->db, string->str, &error); - g_string_free (string, TRUE); - - if (error) { - g_warning ("Failed to create threats table insert statement: %s", error->message); - g_error_free (error); - return FALSE; - } - - for (guint i = 0; i < G_N_ELEMENTS (gsb_linux_threat_lists); i++) { - EphyGSBThreatList *list = ephy_gsb_threat_list_new (gsb_linux_threat_lists[i][0], - gsb_linux_threat_lists[i][1], - gsb_linux_threat_lists[i][2], - NULL); - bind_threat_list_params (statement, list, i * 3, i * 3 + 1, i * 3 + 2, -1); - ephy_gsb_threat_list_free (list); - } - - ephy_sqlite_statement_step (statement, &error); - g_object_unref (statement); - - if (error) { - g_warning ("Failed to insert initial data into threats table: %s", error->message); - g_error_free (error); - return FALSE; - } - - return TRUE; -} - -static gboolean -ephy_gsb_storage_init_hash_prefix_table (EphyGSBStorage *self) -{ - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (EPHY_IS_SQLITE_CONNECTION (self->db)); - - if (ephy_sqlite_connection_table_exists (self->db, "hash_prefix")) - return TRUE; - - sql = "CREATE TABLE hash_prefix (" - "cue BLOB NOT NULL," /* The first 4 bytes. */ - "value BLOB NOT NULL," /* The prefix itself, can vary from 4 to 32 bytes. */ - "threat_type VARCHAR NOT NULL," - "platform_type VARCHAR NOT NULL," - "threat_entry_type VARCHAR NOT NULL," - "negative_expires_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s', 'now') AS INT))," - "PRIMARY KEY (value, threat_type, platform_type, threat_entry_type)," - "FOREIGN KEY(threat_type, platform_type, threat_entry_type)" - " REFERENCES threats(threat_type, platform_type, threat_entry_type)" - " ON DELETE CASCADE" - ")"; - ephy_sqlite_connection_execute (self->db, sql, &error); - if (error) { - g_warning ("Failed to create hash_prefix table: %s", error->message); - g_error_free (error); - return FALSE; - } - - sql = "CREATE INDEX idx_hash_prefix_cue ON hash_prefix (cue)"; - ephy_sqlite_connection_execute (self->db, sql, &error); - if (error) { - g_warning ("Failed to create idx_hash_prefix_cue index: %s", error->message); - g_error_free (error); - return FALSE; - } - - return TRUE; -} - -static gboolean -ephy_gsb_storage_init_hash_full_table (EphyGSBStorage *self) -{ - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (EPHY_IS_SQLITE_CONNECTION (self->db)); - - if (ephy_sqlite_connection_table_exists (self->db, "hash_full")) - return TRUE; - - sql = "CREATE TABLE hash_full (" - "value BLOB NOT NULL," /* The 32 bytes full hash. */ - "threat_type VARCHAR NOT NULL," - "platform_type VARCHAR NOT NULL," - "threat_entry_type VARCHAR NOT NULL," - "expires_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s', 'now') AS INT))," - "PRIMARY KEY (value, threat_type, platform_type, threat_entry_type)" - ")"; - ephy_sqlite_connection_execute (self->db, sql, &error); - if (error) { - g_warning ("Failed to create hash_full table: %s", error->message); - g_error_free (error); - return FALSE; - } - - sql = "CREATE INDEX idx_hash_full_value ON hash_full (value)"; - ephy_sqlite_connection_execute (self->db, sql, &error); - if (error) { - g_warning ("Failed to create idx_hash_full_value index: %s", error->message); - g_error_free (error); - return FALSE; - } - - return TRUE; -} - -static gboolean -ephy_gsb_storage_open_db (EphyGSBStorage *self) -{ - GError *error = NULL; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (!self->db); - - self->db = ephy_sqlite_connection_new (EPHY_SQLITE_CONNECTION_MODE_READWRITE, self->db_path); - ephy_sqlite_connection_open (self->db, &error); - if (error) { - g_warning ("Failed to open GSB database at %s: %s", self->db_path, error->message); - g_error_free (error); - g_clear_object (&self->db); - return FALSE; - } - - ephy_sqlite_connection_enable_foreign_keys (self->db); - - ephy_sqlite_connection_execute (self->db, "PRAGMA synchronous=OFF", &error); - if (error) { - g_warning ("Failed to disable synchronous pragma: %s", error->message); - g_error_free (error); - } - - return TRUE; -} - -static void -ephy_gsb_storage_clear_db (EphyGSBStorage *self) -{ - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (self->db) { - ephy_sqlite_connection_close (self->db); - ephy_sqlite_connection_delete_database (self->db); - g_clear_object (&self->db); - } -} - -static gboolean -ephy_gsb_storage_init_db (EphyGSBStorage *self) -{ - gboolean success; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (!self->db); - - if (!ephy_gsb_storage_open_db (self)) - return FALSE; - - success = ephy_gsb_storage_init_metadata_table (self) && - ephy_gsb_storage_init_threats_table (self) && - ephy_gsb_storage_init_hash_prefix_table (self) && - ephy_gsb_storage_init_hash_full_table (self); - - if (!success) - ephy_gsb_storage_clear_db (self); - - self->is_operable = success; - - return success; -} - -static gboolean -ephy_gsb_storage_recreate_db (EphyGSBStorage *self) -{ - g_assert (EPHY_IS_GSB_STORAGE (self)); - - ephy_gsb_storage_clear_db (self); - return ephy_gsb_storage_init_db (self); -} - -static inline gboolean -ephy_gsb_storage_check_schema_version (EphyGSBStorage *self) -{ - gint64 schema_version; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (EPHY_IS_SQLITE_CONNECTION (self->db)); - - schema_version = ephy_gsb_storage_get_metadata (self, "schema_version", 0); - - return schema_version == SCHEMA_VERSION; -} - -static void -ephy_gsb_storage_set_property (GObject *object, - guint prop_id, - const GValue *value, - GParamSpec *pspec) -{ - EphyGSBStorage *self = EPHY_GSB_STORAGE (object); - - switch (prop_id) { - case PROP_DB_PATH: - g_free (self->db_path); - self->db_path = g_value_dup_string (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - } -} - -static void -ephy_gsb_storage_get_property (GObject *object, - guint prop_id, - GValue *value, - GParamSpec *pspec) -{ - EphyGSBStorage *self = EPHY_GSB_STORAGE (object); - - switch (prop_id) { - case PROP_DB_PATH: - g_value_set_string (value, self->db_path); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - } -} - -static void -ephy_gsb_storage_finalize (GObject *object) -{ - EphyGSBStorage *self = EPHY_GSB_STORAGE (object); - - g_free (self->db_path); - if (self->db) { - ephy_sqlite_connection_close (self->db); - g_object_unref (self->db); - } - - G_OBJECT_CLASS (ephy_gsb_storage_parent_class)->finalize (object); -} - -static void -ephy_gsb_storage_constructed (GObject *object) -{ - EphyGSBStorage *self = EPHY_GSB_STORAGE (object); - gboolean success; - - G_OBJECT_CLASS (ephy_gsb_storage_parent_class)->constructed (object); - - if (!g_file_test (self->db_path, G_FILE_TEST_EXISTS)) { - LOG ("GSB database does not exist, initializing..."); - ephy_gsb_storage_init_db (self); - } else { - LOG ("GSB database exists, opening..."); - success = ephy_gsb_storage_open_db (self); - if (success && !ephy_gsb_storage_check_schema_version (self)) { - LOG ("GSB database schema incompatibility, recreating database..."); - ephy_gsb_storage_recreate_db (self); - } - } -} - -static void -ephy_gsb_storage_init (EphyGSBStorage *self) -{ -} - -static void -ephy_gsb_storage_class_init (EphyGSBStorageClass *klass) -{ - GObjectClass *object_class = G_OBJECT_CLASS (klass); - - object_class->set_property = ephy_gsb_storage_set_property; - object_class->get_property = ephy_gsb_storage_get_property; - object_class->constructed = ephy_gsb_storage_constructed; - object_class->finalize = ephy_gsb_storage_finalize; - - obj_properties[PROP_DB_PATH] = - g_param_spec_string ("db-path", - "Database path", - "The path of the SQLite file holding the lists of unsafe web resources", - NULL, - G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); - - g_object_class_install_properties (object_class, LAST_PROP, obj_properties); -} - -EphyGSBStorage * -ephy_gsb_storage_new (const char *db_path) -{ - return g_object_new (EPHY_TYPE_GSB_STORAGE, "db-path", db_path, NULL); -} - -/** - * ephy_gsb_storage_is_operable: - * @self: an #EphyGSBStorage - * - * Verify whether the local database is operable, i.e. no error occurred during - * the opening and/or initialization of the database. No operations on @self are - * allowed if the local database is inoperable. - * - * Return value: %TRUE if the local database is operable - **/ -gboolean -ephy_gsb_storage_is_operable (EphyGSBStorage *self) -{ - g_assert (EPHY_IS_GSB_STORAGE (self)); - - return self->is_operable; -} - -/** - * ephy_gsb_storage_get_metadata: - * @self: an #EphyGSBStorage - * @key: the key whose value to retrieve - * @default_value: the value to return in case of error or if @key is missing - * - * Retrieve the value of a key from the metadata table of the local database. - * - * Return value: The metadata value associated with @key - **/ -gint64 -ephy_gsb_storage_get_metadata (EphyGSBStorage *self, - const char *key, - gint64 default_value) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - gint64 value; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (EPHY_IS_SQLITE_CONNECTION (self->db)); - g_assert (key); - - sql = "SELECT value FROM metadata WHERE key=?"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create select metadata statement: %s", error->message); - g_error_free (error); - return default_value; - } - - ephy_sqlite_statement_bind_string (statement, 0, key, &error); - if (error) { - g_warning ("Failed to bind key as string in select metadata statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return default_value; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute select metadata statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - ephy_gsb_storage_recreate_db (self); - return default_value; - } - - value = ephy_sqlite_statement_get_column_as_int64 (statement, 0); - g_object_unref (statement); - - return value; -} - -/** - * ephy_gsb_storage_set_metadata: - * @self: an #EphyGSBStorage - * @key: the key whose value to update - * @value: the updated value - * - * Update the value of a key in the metadata table of the local database. - **/ -void -ephy_gsb_storage_set_metadata (EphyGSBStorage *self, - const char *key, - gint64 value) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (key); - - if (!self->is_operable) - return; - - sql = "UPDATE metadata SET value=? WHERE key=?"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create update metadata statement: %s", error->message); - g_error_free (error); - return; - } - - ephy_sqlite_statement_bind_int64 (statement, 0, value, &error); - if (error) { - g_warning ("Failed to bind value as int64 in update metadata statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return; - } - - ephy_sqlite_statement_bind_string (statement, 1, key, &error); - if (error) { - g_warning ("Failed to bind key as string in update metadata statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return; - } - - ephy_sqlite_statement_step (statement, &error); - g_object_unref (statement); - - if (error) { - g_warning ("Failed to execute update metadata statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } -} - -/** - * ephy_gsb_storage_get_threat_lists: - * @self: an #EphyGSBStorage - * - * Retrieve the list of supported threat lists from the threats table of the - * local database. - * - * Return value: (element-type #EphyGSBThreatList) (transfer full): a #GList - * containing the threat lists. The caller takes ownership - * of the list and its content. Use g_list_free_full() with - * ephy_gsb_threat_list_free() as free_func when done using - * the list. - **/ -GList * -ephy_gsb_storage_get_threat_lists (EphyGSBStorage *self) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GList *threat_lists = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (!self->is_operable) - return NULL; - - sql = "SELECT threat_type, platform_type, threat_entry_type, client_state FROM threats"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create select threat lists statement: %s", error->message); - g_error_free (error); - return NULL; - } - - while (ephy_sqlite_statement_step (statement, &error)) { - const char *threat_type = ephy_sqlite_statement_get_column_as_string (statement, 0); - const char *platform_type = ephy_sqlite_statement_get_column_as_string (statement, 1); - const char *threat_entry_type = ephy_sqlite_statement_get_column_as_string (statement, 2); - const char *client_state = ephy_sqlite_statement_get_column_as_string (statement, 3); - EphyGSBThreatList *list = ephy_gsb_threat_list_new (threat_type, platform_type, - threat_entry_type, client_state); - threat_lists = g_list_prepend (threat_lists, list); - } - - if (error) { - g_warning ("Failed to execute select threat lists statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); - - return g_list_reverse (threat_lists); -} - -/** - * ephy_gsb_storage_compute_checksum: - * @self: an #EphyGSBSTorage - * @list: an #EphyGSBThreatList - * - * Compute the SHA256 checksum of the lexicographically sorted list of all the - * hash prefixes belonging to @list in the local database. - * - * https://developers.google.com/safe-browsing/v4/local-databases#validation-checks - * - * Return value: (transfer full): the base64 encoded checksum or %NULL if error - **/ -char * -ephy_gsb_storage_compute_checksum (EphyGSBStorage *self, - EphyGSBThreatList *list) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - char *retval = NULL; - GChecksum *checksum; - guint8 *digest; - gsize digest_len = GSB_HASH_SIZE; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - - if (!self->is_operable) - return NULL; - - sql = "SELECT value FROM hash_prefix WHERE " - "threat_type=? AND platform_type=? AND threat_entry_type=? " - "ORDER BY value"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create select hash prefix statement: %s", error->message); - g_error_free (error); - return NULL; - } - - if (!bind_threat_list_params (statement, list, 0, 1, 2, -1)) { - g_object_unref (statement); - return NULL; - } - - checksum = g_checksum_new (GSB_HASH_TYPE); - while (ephy_sqlite_statement_step (statement, &error)) { - g_checksum_update (checksum, - ephy_sqlite_statement_get_column_as_blob (statement, 0), - ephy_sqlite_statement_get_column_size (statement, 0)); - } - - if (error) { - g_warning ("Failed to execute select hash prefix statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - goto out; - } - - digest = g_malloc (digest_len); - g_checksum_get_digest (checksum, digest, &digest_len); - retval = g_base64_encode (digest, digest_len); - - g_free (digest); -out: - g_object_unref (statement); - g_checksum_free (checksum); - - return retval; -} - -/** - * ephy_gsb_storage_update_client_state: - * @self: an #EphyGSBStorage - * @list: an #EphyGSBThreatList - * @clear: %TRUE if the client state should be set to %NULL - * - * Update the client state column of @list in the threats table of the local - * database. The new state is set according to the client_state field of @list. - * Set @clear to %TRUE if you wish to reset the state. - **/ -void -ephy_gsb_storage_update_client_state (EphyGSBStorage *self, - EphyGSBThreatList *list, - gboolean clear) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - gboolean success; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - - if (!self->is_operable) - return; - - if (clear) { - sql = "UPDATE threats SET client_state=NULL " - "WHERE threat_type=? AND platform_type=? AND threat_entry_type=?"; - } else { - sql = "UPDATE threats SET client_state=? " - "WHERE threat_type=? AND platform_type=? AND threat_entry_type=?"; - } - - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create update threats statement: %s", error->message); - g_error_free (error); - return; - } - - if (clear) - success = bind_threat_list_params (statement, list, 0, 1, 2, -1); - else - success = bind_threat_list_params (statement, list, 1, 2, 3, 0); - - if (!success) { - g_object_unref (statement); - return; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute update threat statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); -} - -/** - * ephy_gsb_storage_clear_hash_prefixes: - * @self: an #EphyGSBStorage - * @list: an #EphyGSBThreatList - * - * Delete all hash prefixes belonging to @list from the local database. - **/ -void -ephy_gsb_storage_clear_hash_prefixes (EphyGSBStorage *self, - EphyGSBThreatList *list) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - - if (!self->is_operable) - return; - - sql = "DELETE FROM hash_prefix WHERE " - "threat_type=? AND platform_type=? AND threat_entry_type=?"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create delete hash prefix statement: %s", error->message); - g_error_free (error); - return; - } - - if (!bind_threat_list_params (statement, list, 0, 1, 2, -1)) { - g_object_unref (statement); - return; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute clear hash prefix statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); -} - -static GList * -ephy_gsb_storage_get_hash_prefixes_to_delete (EphyGSBStorage *self, - EphyGSBThreatList *list, - GHashTable *indices, - gsize *num_prefixes) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GList *prefixes = NULL; - const char *sql; - guint index = 0; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (indices); - - *num_prefixes = 0; - - if (!self->is_operable) - return NULL; - - sql = "SELECT value FROM hash_prefix WHERE " - "threat_type=? AND platform_type=? AND threat_entry_type=? " - "ORDER BY value"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create select prefix value statement: %s", error->message); - g_error_free (error); - return NULL; - } - - if (!bind_threat_list_params (statement, list, 0, 1, 2, -1)) { - g_object_unref (statement); - return NULL; - } - - while (ephy_sqlite_statement_step (statement, &error)) { - if (g_hash_table_contains (indices, GUINT_TO_POINTER (index))) { - const guint8 *blob = ephy_sqlite_statement_get_column_as_blob (statement, 0); - gsize size = ephy_sqlite_statement_get_column_size (statement, 0); - prefixes = g_list_prepend (prefixes, g_bytes_new (blob, size)); - *num_prefixes += 1; - } - index++; - } - - if (error) { - g_warning ("Failed to execute select prefix value statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); - - return prefixes; -} - -static EphySQLiteStatement * -ephy_gsb_storage_make_delete_hash_prefix_statement (EphyGSBStorage *self, - gsize num_prefixes) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GString *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (!self->is_operable) - return NULL; - - sql = g_string_new ("DELETE FROM hash_prefix WHERE " - "threat_type=? AND platform_type=? and threat_entry_type=? " - "AND value IN ("); - for (gsize i = 0; i < num_prefixes; i++) - g_string_append (sql, "?,"); - /* Replace trailing comma character with close parenthesis character. */ - g_string_overwrite (sql, sql->len - 1, ")"); - - statement = ephy_sqlite_connection_create_statement (self->db, sql->str, &error); - if (error) { - g_warning ("Failed to create delete hash prefix statement: %s", error->message); - g_error_free (error); - } - - g_string_free (sql, TRUE); - - return statement; -} - -static GList * -ephy_gsb_storage_delete_hash_prefixes_batch (EphyGSBStorage *self, - EphyGSBThreatList *list, - GList *prefixes, - gsize num_prefixes, - EphySQLiteStatement *stmt) -{ - EphySQLiteStatement *statement = NULL; - GError *error = NULL; - gboolean free_statement = TRUE; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (prefixes); - - if (!self->is_operable) - return NULL; - - if (stmt) { - statement = stmt; - ephy_sqlite_statement_reset (statement); - free_statement = FALSE; - } else { - statement = ephy_gsb_storage_make_delete_hash_prefix_statement (self, num_prefixes); - if (!statement) - return prefixes; - } - - if (!bind_threat_list_params (statement, list, 0, 1, 2, -1)) - goto out; - - for (gsize i = 0; i < num_prefixes; i++) { - GBytes *prefix = (GBytes *)prefixes->data; - if (!ephy_sqlite_statement_bind_blob (statement, i + 3, - g_bytes_get_data (prefix, NULL), - g_bytes_get_size (prefix), - NULL)) { - g_warning ("Failed to bind values in delete hash prefix statement"); - goto out; - } - prefixes = prefixes->next; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute delete hash prefix statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - -out: - if (free_statement && statement) - g_object_unref (statement); - - /* Return where we left off. */ - return prefixes; -} - -static void -ephy_gsb_storage_delete_hash_prefixes_internal (EphyGSBStorage *self, - EphyGSBThreatList *list, - guint32 *indices, - gsize num_indices) -{ - EphySQLiteStatement *statement = NULL; - GList *prefixes = NULL; - GList *head = NULL; - GHashTable *set; - gsize num_prefixes = 0; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (indices); - - if (!self->is_operable) - return; - - LOG ("Deleting %lu hash prefixes...", num_indices); - - /* Move indices from the array to a hash table set. */ - set = g_hash_table_new (g_direct_hash, g_direct_equal); - for (gsize i = 0; i < num_indices; i++) - g_hash_table_add (set, GUINT_TO_POINTER (indices[i])); - - prefixes = ephy_gsb_storage_get_hash_prefixes_to_delete (self, list, set, &num_prefixes); - head = prefixes; - - ephy_gsb_storage_start_transaction (self); - - if (num_prefixes / BATCH_SIZE > 0) { - /* Reuse statement to increase performance. */ - statement = ephy_gsb_storage_make_delete_hash_prefix_statement (self, BATCH_SIZE); - - for (gsize i = 0; i < num_prefixes / BATCH_SIZE; i++) { - head = ephy_gsb_storage_delete_hash_prefixes_batch (self, list, - head, BATCH_SIZE, - statement); - } - } - - if (num_prefixes % BATCH_SIZE != 0) { - ephy_gsb_storage_delete_hash_prefixes_batch (self, list, - head, num_prefixes % BATCH_SIZE, - NULL); - } - - ephy_gsb_storage_end_transaction (self); - - g_hash_table_unref (set); - g_list_free_full (prefixes, (GDestroyNotify)g_bytes_unref); - if (statement) - g_object_unref (statement); -} - -/** - * ephy_gsb_storage_delete_hash_prefixes: - * @self: an #EphyGSBStorage - * @list: an #EphyGSBThreatList - * @tes: a ThreatEntrySet object as a #JsonObject - * - * Delete hash prefixes belonging to @list from the local database. Use this - * when handling the response of a threatListUpdates:fetch request. - **/ -void -ephy_gsb_storage_delete_hash_prefixes (EphyGSBStorage *self, - EphyGSBThreatList *list, - JsonObject *tes) -{ - JsonObject *raw_indices; - JsonObject *rice_indices; - JsonArray *indices_arr; - const char *compression; - guint32 *indices; - gsize num_indices; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (tes); - - if (!self->is_operable) - return; - - compression = json_object_get_string_member (tes, "compressionType"); - if (!g_strcmp0 (compression, GSB_COMPRESSION_TYPE_RICE)) { - rice_indices = json_object_get_object_member (tes, "riceIndices"); - indices = ephy_gsb_utils_rice_delta_decode (rice_indices, &num_indices); - } else { - raw_indices = json_object_get_object_member (tes, "rawIndices"); - indices_arr = json_object_get_array_member (raw_indices, "indices"); - num_indices = json_array_get_length (indices_arr); - - indices = g_malloc (num_indices * sizeof (guint32)); - for (gsize i = 0; i < num_indices; i++) - indices[i] = json_array_get_int_element (indices_arr, i); - } - - ephy_gsb_storage_delete_hash_prefixes_internal (self, list, indices, num_indices); - - g_free (indices); -} - -static EphySQLiteStatement * -ephy_gsb_storage_make_insert_hash_prefix_statement (EphyGSBStorage *self, - gsize num_prefixes) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GString *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (!self->is_operable) - return NULL; - - sql = g_string_new ("INSERT INTO hash_prefix " - "(cue, value, threat_type, platform_type, threat_entry_type) VALUES "); - for (gsize i = 0; i < num_prefixes; i++) - g_string_append (sql, "(?, ?, ?, ?, ?),"); - /* Remove trailing comma character. */ - g_string_erase (sql, sql->len - 1, -1); - - statement = ephy_sqlite_connection_create_statement (self->db, sql->str, &error); - if (error) { - g_warning ("Failed to create insert hash prefix statement: %s", error->message); - g_error_free (error); - } - - g_string_free (sql, TRUE); - - return statement; -} - -static void -ephy_gsb_storage_insert_hash_prefixes_batch (EphyGSBStorage *self, - EphyGSBThreatList *list, - const guint8 *prefixes, - gsize start, - gsize end, - gsize len, - EphySQLiteStatement *stmt) -{ - EphySQLiteStatement *statement = NULL; - GError *error = NULL; - gsize id = 0; - gboolean free_statement = TRUE; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (prefixes); - - if (!self->is_operable) - return; - - if (stmt) { - statement = stmt; - ephy_sqlite_statement_reset (statement); - free_statement = FALSE; - } else { - statement = ephy_gsb_storage_make_insert_hash_prefix_statement (self, (end - start + 1) / len); - if (!statement) - return; - } - - for (gsize k = start; k < end; k += len) { - if (!ephy_sqlite_statement_bind_blob (statement, id++, prefixes + k, GSB_HASH_CUE_LEN, NULL) || - !ephy_sqlite_statement_bind_blob (statement, id++, prefixes + k, len, NULL) || - !bind_threat_list_params (statement, list, id, id + 1, id + 2, -1)) { - g_warning ("Failed to bind values in hash prefix statement"); - goto out; - } - id += 3; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute insert hash prefix statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - -out: - if (free_statement && statement) - g_object_unref (statement); -} - -static void -ephy_gsb_storage_insert_hash_prefixes_internal (EphyGSBStorage *self, - EphyGSBThreatList *list, - const guint8 *prefixes, - gsize num_prefixes, - gsize prefix_len) -{ - EphySQLiteStatement *statement = NULL; - gsize num_batches; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (prefixes); - - if (!self->is_operable) - return; - - LOG ("Inserting %lu hash prefixes of size %ld...", num_prefixes, prefix_len); - - ephy_gsb_storage_start_transaction (self); - - num_batches = num_prefixes / BATCH_SIZE; - if (num_batches > 0) { - /* Reuse statement to increase performance. */ - statement = ephy_gsb_storage_make_insert_hash_prefix_statement (self, BATCH_SIZE); - - for (gsize i = 0; i < num_batches; i++) { - ephy_gsb_storage_insert_hash_prefixes_batch (self, list, prefixes, - i * prefix_len * BATCH_SIZE, - (i + 1) * prefix_len * BATCH_SIZE, - prefix_len, - statement); - } - } - - if (num_prefixes % BATCH_SIZE != 0) { - ephy_gsb_storage_insert_hash_prefixes_batch (self, list, prefixes, - num_batches * prefix_len * BATCH_SIZE, - num_prefixes * prefix_len - 1, - prefix_len, - NULL); - } - - ephy_gsb_storage_end_transaction (self); - - if (statement) - g_object_unref (statement); -} - -/** - * ephy_gsb_storage_insert_hash_prefixes: - * @self: an #EphyGSBStorage - * @list: an #EphyGSBThreatList - * @tes: a ThreatEntrySet object as a #JsonObject - * - * Insert hash prefixes belonging to @list in the local database. Use this - * when handling the response of a threatListUpdates:fetch request. - **/ -void -ephy_gsb_storage_insert_hash_prefixes (EphyGSBStorage *self, - EphyGSBThreatList *list, - JsonObject *tes) -{ - JsonObject *raw_hashes; - JsonObject *rice_hashes; - const char *compression; - const char *prefixes_b64; - guint32 *items = NULL; - guint8 *prefixes; - gsize prefixes_len; - gsize prefix_len; - gsize num_prefixes; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (tes); - - if (!self->is_operable) - return; - - compression = json_object_get_string_member (tes, "compressionType"); - if (!g_strcmp0 (compression, GSB_COMPRESSION_TYPE_RICE)) { - rice_hashes = json_object_get_object_member (tes, "riceHashes"); - items = ephy_gsb_utils_rice_delta_decode (rice_hashes, &num_prefixes); - - prefixes = g_malloc (num_prefixes * GSB_RICE_PREFIX_LEN); - for (gsize i = 0; i < num_prefixes; i++) - memcpy (prefixes + i * GSB_RICE_PREFIX_LEN, &items[i], GSB_RICE_PREFIX_LEN); - - prefix_len = GSB_RICE_PREFIX_LEN; - } else { - raw_hashes = json_object_get_object_member (tes, "rawHashes"); - prefix_len = json_object_get_int_member (raw_hashes, "prefixSize"); - prefixes_b64 = json_object_get_string_member (raw_hashes, "rawHashes"); - - prefixes = g_base64_decode (prefixes_b64, &prefixes_len); - num_prefixes = prefixes_len / prefix_len; - } - - ephy_gsb_storage_insert_hash_prefixes_internal (self, list, prefixes, num_prefixes, prefix_len); - - g_free (items); - g_free (prefixes); -} - -/** - * ephy_gsb_storage_lookup_hash_prefixes: - * @self: an #EphyGSBStorage - * @cues: a #GList of hash cues as #GBytes - * - * Retrieve the hash prefixes and their negative cache expiration time from the - * local database that begin with the hash cues in @cues. The hash cue length is - * specified by the GSB_HASH_CUE_LEN macro. - * - * Return value: (element-type #EphyGSBHashPrefixLookup) (transfer-full): - * a #GList containing the lookup result. The caller takes - * ownership of the list and its content. Use g_list_free_full() - * with ephy_gsb_hash_prefix_lookup_free() as free_func when done - * using the list. - **/ -GList * -ephy_gsb_storage_lookup_hash_prefixes (EphyGSBStorage *self, - GList *cues) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GList *retval = NULL; - GString *sql; - guint id = 0; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (cues); - - if (!self->is_operable) - return NULL; - - sql = g_string_new ("SELECT value, negative_expires_at <= (CAST(strftime('%s', 'now') AS INT)) " - "FROM hash_prefix WHERE cue IN ("); - for (GList *l = cues; l && l->data; l = l->next) - g_string_append (sql, "?,"); - /* Replace trailing comma character with close parenthesis character. */ - g_string_overwrite (sql, sql->len - 1, ")"); - - statement = ephy_sqlite_connection_create_statement (self->db, sql->str, &error); - g_string_free (sql, TRUE); - - if (error) { - g_warning ("Failed to create select hash prefix statement: %s", error->message); - g_error_free (error); - return NULL; - } - - for (GList *l = cues; l && l->data; l = l->next) { - ephy_sqlite_statement_bind_blob (statement, id++, - g_bytes_get_data (l->data, NULL), GSB_HASH_CUE_LEN, - &error); - if (error) { - g_warning ("Failed to bind cue value as blob: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return NULL; - } - } - - while (ephy_sqlite_statement_step (statement, &error)) { - const guint8 *blob = ephy_sqlite_statement_get_column_as_blob (statement, 0); - gsize size = ephy_sqlite_statement_get_column_size (statement, 0); - gboolean negative_expired = ephy_sqlite_statement_get_column_as_boolean (statement, 1); - retval = g_list_prepend (retval, ephy_gsb_hash_prefix_lookup_new (blob, size, negative_expired)); - } - - if (error) { - g_warning ("Failed to execute select hash prefix statement: %s", error->message); - g_error_free (error); - g_list_free_full (retval, (GDestroyNotify)ephy_gsb_hash_prefix_lookup_free); - retval = NULL; - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); - - return g_list_reverse (retval); -} - -/** - * ephy_gsb_storage_lookup_full_hashes: - * @self: an #EphyGSBStorage - * @hashes: a #GList of full hashes as #GBytes - * - * Retrieve the full hashes together with their positive cache expiration time - * and threat parameters from the local database that match any of the hashes - * in @hashes. - * - * Return value: (element-type #EphyGSBHashFullLookup) (transfer-full): - * a #GList containing the lookup result. The caller takes - * ownership of the list and its content. Use g_list_free_full() - * with ephy_gsb_hash_full_lookup_free() as free_func when done - * using the list. - **/ -GList * -ephy_gsb_storage_lookup_full_hashes (EphyGSBStorage *self, - GList *hashes) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - GList *retval = NULL; - GString *sql; - guint id = 0; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (hashes); - - if (!self->is_operable) - return NULL; - - sql = g_string_new ("SELECT value, threat_type, platform_type, threat_entry_type, " - "expires_at <= (CAST(strftime('%s', 'now') AS INT)) " - "FROM hash_full WHERE value IN ("); - for (GList *l = hashes; l && l->data; l = l->next) - g_string_append (sql, "?,"); - /* Replace trailing comma character with close parenthesis character. */ - g_string_overwrite (sql, sql->len - 1, ")"); - - statement = ephy_sqlite_connection_create_statement (self->db, sql->str, &error); - g_string_free (sql, TRUE); - - if (error) { - g_warning ("Failed to create select full hash statement: %s", error->message); - g_error_free (error); - return NULL; - } - - for (GList *l = hashes; l && l->data; l = l->next) { - ephy_sqlite_statement_bind_blob (statement, id++, - g_bytes_get_data (l->data, NULL), GSB_HASH_SIZE, - &error); - if (error) { - g_warning ("Failed to bind hash value as blob: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return NULL; - } - } - - while (ephy_sqlite_statement_step (statement, &error)) { - const guint8 *blob = ephy_sqlite_statement_get_column_as_blob (statement, 0); - const char *threat_type = ephy_sqlite_statement_get_column_as_string (statement, 1); - const char *platform_type = ephy_sqlite_statement_get_column_as_string (statement, 2); - const char *threat_entry_type = ephy_sqlite_statement_get_column_as_string (statement, 3); - gboolean expired = ephy_sqlite_statement_get_column_as_boolean (statement, 4); - EphyGSBHashFullLookup *lookup = ephy_gsb_hash_full_lookup_new (blob, - threat_type, - platform_type, - threat_entry_type, - expired); - retval = g_list_prepend (retval, lookup); - } - - if (error) { - g_warning ("Failed to execute select full hash statement: %s", error->message); - g_error_free (error); - g_list_free_full (retval, (GDestroyNotify)ephy_gsb_hash_full_lookup_free); - retval = NULL; - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); - - return g_list_reverse (retval); -} - -/** - * ephy_gsb_storage_insert_full_hash: - * @self: an #EphyGSBStorage - * @list: an #EphyGSBThreatList - * @hash: the full SHA256 hash - * @duration: the positive cache duration - * - * Insert a full hash belonging to @list in the local database. Use this - * when handling the response from a fullHashes:find request. If @hash - * already exists in the database and belongs to @list, then only the - * duration is updated. Otherwise, a new record is created. - **/ -void -ephy_gsb_storage_insert_full_hash (EphyGSBStorage *self, - EphyGSBThreatList *list, - const guint8 *hash, - gint64 duration) -{ - EphySQLiteStatement *statement = NULL; - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (list); - g_assert (hash); - - if (!self->is_operable) - return; - - LOG ("Inserting full hash with duration %ld for list %s/%s/%s", - duration, list->threat_type, list->platform_type, list->threat_entry_type); - - sql = "INSERT OR IGNORE INTO hash_full " - "(value, threat_type, platform_type, threat_entry_type) " - "VALUES (?, ?, ?, ?)"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create insert full hash statement: %s", error->message); - goto out; - } - - if (!bind_threat_list_params (statement, list, 1, 2, 3, -1)) - goto out; - ephy_sqlite_statement_bind_blob (statement, 0, hash, GSB_HASH_SIZE, &error); - if (error) { - g_warning ("Failed to bind blob in insert full hash statement: %s", error->message); - goto out; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute insert full hash statement: %s", error->message); - ephy_gsb_storage_recreate_db (self); - goto out; - } - - /* Update expiration time. */ - g_clear_object (&statement); - sql = "UPDATE hash_full SET expires_at=(CAST(strftime('%s', 'now') AS INT)) + ? " - "WHERE value=? AND threat_type=? AND platform_type=? AND threat_entry_type=?"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create update full hash statement: %s", error->message); - goto out; - } - - ephy_sqlite_statement_bind_int64 (statement, 0, duration, &error); - if (error) { - g_warning ("Failed to bind int64 in update full hash statement: %s", error->message); - goto out; - } - ephy_sqlite_statement_bind_blob (statement, 1, hash, GSB_HASH_SIZE, &error); - if (error) { - g_warning ("Failed to bind blob in update full hash statement: %s", error->message); - goto out; - } - if (!bind_threat_list_params (statement, list, 2, 3, 4, -1)) - goto out; - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute insert full hash statement: %s", error->message); - ephy_gsb_storage_recreate_db (self); - } - -out: - if (statement) - g_object_unref (statement); - if (error) - g_error_free (error); -} - -/** - * ephy_gsb_storage_delete_old_full_hashes: - * @self: an #EphyGSBStorage - * - * Delete long expired full hashes from the local database. The expiration - * threshold is specified by the EXPIRATION_THRESHOLD macro. - **/ -void -ephy_gsb_storage_delete_old_full_hashes (EphyGSBStorage *self) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - - if (!self->is_operable) - return; - - LOG ("Deleting full hashes expired for more than %d seconds", EXPIRATION_THRESHOLD); - - sql = "DELETE FROM hash_full " - "WHERE expires_at <= (CAST(strftime('%s', 'now') AS INT)) - ?"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create delete full hash statement: %s", error->message); - g_error_free (error); - return; - } - - ephy_sqlite_statement_bind_int64 (statement, 0, EXPIRATION_THRESHOLD, &error); - if (error) { - g_warning ("Failed to bind int64 in delete full hash statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute delete full hash statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); -} - -/** - * ephy_gsb_storage_update_hash_prefix_expiration: - * @self: an #EphyGSBStorage - * @prefix: the hash prefix - * @duration: the negative cache duration - * - * Update the negative cache expiration time of a hash prefix in the local database. - **/ -void -ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self, - GBytes *prefix, - gint64 duration) -{ - EphySQLiteStatement *statement; - GError *error = NULL; - const char *sql; - - g_assert (EPHY_IS_GSB_STORAGE (self)); - g_assert (prefix); - - if (!self->is_operable) - return; - - sql = "UPDATE hash_prefix " - "SET negative_expires_at=(CAST(strftime('%s', 'now') AS INT)) + ? " - "WHERE value=?"; - statement = ephy_sqlite_connection_create_statement (self->db, sql, &error); - if (error) { - g_warning ("Failed to create update hash prefix statement: %s", error->message); - g_error_free (error); - return; - } - - ephy_sqlite_statement_bind_int64 (statement, 0, duration, &error); - if (error) { - g_warning ("Failed to bind int64 in update hash prefix statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return; - } - ephy_sqlite_statement_bind_blob (statement, 1, - g_bytes_get_data (prefix, NULL), - g_bytes_get_size (prefix), - &error); - if (error) { - g_warning ("Failed to bind blob in update hash prefix statement: %s", error->message); - g_error_free (error); - g_object_unref (statement); - return; - } - - ephy_sqlite_statement_step (statement, &error); - if (error) { - g_warning ("Failed to execute update hash prefix statement: %s", error->message); - g_error_free (error); - ephy_gsb_storage_recreate_db (self); - } - - g_object_unref (statement); -} diff --git a/lib/safe-browsing/ephy-gsb-storage.h b/lib/safe-browsing/ephy-gsb-storage.h deleted file mode 100644 index ed41a7e23..000000000 --- a/lib/safe-browsing/ephy-gsb-storage.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* - * Copyright © 2017 Gabriel Ivascu <gabrielivascu@gnome.org> - * - * This file is part of Epiphany. - * - * Epiphany is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Epiphany is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Epiphany. If not, see <http://www.gnu.org/licenses/>. - */ - -#pragma once - -#include "ephy-gsb-utils.h" - -#include <glib-object.h> - -G_BEGIN_DECLS - -#define EPHY_TYPE_GSB_STORAGE (ephy_gsb_storage_get_type ()) - -G_DECLARE_FINAL_TYPE (EphyGSBStorage, ephy_gsb_storage, EPHY, GSB_STORAGE, GObject) - -EphyGSBStorage *ephy_gsb_storage_new (const char *db_path); -gboolean ephy_gsb_storage_is_operable (EphyGSBStorage *self); -gint64 ephy_gsb_storage_get_metadata (EphyGSBStorage *self, - const char *key, - gint64 default_value); -void ephy_gsb_storage_set_metadata (EphyGSBStorage *self, - const char *key, - gint64 value); -GList *ephy_gsb_storage_get_threat_lists (EphyGSBStorage *self); -char *ephy_gsb_storage_compute_checksum (EphyGSBStorage *self, - EphyGSBThreatList *list); -void ephy_gsb_storage_update_client_state (EphyGSBStorage *self, - EphyGSBThreatList *list, - gboolean clear); -void ephy_gsb_storage_clear_hash_prefixes (EphyGSBStorage *self, - EphyGSBThreatList *list); -void ephy_gsb_storage_delete_hash_prefixes (EphyGSBStorage *self, - EphyGSBThreatList *list, - JsonObject *tes); -void ephy_gsb_storage_insert_hash_prefixes (EphyGSBStorage *self, - EphyGSBThreatList *list, - JsonObject *tes); -GList *ephy_gsb_storage_lookup_hash_prefixes (EphyGSBStorage *self, - GList *cues); -GList *ephy_gsb_storage_lookup_full_hashes (EphyGSBStorage *self, - GList *hashes); -void ephy_gsb_storage_insert_full_hash (EphyGSBStorage *self, - EphyGSBThreatList *list, - const guint8 *hash, - gint64 duration); -void ephy_gsb_storage_delete_old_full_hashes (EphyGSBStorage *self); -void ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self, - GBytes *prefix, - gint64 duration); - -G_END_DECLS diff --git a/lib/safe-browsing/ephy-gsb-utils.c b/lib/safe-browsing/ephy-gsb-utils.c deleted file mode 100644 index d3cc81605..000000000 --- a/lib/safe-browsing/ephy-gsb-utils.c +++ /dev/null @@ -1,919 +0,0 @@ -/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* - * Copyright © 2017 Gabriel Ivascu <gabrielivascu@gnome.org> - * - * This file is part of Epiphany. - * - * Epiphany is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Epiphany is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Epiphany. If not, see <http://www.gnu.org/licenses/>. - */ - -#include "config.h" -#include "ephy-gsb-utils.h" - -#include "ephy-debug.h" -#include "ephy-string.h" - -#include <arpa/inet.h> -#include <libsoup/soup.h> -#include <stdio.h> -#include <string.h> - -#define MAX_HOST_SUFFIXES 5 -#define MAX_PATH_PREFIXES 6 -#define MAX_UNESCAPE_STEP 1024 - -typedef struct { - guint8 *data; /* The bit stream as an array of bytes */ - gsize data_len; /* The number of bytes in the array */ - guint8 *curr; /* The current byte in the bit stream */ - guint8 mask; /* Bit mask to read a bit within a byte */ - gsize num_read; /* The number of bits read so far */ -} EphyGSBBitReader; - -typedef struct { - EphyGSBBitReader *reader; - guint parameter; /* Golomb-Rice parameter, between 2 and 28 */ -} EphyGSBRiceDecoder; - -static inline EphyGSBBitReader * -ephy_gsb_bit_reader_new (const guint8 *data, - gsize data_len) -{ - EphyGSBBitReader *reader; - - g_assert (data); - g_assert (data_len > 0); - - reader = g_new (EphyGSBBitReader, 1); - reader->curr = reader->data = g_malloc (data_len); - memcpy (reader->data, data, data_len); - reader->data_len = data_len; - reader->mask = 0x01; - reader->num_read = 0; - - return reader; -} - -static inline void -ephy_gsb_bit_reader_free (EphyGSBBitReader *reader) -{ - g_assert (reader); - - g_free (reader->data); - g_free (reader); -} - -/* - * https://developers.google.com/safe-browsing/v4/compression#bit-encoderdecoder - */ -static guint32 -ephy_gsb_bit_reader_read (EphyGSBBitReader *reader, - guint num_bits) -{ - guint32 retval = 0; - - /* Cannot read more than 4 bytes at once. */ - g_assert (num_bits <= 32); - /* Cannot read more bits than the buffer has left. */ - g_assert (reader->num_read + num_bits <= reader->data_len * 8); - - /* Within a byte, the least-significant bits come before the most-significant - * bits in the bit stream. */ - for (guint i = 0; i < num_bits; i++) { - if (*reader->curr & reader->mask) - retval |= 1 << i; - - reader->mask <<= 1; - if (reader->mask == 0) { - reader->curr++; - reader->mask = 0x01; - } - } - - reader->num_read += num_bits; - - return retval; -} - -static inline EphyGSBRiceDecoder * -ephy_gsb_rice_decoder_new (const guint8 *data, - gsize data_len, - guint parameter) -{ - EphyGSBRiceDecoder *decoder; - - g_assert (data); - g_assert (data_len > 0); - - decoder = g_new (EphyGSBRiceDecoder, 1); - decoder->reader = ephy_gsb_bit_reader_new (data, data_len); - decoder->parameter = parameter; - - return decoder; -} - -static inline void -ephy_gsb_rice_decoder_free (EphyGSBRiceDecoder *decoder) -{ - g_assert (decoder); - - ephy_gsb_bit_reader_free (decoder->reader); - g_free (decoder); -} - -static guint32 -ephy_gsb_rice_decoder_next (EphyGSBRiceDecoder *decoder) -{ - guint32 quotient = 0; - guint32 remainder; - guint32 bit; - - g_assert (decoder); - - while ((bit = ephy_gsb_bit_reader_read (decoder->reader, 1)) != 0) - quotient += bit; - - remainder = ephy_gsb_bit_reader_read (decoder->reader, decoder->parameter); - - return (quotient << decoder->parameter) + remainder; -} - -EphyGSBThreatList * -ephy_gsb_threat_list_new (const char *threat_type, - const char *platform_type, - const char *threat_entry_type, - const char *client_state) -{ - EphyGSBThreatList *list; - - g_assert (threat_type); - g_assert (platform_type); - g_assert (threat_entry_type); - - list = g_new (EphyGSBThreatList, 1); - list->threat_type = g_strdup (threat_type); - list->platform_type = g_strdup (platform_type); - list->threat_entry_type = g_strdup (threat_entry_type); - list->client_state = g_strdup (client_state); - - return list; -} -void -ephy_gsb_threat_list_free (EphyGSBThreatList *list) -{ - g_assert (list); - - g_free (list->threat_type); - g_free (list->platform_type); - g_free (list->threat_entry_type); - g_free (list->client_state); - g_free (list); -} - -gboolean -ephy_gsb_threat_list_equal (EphyGSBThreatList *l1, - EphyGSBThreatList *l2) -{ - g_assert (l1); - g_assert (l2); - - if (g_strcmp0 (l1->threat_type, l2->threat_type) != 0) - return FALSE; - if (g_strcmp0 (l1->platform_type, l2->platform_type) != 0) - return FALSE; - if (g_strcmp0 (l1->threat_entry_type, l2->threat_entry_type) != 0) - return FALSE; - - return TRUE; -} - -EphyGSBHashPrefixLookup * -ephy_gsb_hash_prefix_lookup_new (const guint8 *prefix, - gsize length, - gboolean negative_expired) -{ - EphyGSBHashPrefixLookup *lookup; - - g_assert (prefix); - - lookup = g_new (EphyGSBHashPrefixLookup, 1); - lookup->prefix = g_bytes_new (prefix, length); - lookup->negative_expired = negative_expired; - - return lookup; -} - -void -ephy_gsb_hash_prefix_lookup_free (EphyGSBHashPrefixLookup *lookup) -{ - g_assert (lookup); - - g_bytes_unref (lookup->prefix); - g_free (lookup); -} - -EphyGSBHashFullLookup * -ephy_gsb_hash_full_lookup_new (const guint8 *hash, - const char *threat_type, - const char *platform_type, - const char *threat_entry_type, - gboolean expired) -{ - EphyGSBHashFullLookup *lookup; - - g_assert (hash); - g_assert (threat_type); - g_assert (platform_type); - g_assert (threat_entry_type); - - lookup = g_new (EphyGSBHashFullLookup, 1); - lookup->hash = g_bytes_new (hash, GSB_HASH_SIZE); - lookup->threat_type = g_strdup (threat_type); - lookup->platform_type = g_strdup (platform_type); - lookup->threat_entry_type = g_strdup (threat_entry_type); - lookup->expired = expired; - - return lookup; -} - -void -ephy_gsb_hash_full_lookup_free (EphyGSBHashFullLookup *lookup) -{ - g_assert (lookup); - - g_bytes_unref (lookup->hash); - g_free (lookup->threat_type); - g_free (lookup->platform_type); - g_free (lookup->threat_entry_type); - g_free (lookup); -} - -static JsonObject * -ephy_gsb_utils_make_client_info (void) -{ - JsonObject *client_info; - - client_info = json_object_new (); - json_object_set_string_member (client_info, "clientId", "Epiphany"); - json_object_set_string_member (client_info, "clientVersion", VERSION); - - return client_info; -} - -static JsonObject * -ephy_gsb_utils_make_contraints (void) -{ - JsonObject *constraints; - JsonArray *compressions; - - compressions = json_array_new (); - json_array_add_string_element (compressions, GSB_COMPRESSION_TYPE_RAW); - json_array_add_string_element (compressions, GSB_COMPRESSION_TYPE_RICE); - - constraints = json_object_new (); - /* No restriction for the number of update entries. */ - json_object_set_int_member (constraints, "maxUpdateEntries", 0); - /* No restriction for the number of database entries. */ - json_object_set_int_member (constraints, "maxDatabaseEntries", 0); - /* Let the server pick the geographic region automatically. */ - json_object_set_null_member (constraints, "region"); - json_object_set_array_member (constraints, "supportedCompressions", compressions); - - return constraints; -} - -/** - * ephy_gsb_utils_make_list_updates_request: - * @threat_lists: a #GList of #EphyGSBThreatList - * - * Create the request body for a threatListUpdates:fetch request. - * - * https://developers.google.com/safe-browsing/v4/reference/rest/v4/threatListUpdates/fetch#request-body - * - * Return value: (transfer full): the string representation of the request body - **/ -char * -ephy_gsb_utils_make_list_updates_request (GList *threat_lists) -{ - JsonArray *requests; - JsonObject *body_obj; - JsonNode *body_node; - char *retval; - - g_assert (threat_lists); - - requests = json_array_new (); - for (GList *l = threat_lists; l && l->data; l = l->next) { - EphyGSBThreatList *list = (EphyGSBThreatList *)l->data; - JsonObject *request = json_object_new (); - - json_object_set_string_member (request, "threatType", list->threat_type); - json_object_set_string_member (request, "platformType", list->platform_type); - json_object_set_string_member (request, "threatEntryType", list->threat_entry_type); - json_object_set_string_member (request, "state", list->client_state); - json_object_set_object_member (request, "constraints", ephy_gsb_utils_make_contraints ()); - json_array_add_object_element (requests, request); - } - - body_obj = json_object_new (); - json_object_set_object_member (body_obj, "client", ephy_gsb_utils_make_client_info ()); - json_object_set_array_member (body_obj, "listUpdateRequests", requests); - - body_node = json_node_new (JSON_NODE_OBJECT); - json_node_set_object (body_node, body_obj); - retval = json_to_string (body_node, FALSE); - - json_object_unref (body_obj); - json_node_unref (body_node); - - return retval; -} - -/** - * ephy_gsb_utils_make_full_hashes_request: - * @threat_lists: a #GList of #EphyGSBThreatList - * @hash_prefixes: a #GList of #GBytes - * - * Create the request body for a fullHashes:find request. - * - * https://developers.google.com/safe-browsing/v4/reference/rest/v4/fullHashes/find#request-body - * - * Return value: (transfer full): the string representation of the request body - **/ -char * -ephy_gsb_utils_make_full_hashes_request (GList *threat_lists, - GList *hash_prefixes) -{ - GHashTable *threat_types_set; - GHashTable *platform_types_set; - GHashTable *threat_entry_types_set; - GList *threat_types_list; - GList *platform_types_list; - GList *threat_entry_types_list; - JsonArray *threat_types; - JsonArray *platform_types; - JsonArray *threat_entry_types; - JsonArray *threat_entries; - JsonArray *client_states; - JsonObject *threat_info; - JsonObject *body_obj; - JsonNode *body_node; - char *body; - - g_assert (threat_lists); - g_assert (hash_prefixes); - - client_states = json_array_new (); - threat_types_set = g_hash_table_new (g_str_hash, g_str_equal); - platform_types_set = g_hash_table_new (g_str_hash, g_str_equal); - threat_entry_types_set = g_hash_table_new (g_str_hash, g_str_equal); - - for (GList *l = threat_lists; l && l->data; l = l->next) { - EphyGSBThreatList *list = (EphyGSBThreatList *)l->data; - - if (!g_hash_table_contains (threat_types_set, list->threat_type)) - g_hash_table_add (threat_types_set, list->threat_type); - if (!g_hash_table_contains (platform_types_set, list->platform_type)) - g_hash_table_add (platform_types_set, list->platform_type); - if (!g_hash_table_contains (threat_entry_types_set, list->threat_entry_type)) - g_hash_table_add (threat_entry_types_set, list->threat_entry_type); - - json_array_add_string_element (client_states, list->client_state); - } - - threat_types = json_array_new (); - threat_types_list = g_hash_table_get_keys (threat_types_set); - for (GList *l = threat_types_list; l && l->data; l = l->next) - json_array_add_string_element (threat_types, (const char *)l->data); - - platform_types = json_array_new (); - platform_types_list = g_hash_table_get_keys (platform_types_set); - for (GList *l = platform_types_list; l && l->data; l = l->next) - json_array_add_string_element (platform_types, (const char *)l->data); - - threat_entry_types = json_array_new (); - threat_entry_types_list = g_hash_table_get_keys (threat_entry_types_set); - for (GList *l = threat_entry_types_list; l && l->data; l = l->next) - json_array_add_string_element (threat_entry_types, (const char *)l->data); - - threat_entries = json_array_new (); - for (GList *l = hash_prefixes; l && l->data; l = l->next) { - JsonObject *threat_entry = json_object_new (); - char *hash = g_base64_encode (g_bytes_get_data (l->data, NULL), - g_bytes_get_size (l->data)); - - json_object_set_string_member (threat_entry, "hash", hash); - json_array_add_object_element (threat_entries, threat_entry); - - g_free (hash); - } - - threat_info = json_object_new (); - json_object_set_array_member (threat_info, "threatTypes", threat_types); - json_object_set_array_member (threat_info, "platformTypes", platform_types); - json_object_set_array_member (threat_info, "threatEntryTypes", threat_entry_types); - json_object_set_array_member (threat_info, "threatEntries", threat_entries); - - body_obj = json_object_new (); - json_object_set_object_member (body_obj, "client", ephy_gsb_utils_make_client_info ()); - json_object_set_array_member (body_obj, "clientStates", client_states); - json_object_set_object_member (body_obj, "threatInfo", threat_info); - json_object_set_null_member (body_obj, "apiClient"); - - body_node = json_node_new (JSON_NODE_OBJECT); - json_node_set_object (body_node, body_obj); - body = json_to_string (body_node, TRUE); - - g_list_free (threat_types_list); - g_list_free (platform_types_list); - g_list_free (threat_entry_types_list); - g_hash_table_unref (threat_types_set); - g_hash_table_unref (platform_types_set); - g_hash_table_unref (threat_entry_types_set); - json_object_unref (body_obj); - json_node_unref (body_node); - - return body; -} - -/** - * ephy_gsb_utils_rice_delta_decode: - * @rde: a RiceDeltaEncoding object as a #JsonObject - * @num_items: out parameter for the length of the returned array. This will be - * equal to 1 + RiceDeltaEncoding.numEntries - * - * Decompress the Rice-encoded data of a ThreatEntrySet received from a - * threatListUpdates:fetch response. - * - * https://developers.google.com/safe-browsing/v4/compression#rice-compression - * https://developers.google.com/safe-browsing/v4/reference/rest/v4/threatListUpdates/fetch#ricedeltaencoding - * - * Return value: (transfer full): the decompressed values as an array of guint32s - **/ -guint32 * -ephy_gsb_utils_rice_delta_decode (JsonObject *rde, - gsize *num_items) -{ - EphyGSBRiceDecoder *decoder; - const char *data_b64 = NULL; - const char *first_value_str = NULL; - guint32 *items; - guint8 *data; - gsize data_len; - gsize num_entries = 0; - guint parameter = 0; - - g_assert (rde); - g_assert (num_items); - - if (json_object_has_member (rde, "firstValue")) - first_value_str = json_object_get_string_member (rde, "firstValue"); - if (json_object_has_member (rde, "riceParameter")) - parameter = json_object_get_int_member (rde, "riceParameter"); - if (json_object_has_member (rde, "numEntries")) - num_entries = json_object_get_int_member (rde, "numEntries"); - if (json_object_has_member (rde, "encodedData")) - data_b64 = json_object_get_string_member (rde, "encodedData"); - - *num_items = 1 + num_entries; - items = g_malloc (*num_items * sizeof (guint32)); - items[0] = first_value_str ? g_ascii_strtoull (first_value_str, NULL, 10) : 0; - - if (num_entries == 0) - return items; - - /* Sanity check. */ - if (parameter < 2 || parameter > 28 || data_b64 == NULL) - return items; - - data = g_base64_decode (data_b64, &data_len); - decoder = ephy_gsb_rice_decoder_new (data, data_len, parameter); - - for (gsize i = 1; i <= num_entries; i++) - items[i] = items[i - 1] + ephy_gsb_rice_decoder_next (decoder); - - g_free (data); - ephy_gsb_rice_decoder_free (decoder); - - return items; -} - -static char * -ephy_gsb_utils_full_unescape (const char *part) -{ - char *prev; - char *prev_prev; - char *retval; - int attempts = 0; - - g_assert (part); - - prev = g_strdup (part); - retval = soup_uri_decode (part); - - /* Iteratively unescape the string until it cannot be unescaped anymore. - * This is useful for strings that have been escaped multiple times. - */ - while (g_strcmp0 (prev, retval) != 0 && attempts++ < MAX_UNESCAPE_STEP) { - prev_prev = prev; - prev = retval; - retval = soup_uri_decode (retval); - g_free (prev_prev); - } - - g_free (prev); - - return retval; -} - -static char * -ephy_gsb_utils_escape (const char *part) -{ - const guchar *s = (const guchar *)part; - GString *str; - - g_assert (part); - - str = g_string_new (NULL); - - /* Use this instead of soup_uri_encode() because that escapes other - * characters that we don't want to be escaped. - */ - while (*s) { - if (*s < 0x20 || *s >= 0x7f || *s == ' ' || *s == '#' || *s == '%') - g_string_append_printf (str, "%%%02X", *s++); - else - g_string_append_c (str, *s++); - } - - return g_string_free (str, FALSE); -} - -static char * -ephy_gsb_utils_normalize_escape (const char *part) -{ - char *tmp; - char *retval; - - g_assert (part); - - /* Perform a full unescape and then escape the string exactly once. */ - tmp = ephy_gsb_utils_full_unescape (part); - retval = ephy_gsb_utils_escape (tmp); - - g_free (tmp); - - return retval; -} - -static char * -ephy_gsb_utils_canonicalize_host (const char *host) -{ - struct in_addr addr; - char *tmp; - char *trimmed; - char *retval; - - g_assert (host); - - trimmed = g_strdup (host); - ephy_string_remove_leading (trimmed, '.'); - ephy_string_remove_trailing (trimmed, '.'); - - /* This actually replaces groups of consecutive dots with a single dot. */ - tmp = ephy_string_find_and_replace (trimmed, "..", "."); - - /* If host is as an IP address, normalize it to 4 dot-separated decimal values. - * If host is not an IP address, then it's a string and needs to be lowercased. - * - * inet_aton() handles octal, hex and fewer than 4 components addresses. - * See https://linux.die.net/man/3/inet_network - */ - if (inet_aton (tmp, &addr) != 0) { - retval = g_strdup (inet_ntoa (addr)); - } else { - retval = g_ascii_strdown (tmp, -1); - } - - g_free (trimmed); - g_free (tmp); - - return retval; -} - -/** - * ephy_gsb_utils_canonicalize: - * @url: the URL to canonicalize - * @host_out: out parameter for the host value of the canonicalized URL or %NULL - * @path_out: out parameter for the path value of the canonicalized URL or %NULL - * @query_out: out parameter for the query value of the canonicalized URL or %NULL - * - * Canonicalize @url according to Google Safe Browsing API v4 specification. - * - * https://developers.google.com/safe-browsing/v4/urls-hashing#canonicalization - * - * Return value: (transfer full): the canonical form of @url or %NULL if @url - * is not a valid URL - **/ -char * -ephy_gsb_utils_canonicalize (const char *url, - char **host_out, - char **path_out, - char **query_out) -{ - SoupURI *uri; - char *tmp; - char *host; - char *path; - char *host_canonical; - char *path_canonical; - char *retval; - const char *query; - - g_assert (url); - - /* Handle URLs with no scheme. */ - if (g_str_has_prefix (url, "//")) - tmp = g_strdup_printf ("http:%s", url); - else if (g_str_has_prefix (url, "://")) - tmp = g_strdup_printf ("http%s", url); - else if (!strstr (url, "://")) - tmp = g_strdup_printf ("http://%s", url); - else - tmp = g_strdup (url); - - /* soup_uri_new() prepares the URL for us: - * 1. Strips trailing and leading whitespaces. - * 2. Includes the path component if missing. - * 3. Removes tab (0x09), CR (0x0d), LF (0x0a) characters. - */ - uri = soup_uri_new (tmp); - g_free (tmp); - if (!uri) { - LOG ("Cannot make SoupURI from URL %s", url); - return NULL; - } - - /* Check for e.g. blob or data URIs */ - if (!uri->host) { - soup_uri_free (uri); - return NULL; - } - - /* Remove fragment. */ - soup_uri_set_fragment (uri, NULL); - - /* Canonicalize host. */ - host = ephy_gsb_utils_normalize_escape (soup_uri_get_host (uri)); - host_canonical = ephy_gsb_utils_canonicalize_host (host); - - /* Canonicalize path. "/../" and "/./" have already been resolved by soup_uri_new(). */ - path = ephy_gsb_utils_normalize_escape (soup_uri_get_path (uri)); - path_canonical = ephy_string_find_and_replace (path, "//", "/"); - - /* Combine all parts. */ - query = soup_uri_get_query (uri); - if (query) { - retval = g_strdup_printf ("%s://%s%s?%s", - soup_uri_get_scheme (uri), - host_canonical, path_canonical, - query); - } else { - retval = g_strdup_printf ("%s://%s%s", - soup_uri_get_scheme (uri), - host_canonical, path_canonical); - } - - if (host_out) - *host_out = g_strdup (host_canonical); - if (path_out) - *path_out = g_strdup (path_canonical); - if (query_out) - *query_out = g_strdup (query); - - g_free (host); - g_free (path); - g_free (host_canonical); - g_free (path_canonical); - soup_uri_free (uri); - - return retval; -} - -/* - * https://developers.google.com/safe-browsing/v4/urls-hashing#suffixprefix-expressions - */ -static GList * -ephy_gsb_utils_compute_host_suffixes (const char *host) -{ - GList *retval = NULL; - struct in_addr addr; - char **tokens; - int steps; - int start; - int num_tokens; - - g_assert (host); - - retval = g_list_prepend (retval, g_strdup (host)); - - /* If host is an IP address, return immediately. */ - if (inet_aton (host, &addr) != 0) - return retval; - - tokens = g_strsplit (host, ".", -1); - num_tokens = g_strv_length (tokens); - start = MAX (num_tokens - MAX_HOST_SUFFIXES, 1); - steps = MIN (num_tokens - 1 - start, MAX_HOST_SUFFIXES - 1); - - for (int i = start; i < start + steps; i++) - retval = g_list_prepend (retval, g_strjoinv (".", tokens + i)); - - g_strfreev (tokens); - - return g_list_reverse (retval); -} - -/* - * https://developers.google.com/safe-browsing/v4/urls-hashing#suffixprefix-expressions - */ -static GList * -ephy_gsb_utils_compute_path_prefixes (const char *path, - const char *query) -{ - GList *retval = NULL; - char *no_trailing; - char **tokens; - int steps; - int num_tokens; - int no_trailing_len; - gboolean has_trailing; - - g_assert (path); - - if (query) - retval = g_list_prepend (retval, g_strjoin ("?", path, query, NULL)); - retval = g_list_prepend (retval, g_strdup (path)); - - if (!g_strcmp0 (path, "/")) - return retval; - - has_trailing = path[strlen (path) - 1] == '/'; - no_trailing = ephy_string_remove_trailing (g_strdup (path), '/'); - no_trailing_len = strlen (no_trailing); - - tokens = g_strsplit (no_trailing, "/", -1); - num_tokens = g_strv_length (tokens); - steps = MIN (num_tokens, MAX_PATH_PREFIXES - 2); - - for (int i = 0; i < steps; i++) { - char *value = g_strconcat (i > 0 ? retval->data : "", tokens[i], "/", NULL); - - if ((has_trailing && !g_strcmp0 (value, path)) || - (!has_trailing && !strncmp (value, no_trailing, no_trailing_len))) { - g_free (value); - break; - } - - retval = g_list_prepend (retval, value); - } - - g_free (no_trailing); - g_strfreev (tokens); - - return g_list_reverse (retval); -} - -/** - * ephy_gsb_utils_compute_hashes: - * @url: the URL whose hashes to be computed - * - * Compute the SHA256 hashes of @url. - * - * https://developers.google.com/safe-browsing/v4/urls-hashing#hash-computations - * - * Return value: (element-type #GBytes) (transfer full): a #GList containing the - * full hashes of @url. The caller takes ownership of the list and - * its content. Use g_list_free_full() with g_bytes_unref() as - * free_func when done using the list. - **/ -GList * -ephy_gsb_utils_compute_hashes (const char *url) -{ - GChecksum *checksum; - GList *retval = NULL; - GList *host_suffixes; - GList *path_prefixes; - char *url_canonical; - char *host = NULL; - char *path = NULL; - char *query = NULL; - gsize hash_len = GSB_HASH_SIZE; - - g_assert (url); - - url_canonical = ephy_gsb_utils_canonicalize (url, &host, &path, &query); - if (!url_canonical) - return NULL; - - host_suffixes = ephy_gsb_utils_compute_host_suffixes (host); - path_prefixes = ephy_gsb_utils_compute_path_prefixes (path, query); - checksum = g_checksum_new (G_CHECKSUM_SHA256); - - /* Get the hash of every host-path combination. - * The maximum number of combinations is MAX_HOST_SUFFIXES * MAX_PATH_PREFIXES. - */ - for (GList *h = host_suffixes; h && h->data; h = h->next) { - for (GList *p = path_prefixes; p && p->data; p = p->next) { - char *value = g_strconcat (h->data, p->data, NULL); - guint8 *hash = g_malloc (hash_len); - - g_checksum_reset (checksum); - g_checksum_update (checksum, (const guint8 *)value, strlen (value)); - g_checksum_get_digest (checksum, hash, &hash_len); - retval = g_list_prepend (retval, g_bytes_new (hash, hash_len)); - - g_free (hash); - g_free (value); - } - } - - g_free (host); - g_free (path); - g_free (query); - g_free (url_canonical); - g_checksum_free (checksum); - g_list_free_full (host_suffixes, g_free); - g_list_free_full (path_prefixes, g_free); - - return g_list_reverse (retval); -} - -/** - * ephy_gsb_utils_get_hash_cues: - * @hashes: a #GList of #GBytes - * - * Get the hash cues from a list of full hashes. The hash cue length is - * specified by the GSB_HASH_CUE_LEN macro. - * - * Return value: (element-type #GBytes) (transfer full): a #GList containing - * the cues of each hash in @hashes. The caller takes ownership - * of the list and its content. Use g_list_free_full() with - * g_bytes_unref() as free_func when done using the list. - **/ -GList * -ephy_gsb_utils_get_hash_cues (GList *hashes) -{ - GList *retval = NULL; - - g_assert (hashes); - - for (GList *l = hashes; l && l->data; l = l->next) { - const char *hash = g_bytes_get_data (l->data, NULL); - retval = g_list_prepend (retval, g_bytes_new (hash, GSB_HASH_CUE_LEN)); - } - - return g_list_reverse (retval); -} - -/** - * ephy_gsb_utils_hash_has_prefix: - * @hash: the full hash to verify - * @prefix: the hash prefix to verify - * - * Verify whether @hash begins with the prefix @prefix. - * - * Return value: %TRUE if @hash begins with @prefix - **/ -gboolean -ephy_gsb_utils_hash_has_prefix (GBytes *hash, - GBytes *prefix) -{ - const guint8 *hash_data; - const guint8 *prefix_data; - gsize prefix_len; - - g_assert (hash); - g_assert (prefix); - - hash_data = g_bytes_get_data (hash, NULL); - prefix_data = g_bytes_get_data (prefix, &prefix_len); - - for (gsize i = 0; i < prefix_len; i++) { - if (hash_data[i] != prefix_data[i]) - return FALSE; - } - - return TRUE; -} diff --git a/lib/safe-browsing/ephy-gsb-utils.h b/lib/safe-browsing/ephy-gsb-utils.h deleted file mode 100644 index cfeb75dd8..000000000 --- a/lib/safe-browsing/ephy-gsb-utils.h +++ /dev/null @@ -1,98 +0,0 @@ -/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* - * Copyright © 2017 Gabriel Ivascu <gabrielivascu@gnome.org> - * - * This file is part of Epiphany. - * - * Epiphany is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Epiphany is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Epiphany. If not, see <http://www.gnu.org/licenses/>. - */ - -#pragma once - -#include <glib.h> -#include <json-glib/json-glib.h> - -G_BEGIN_DECLS - -#define GSB_HASH_CUE_LEN 4 -#define GSB_RICE_PREFIX_LEN 4 - -#define GSB_HASH_TYPE G_CHECKSUM_SHA256 -#define GSB_HASH_SIZE (g_checksum_type_get_length (GSB_HASH_TYPE)) - -#define GSB_COMPRESSION_TYPE_RAW "RAW" -#define GSB_COMPRESSION_TYPE_RICE "RICE" -#define GSB_COMPRESSION_TYPE_UNSPECIFIED "COMPRESSION_TYPE_UNSPECIFIED" - -#define GSB_THREAT_TYPE_MALWARE "MALWARE" -#define GSB_THREAT_TYPE_SOCIAL_ENGINEERING "SOCIAL_ENGINEERING" -#define GSB_THREAT_TYPE_UNWANTED_SOFTWARE "UNWANTED_SOFTWARE" - -typedef struct { - char *threat_type; - char *platform_type; - char *threat_entry_type; - char *client_state; -} EphyGSBThreatList; - -typedef struct { - GBytes *prefix; /* The first 4-32 bytes of the hash */ - gboolean negative_expired; -} EphyGSBHashPrefixLookup; - -typedef struct { - GBytes *hash; /* The 32 bytes full hash */ - char *threat_type; - char *platform_type; - char *threat_entry_type; - gboolean expired; -} EphyGSBHashFullLookup; - -EphyGSBThreatList *ephy_gsb_threat_list_new (const char *threat_type, - const char *platform_type, - const char *threat_entry_type, - const char *client_state); -void ephy_gsb_threat_list_free (EphyGSBThreatList *list); -gboolean ephy_gsb_threat_list_equal (EphyGSBThreatList *l1, - EphyGSBThreatList *l2); - -EphyGSBHashPrefixLookup *ephy_gsb_hash_prefix_lookup_new (const guint8 *prefix, - gsize length, - gboolean negative_expired); -void ephy_gsb_hash_prefix_lookup_free (EphyGSBHashPrefixLookup *lookup); - -EphyGSBHashFullLookup *ephy_gsb_hash_full_lookup_new (const guint8 *hash, - const char *threat_type, - const char *platform_type, - const char *threat_entry_type, - gboolean expired); -void ephy_gsb_hash_full_lookup_free (EphyGSBHashFullLookup *lookup); - -char *ephy_gsb_utils_make_list_updates_request (GList *threat_lists); -char *ephy_gsb_utils_make_full_hashes_request (GList *threat_lists, - GList *hash_prefixes); - -guint32 *ephy_gsb_utils_rice_delta_decode (JsonObject *rde, - gsize *num_items); - -char *ephy_gsb_utils_canonicalize (const char *url, - char **host_out, - char **path_out, - char **query_out); -GList *ephy_gsb_utils_compute_hashes (const char *url); -GList *ephy_gsb_utils_get_hash_cues (GList *hashes); -gboolean ephy_gsb_utils_hash_has_prefix (GBytes *hash, - GBytes *prefix); - -G_END_DECLS |