s3/vfs: add SMB_VFS_OFFLOAD_READ_SEND/RECV

Add SMB_VFS_OFFLOAD_READ_SEND an SMB_VFS_OFFLOAD_READ_RECV.

This paves the way for supporting server-side copy-chunk with source and
destination file-handles on different shares. It can be used to
implement copy offload fsctl in the future, but for now this will be
used as a mere copy-chunk replacement.

SMB_VFS_OFFLOAD_READ generates a token that associates an fsp with the
token and stores the fsp in a in-memory db.

Initially only a copy-chunk resume key fsctl is supported. In the future
this can be enhanced to support real offload fsctl.

Signed-off-by: Ralph Boehme <slow@samba.org>
Reviewed-by: Stefan Metzmacher <metze@samba.org>

13 files changed, 907 insertions, 3 deletions

diff --git a/examples/VFS/skel_opaque.c b/examples/VFS/skel_opaque.c
index 74ffb672024..2294375d94f 100644
--- a/examples/VFS/skel_opaque.c
+++ b/examples/VFS/skel_opaque.c
@@ -537,6 +537,48 @@ static struct file_id skel_file_id_create(vfs_handle_struct *handle,
 	return id;
 }
 
+struct skel_offload_read_state {
+	bool dummy;
+};
+
+static struct tevent_req *skel_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	struct files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+	struct skel_offload_read_state *state = NULL;
+
+	req = tevent_req_create(mem_ctx, &state, struct skel_offload_read_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	tevent_req_nterror(req, NT_STATUS_NOT_IMPLEMENTED);
+	return tevent_req_post(req, ev);
+}
+
+static NTSTATUS skel_offload_read_recv(struct tevent_req *req,
+				       struct vfs_handle_struct *handle,
+				       TALLOC_CTX *mem_ctx,
+				       DATA_BLOB *_token_blob)
+{
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+	tevent_req_received(req);
+
+	return NT_STATUS_OK;
+}
+
 struct skel_cc_state {
 	uint64_t unused;
 };
@@ -967,6 +1009,8 @@ struct vfs_fn_pointers skel_opaque_fns = {
 	.realpath_fn = skel_realpath,
 	.chflags_fn = skel_chflags,
 	.file_id_create_fn = skel_file_id_create,
+	.offload_read_send_fn = skel_offload_read_send,
+	.offload_read_recv_fn = skel_offload_read_recv,
 	.copy_chunk_send_fn = skel_copy_chunk_send,
 	.copy_chunk_recv_fn = skel_copy_chunk_recv,
 	.get_compression_fn = skel_get_compression,
diff --git a/examples/VFS/skel_transparent.c b/examples/VFS/skel_transparent.c
index e584d514655..51312f2c876 100644
--- a/examples/VFS/skel_transparent.c
+++ b/examples/VFS/skel_transparent.c
@@ -622,6 +622,94 @@ static struct file_id skel_file_id_create(vfs_handle_struct *handle,
 	return SMB_VFS_NEXT_FILE_ID_CREATE(handle, sbuf);
 }
 
+struct skel_offload_read_state {
+	struct vfs_handle_struct *handle;
+	DATA_BLOB token;
+};
+
+static void skel_offload_read_done(struct tevent_req *subreq);
+
+static struct tevent_req *skel_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	struct files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+	struct skel_offload_read_state *state = NULL;
+	struct tevent_req *subreq = NULL;
+
+	req = tevent_req_create(mem_ctx, &state, struct skel_offload_read_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	*state = (struct skel_offload_read_state) {
+		.handle = handle,
+	};
+
+	subreq = SMB_VFS_NEXT_OFFLOAD_READ_SEND(mem_ctx, ev, handle, fsp,
+						fsctl, ttl, offset, to_copy);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, skel_offload_read_done, req);
+	return req;
+}
+
+static void skel_offload_read_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct skel_offload_read_state *state = tevent_req_data(
+		req, struct skel_offload_read_state);
+	NTSTATUS status;
+
+	status = SMB_VFS_NEXT_OFFLOAD_READ_RECV(subreq,
+						state->handle,
+						state,
+						&state->token);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	tevent_req_done(req);
+	return;
+}
+
+static NTSTATUS skel_offload_read_recv(struct tevent_req *req,
+				       struct vfs_handle_struct *handle,
+				       TALLOC_CTX *mem_ctx,
+				       DATA_BLOB *_token)
+{
+	struct skel_offload_read_state *state = tevent_req_data(
+		req, struct skel_offload_read_state);
+	DATA_BLOB token;
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+
+	token = data_blob_talloc(mem_ctx,
+				 state->token.data,
+				 state->token.length);
+
+	tevent_req_received(req);
+
+	if (token.data == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	*_token = token;
+	return NT_STATUS_OK;
+}
+
 struct skel_cc_state {
 	struct vfs_handle_struct *handle;
 	off_t copied;
@@ -1094,6 +1182,8 @@ struct vfs_fn_pointers skel_transparent_fns = {
 	.realpath_fn = skel_realpath,
 	.chflags_fn = skel_chflags,
 	.file_id_create_fn = skel_file_id_create,
+	.offload_read_send_fn = skel_offload_read_send,
+	.offload_read_recv_fn = skel_offload_read_recv,
 	.copy_chunk_send_fn = skel_copy_chunk_send,
 	.copy_chunk_recv_fn = skel_copy_chunk_recv,
 	.get_compression_fn = skel_get_compression,
diff --git a/source3/include/vfs.h b/source3/include/vfs.h
index 6a3f6c9c42e..295c005fbd6 100644
--- a/source3/include/vfs.h
+++ b/source3/include/vfs.h
@@ -238,6 +238,7 @@
 		to struct smb_filename * */
 /* Version 37 - Change connectpath from char *
 		to struct smb_filename * */
+/* Version 37 - Add SMB_VFS_OFFLOAD_READ_SEND/RECV */
 
 #define SMB_VFS_INTERFACE_VERSION 37
 
@@ -781,6 +782,18 @@ struct vfs_fn_pointers {
 				unsigned int flags);
 	struct file_id (*file_id_create_fn)(struct vfs_handle_struct *handle,
 					    const SMB_STRUCT_STAT *sbuf);
+	struct tevent_req *(*offload_read_send_fn)(TALLOC_CTX *mem_ctx,
+						   struct tevent_context *ev,
+						   struct vfs_handle_struct *handle,
+						   struct files_struct *fsp,
+						   uint32_t fsctl,
+						   uint32_t ttl,
+						   off_t offset,
+						   size_t to_copy);
+	NTSTATUS (*offload_read_recv_fn)(struct tevent_req *req,
+					 struct vfs_handle_struct *handle,
+					 TALLOC_CTX *mem_ctx,
+					 DATA_BLOB *token_blob);
 	struct tevent_req *(*copy_chunk_send_fn)(struct vfs_handle_struct *handle,
 						 TALLOC_CTX *mem_ctx,
 						 struct tevent_context *ev,
@@ -1348,6 +1361,19 @@ NTSTATUS smb_vfs_call_set_dos_attributes(struct vfs_handle_struct *handle,
 NTSTATUS smb_vfs_call_fset_dos_attributes(struct vfs_handle_struct *handle,
 					  struct files_struct *fsp,
 					  uint32_t dosmode);
+struct tevent_req *smb_vfs_call_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	struct files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy);
+NTSTATUS smb_vfs_call_offload_read_recv(struct tevent_req *req,
+					struct vfs_handle_struct *handle,
+					TALLOC_CTX *mem_ctx,
+					DATA_BLOB *token_blob);
 struct tevent_req *smb_vfs_call_copy_chunk_send(struct vfs_handle_struct *handle,
 						TALLOC_CTX *mem_ctx,
 						struct tevent_context *ev,
diff --git a/source3/include/vfs_macros.h b/source3/include/vfs_macros.h
index 4365f15b3d1..65c7b7e091e 100644
--- a/source3/include/vfs_macros.h
+++ b/source3/include/vfs_macros.h
@@ -415,6 +415,16 @@
 #define SMB_VFS_NEXT_FSET_DOS_ATTRIBUTES(handle, fsp, attributes) \
 	smb_vfs_call_fset_dos_attributes((handle)->next, (fsp), (attributes))
 
+#define SMB_VFS_OFFLOAD_READ_SEND(mem_ctx, ev, fsp, fsctl, ttl, offset, to_copy) \
+	smb_vfs_call_offload_read_send((mem_ctx), (ev), (fsp)->conn->vfs_handles, fsp, (fsctl), (ttl), (offset), (to_copy))
+#define SMB_VFS_NEXT_OFFLOAD_READ_SEND(mem_ctx, ev, handle, fsp, fsctl, ttl, offset, to_copy) \
+	smb_vfs_call_offload_read_send((mem_ctx), (ev), (handle)->next, (fsp), (fsctl), (ttl), (offset), (to_copy))
+
+#define SMB_VFS_OFFLOAD_READ_RECV(req, conn, mem_ctx, token_blob) \
+	smb_vfs_call_offload_read_recv((req), (conn)->vfs_handles, (mem_ctx), (token_blob))
+#define SMB_VFS_NEXT_OFFLOAD_READ_RECV(req, handle, mem_ctx, token_blob) \
+	smb_vfs_call_offload_read_recv((req), (handle)->next, (mem_ctx), (token_blob))
+
 #define SMB_VFS_COPY_CHUNK_SEND(conn, mem_ctx, ev, src_fsp, src_off, dest_fsp, dest_off, num, flags) \
 	smb_vfs_call_copy_chunk_send((conn)->vfs_handles, (mem_ctx), (ev), (src_fsp), (src_off), (dest_fsp), (dest_off), (num), (flags))
 #define SMB_VFS_NEXT_COPY_CHUNK_SEND(handle, mem_ctx, ev, src_fsp, src_off, dest_fsp, dest_off, num, flags) \
diff --git a/source3/modules/offload_token.c b/source3/modules/offload_token.c
new file mode 100644
index 00000000000..2969262f48e
--- /dev/null
+++ b/source3/modules/offload_token.c
@@ -0,0 +1,222 @@
+/*
+   Unix SMB/CIFS implementation.
+
+   Copyright (C) Ralph Boehme 2017
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "smbd/smbd.h"
+#include "smbd/globals.h"
+#include "dbwrap/dbwrap.h"
+#include "dbwrap/dbwrap_rbt.h"
+#include "dbwrap/dbwrap_open.h"
+#include "../lib/util/util_tdb.h"
+#include "librpc/gen_ndr/ndr_ioctl.h"
+#include "librpc/gen_ndr/ioctl.h"
+#include "offload_token.h"
+
+struct vfs_offload_ctx {
+	bool initialized;
+	struct db_context *db_ctx;
+};
+
+NTSTATUS vfs_offload_token_ctx_init(TALLOC_CTX *mem_ctx,
+				    struct vfs_offload_ctx **_ctx)
+{
+	struct vfs_offload_ctx *ctx = *_ctx;
+
+	if (ctx != NULL) {
+		if (!ctx->initialized) {
+			return NT_STATUS_INTERNAL_ERROR;
+		}
+		return NT_STATUS_OK;
+	}
+
+	ctx = talloc_zero(mem_ctx, struct vfs_offload_ctx);
+	if (ctx == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	ctx->db_ctx = db_open_rbt(mem_ctx);
+	if (ctx->db_ctx == NULL) {
+		TALLOC_FREE(ctx);
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	ctx->initialized = true;
+	*_ctx = ctx;
+	return NT_STATUS_OK;
+}
+
+struct fsp_token_link {
+	struct vfs_offload_ctx *ctx;
+	DATA_BLOB token_blob;
+};
+
+static int fsp_token_link_destructor(struct fsp_token_link *link)
+{
+	DATA_BLOB token_blob = link->token_blob;
+	TDB_DATA key = make_tdb_data(token_blob.data, token_blob.length);
+	NTSTATUS status;
+
+	status = dbwrap_delete(link->ctx->db_ctx, key);
+	if (!NT_STATUS_IS_OK(status)) {
+		DBG_ERR("dbwrap_delete failed: %s. Token:\n", nt_errstr(status));
+		dump_data(0, token_blob.data, token_blob.length);
+		return -1;
+	}
+
+	return 0;
+}
+
+NTSTATUS vfs_offload_token_db_store_fsp(struct vfs_offload_ctx *ctx,
+					const files_struct *fsp,
+					const DATA_BLOB *token_blob)
+{
+	struct db_record *rec = NULL;
+	struct fsp_token_link *link = NULL;
+	TDB_DATA key = make_tdb_data(token_blob->data, token_blob->length);
+	TDB_DATA value;
+	NTSTATUS status;
+
+	rec = dbwrap_fetch_locked(ctx->db_ctx, talloc_tos(), key);
+	if (rec == NULL) {
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	value = dbwrap_record_get_value(rec);
+	if (value.dsize != 0) {
+		void *ptr = NULL;
+		files_struct *token_db_fsp = NULL;
+
+		if (value.dsize != sizeof(ptr)) {
+			DBG_ERR("Bad db entry for token:\n");
+			dump_data(1, token_blob->data, token_blob->length);
+			TALLOC_FREE(rec);
+			return NT_STATUS_INTERNAL_ERROR;
+		}
+		memcpy(&ptr, value.dptr, value.dsize);
+		TALLOC_FREE(rec);
+
+		token_db_fsp = talloc_get_type_abort(ptr, struct files_struct);
+		if (token_db_fsp != fsp) {
+			DBG_ERR("token for fsp [%s] matches already known "
+				"but different fsp [%s]:\n",
+				fsp_str_dbg(fsp), fsp_str_dbg(token_db_fsp));
+			dump_data(1, token_blob->data, token_blob->length);
+			return NT_STATUS_INTERNAL_ERROR;
+		}
+
+		return NT_STATUS_OK;
+	}
+
+	link = talloc_zero(fsp, struct fsp_token_link);
+	if (link == NULL) {
+		return NT_STATUS_NO_MEMORY;
+	}
+	link->ctx = ctx;
+	link->token_blob = data_blob_talloc(link, token_blob->data,
+					    token_blob->length);
+	if (link->token_blob.data == NULL) {
+		TALLOC_FREE(link);
+		return NT_STATUS_NO_MEMORY;
+	}
+	talloc_set_destructor(link, fsp_token_link_destructor);
+
+	value = make_tdb_data((uint8_t *)&fsp, sizeof(files_struct *));
+
+	status = dbwrap_record_store(rec, value, 0);
+	if (!NT_STATUS_IS_OK(status)) {
+		DBG_ERR("dbwrap_record_store for [%s] failed: %s. Token\n",
+			fsp_str_dbg(fsp), nt_errstr(status));
+		dump_data(0, token_blob->data, token_blob->length);
+		TALLOC_FREE(link);
+		TALLOC_FREE(rec);
+		return status;
+	}
+
+	TALLOC_FREE(rec);
+	return NT_STATUS_OK;
+}
+
+NTSTATUS vfs_offload_token_db_fetch_fsp(struct vfs_offload_ctx *ctx,
+					const DATA_BLOB *token_blob,
+					files_struct **fsp)
+{
+	struct db_record *rec = NULL;
+	TDB_DATA key = make_tdb_data(token_blob->data, token_blob->length);
+	TDB_DATA value;
+	void *ptr = NULL;
+
+	rec = dbwrap_fetch_locked(ctx->db_ctx, talloc_tos(), key);
+	if (rec == NULL) {
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	value = dbwrap_record_get_value(rec);
+	if (value.dsize == 0) {
+		DBG_DEBUG("Unknown token:\n");
+		dump_data(10, token_blob->data, token_blob->length);
+		TALLOC_FREE(rec);
+		return NT_STATUS_OBJECT_NAME_NOT_FOUND;
+	}
+
+	if (value.dsize != sizeof(ptr)) {
+		DBG_ERR("Bad db entry for token:\n");
+		dump_data(1, token_blob->data, token_blob->length);
+		TALLOC_FREE(rec);
+		return NT_STATUS_INTERNAL_ERROR;
+	}
+
+	memcpy(&ptr, value.dptr, value.dsize);
+	TALLOC_FREE(rec);
+
+	*fsp = talloc_get_type_abort(ptr, struct files_struct);
+	return NT_STATUS_OK;
+}
+
+NTSTATUS vfs_offload_token_create_blob(TALLOC_CTX *mem_ctx,
+				       const files_struct *fsp,
+				       uint32_t fsctl,
+				       DATA_BLOB *token_blob)
+{
+	size_t len;
+
+	switch (fsctl) {
+	case FSCTL_DUP_EXTENTS_TO_FILE:
+		len = 20;
+		break;
+	case FSCTL_SRV_REQUEST_RESUME_KEY:
+		len = 24;
+		break;
+	default:
+		DBG_ERR("Invalid fsctl [%" PRIu32 "]\n", fsctl);
+		return NT_STATUS_NOT_SUPPORTED;
+	}
+
+	*token_blob = data_blob_talloc_zero(mem_ctx, len);
+	if (token_blob->length == 0) {
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	/* combine persistent and volatile handles for the resume key */
+	SBVAL(token_blob->data,  0, fsp->op->global->open_persistent_id);
+	SBVAL(token_blob->data,  8, fsp->op->global->open_volatile_id);
+	SIVAL(token_blob->data, 16, fsctl);
+
+	return NT_STATUS_OK;
+}
+
diff --git a/source3/modules/offload_token.h b/source3/modules/offload_token.h
new file mode 100644
index 00000000000..569edcfdd17
--- /dev/null
+++ b/source3/modules/offload_token.h
@@ -0,0 +1,37 @@
+/*
+   Unix SMB/Netbios implementation.
+   Copyright (c) 2017 Ralph Boehme <slow@samba.org>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _OFFLOAD_TOKEN_H_
+#define _OFFLOAD_TOKEN_H_
+
+struct vfs_offload_ctx;
+struct req_resume_key_rsp;
+
+NTSTATUS vfs_offload_token_ctx_init(TALLOC_CTX *mem_ctx,
+				    struct vfs_offload_ctx **_ctx);
+NTSTATUS vfs_offload_token_db_store_fsp(struct vfs_offload_ctx *ctx,
+					const files_struct *fsp,
+					const DATA_BLOB *token_blob);
+NTSTATUS vfs_offload_token_db_fetch_fsp(struct vfs_offload_ctx *ctx,
+					const DATA_BLOB *token_blob,
+					files_struct **fsp);
+NTSTATUS vfs_offload_token_create_blob(TALLOC_CTX *mem_ctx,
+				       const files_struct *fsp,
+				       uint32_t fsctl,
+				       DATA_BLOB *token_blob);
+#endif
diff --git a/source3/modules/vfs_btrfs.c b/source3/modules/vfs_btrfs.c
index e306ecec89f..b175a8437ce 100644
--- a/source3/modules/vfs_btrfs.c
+++ b/source3/modules/vfs_btrfs.c
@@ -27,9 +27,11 @@
 #include "system/filesys.h"
 #include "includes.h"
 #include "smbd/smbd.h"
+#include "smbd/globals.h"
 #include "librpc/gen_ndr/smbXsrv.h"
 #include "librpc/gen_ndr/ioctl.h"
 #include "lib/util/tevent_ntstatus.h"
+#include "offload_token.h"
 
 static uint32_t btrfs_fs_capabilities(struct vfs_handle_struct *handle,
 				      enum timestamp_set_resolution *_ts_res)
@@ -79,6 +81,121 @@ struct btrfs_ioctl_clone_range_args {
 #define BTRFS_IOC_SNAP_CREATE_V2 _IOW(BTRFS_IOCTL_MAGIC, 23, \
 				      struct btrfs_ioctl_vol_args_v2)
 
+static struct vfs_offload_ctx *btrfs_offload_ctx;
+
+struct btrfs_offload_read_state {
+	struct vfs_handle_struct *handle;
+	files_struct *fsp;
+	DATA_BLOB token;
+};
+
+static void btrfs_offload_read_done(struct tevent_req *subreq);
+
+static struct tevent_req *btrfs_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+	struct tevent_req *subreq = NULL;
+	struct btrfs_offload_read_state *state = NULL;
+	NTSTATUS status;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct btrfs_offload_read_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	*state = (struct btrfs_offload_read_state) {
+		.handle = handle,
+		.fsp = fsp,
+	};
+
+	status = vfs_offload_token_ctx_init(fsp->conn->sconn->client,
+					    &btrfs_offload_ctx);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	if (fsctl == FSCTL_DUP_EXTENTS_TO_FILE) {
+		status = vfs_offload_token_create_blob(state, fsp, fsctl,
+						       &state->token);
+		if (tevent_req_nterror(req, status)) {
+			return tevent_req_post(req, ev);
+		}
+
+		status = vfs_offload_token_db_store_fsp(btrfs_offload_ctx, fsp,
+							&state->token);
+		if (tevent_req_nterror(req, status)) {
+			return tevent_req_post(req, ev);
+		}
+		tevent_req_done(req);
+		return tevent_req_post(req, ev);
+	}
+
+	subreq = SMB_VFS_NEXT_OFFLOAD_READ_SEND(mem_ctx, ev, handle, fsp,
+						fsctl, ttl, offset, to_copy);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, btrfs_offload_read_done, req);
+	return req;
+}
+
+static void btrfs_offload_read_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct btrfs_offload_read_state *state = tevent_req_data(
+		req, struct btrfs_offload_read_state);
+	NTSTATUS status;
+
+	status = SMB_VFS_NEXT_OFFLOAD_READ_RECV(subreq,
+						state->handle,
+						state,
+						&state->token);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	status = vfs_offload_token_db_store_fsp(btrfs_offload_ctx,
+						state->fsp,
+						&state->token);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	tevent_req_done(req);
+	return;
+}
+
+static NTSTATUS btrfs_offload_read_recv(struct tevent_req *req,
+					struct vfs_handle_struct *handle,
+					TALLOC_CTX *mem_ctx,
+					DATA_BLOB *token)
+{
+	struct btrfs_offload_read_state *state = tevent_req_data(
+		req, struct btrfs_offload_read_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+
+	token->length = state->token.length;
+	token->data = talloc_move(mem_ctx, &state->token.data);
+
+	tevent_req_received(req);
+	return NT_STATUS_OK;
+}
+
 struct btrfs_cc_state {
 	struct vfs_handle_struct *handle;
 	off_t copied;
@@ -681,6 +798,8 @@ static NTSTATUS btrfs_snap_delete(struct vfs_handle_struct *handle,
 
 static struct vfs_fn_pointers btrfs_fns = {
 	.fs_capabilities_fn = btrfs_fs_capabilities,
+	.offload_read_send_fn = btrfs_offload_read_send,
+	.offload_read_recv_fn = btrfs_offload_read_recv,
 	.copy_chunk_send_fn = btrfs_copy_chunk_send,
 	.copy_chunk_recv_fn = btrfs_copy_chunk_recv,
 	.get_compression_fn = btrfs_get_compression,
diff --git a/source3/modules/vfs_default.c b/source3/modules/vfs_default.c
index 8a08aed83f8..4b5e6f15f13 100644
--- a/source3/modules/vfs_default.c
+++ b/source3/modules/vfs_default.c
@@ -34,6 +34,7 @@
 #include "lib/util/sys_rw.h"
 #include "lib/pthreadpool/pthreadpool_tevent.h"
 #include "librpc/gen_ndr/ndr_ioctl.h"
+#include "offload_token.h"
 
 #undef DBGC_CLASS
 #define DBGC_CLASS DBGC_VFS
@@ -1607,6 +1608,80 @@ static NTSTATUS vfswrap_fset_dos_attributes(struct vfs_handle_struct *handle,
 	return set_ea_dos_attribute(handle->conn, fsp->fsp_name, dosmode);
 }
 
+static struct vfs_offload_ctx *vfswrap_offload_ctx;
+
+struct vfswrap_offload_read_state {
+	DATA_BLOB token;
+};
+
+static struct tevent_req *vfswrap_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	struct files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+	struct vfswrap_offload_read_state *state = NULL;
+	NTSTATUS status;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct vfswrap_offload_read_state);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	status = vfs_offload_token_ctx_init(fsp->conn->sconn->client,
+					    &vfswrap_offload_ctx);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	if (fsctl != FSCTL_SRV_REQUEST_RESUME_KEY) {
+		tevent_req_nterror(req, NT_STATUS_INVALID_DEVICE_REQUEST);
+		return tevent_req_post(req, ev);
+	}
+
+	status = vfs_offload_token_create_blob(state, fsp, fsctl,
+					       &state->token);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	status = vfs_offload_token_db_store_fsp(vfswrap_offload_ctx, fsp,
+						&state->token);
+	if (tevent_req_nterror(req, status)) {
+		return tevent_req_post(req, ev);
+	}
+
+	tevent_req_done(req);
+	return tevent_req_post(req, ev);
+}
+
+static NTSTATUS vfswrap_offload_read_recv(struct tevent_req *req,
+					  struct vfs_handle_struct *handle,
+					  TALLOC_CTX *mem_ctx,
+					  DATA_BLOB *token)
+{
+	struct vfswrap_offload_read_state *state = tevent_req_data(
+		req, struct vfswrap_offload_read_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+
+	token->length = state->token.length;
+	token->data = talloc_move(mem_ctx, &state->token.data);
+
+	tevent_req_received(req);
+	return NT_STATUS_OK;
+}
+
 struct vfs_cc_state {
 	struct tevent_context *ev;
 	uint8_t *buf;
@@ -3002,6 +3077,8 @@ static struct vfs_fn_pointers vfs_default_fns = {
 	.fset_dos_attributes_fn = vfswrap_fset_dos_attributes,
 	.get_dos_attributes_fn = vfswrap_get_dos_attributes,
 	.fget_dos_attributes_fn = vfswrap_fget_dos_attributes,
+	.offload_read_send_fn = vfswrap_offload_read_send,
+	.offload_read_recv_fn = vfswrap_offload_read_recv,
 	.copy_chunk_send_fn = vfswrap_copy_chunk_send,
 	.copy_chunk_recv_fn = vfswrap_copy_chunk_recv,
 	.get_compression_fn = vfswrap_get_compression,
diff --git a/source3/modules/vfs_fruit.c b/source3/modules/vfs_fruit.c
index c4277b9ec34..20d15b39af7 100644
--- a/source3/modules/vfs_fruit.c
+++ b/source3/modules/vfs_fruit.c
@@ -32,6 +32,7 @@
 #include "lib/util/sys_rw.h"
 #include "lib/util/tevent_ntstatus.h"
 #include "lib/util/tevent_unix.h"
+#include "offload_token.h"
 
 /*
  * Enhanced OS X and Netatalk compatibility
@@ -5365,6 +5366,113 @@ static NTSTATUS fruit_fset_nt_acl(vfs_handle_struct *handle,
 	return NT_STATUS_OK;
 }
 
+static struct vfs_offload_ctx *fruit_offload_ctx;
+
+struct fruit_offload_read_state {
+	struct vfs_handle_struct *handle;
+	struct tevent_context *ev;
+	files_struct *fsp;
+	uint32_t fsctl;
+	DATA_BLOB token;
+};
+
+static void fruit_offload_read_done(struct tevent_req *subreq);
+
+static struct tevent_req *fruit_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+	struct tevent_req *subreq = NULL;
+	struct fruit_offload_read_state *state = NULL;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct fruit_offload_read_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	*state = (struct fruit_offload_read_state) {
+		.handle = handle,
+		.ev = ev,
+		.fsp = fsp,
+		.fsctl = fsctl,
+	};
+
+	subreq = SMB_VFS_NEXT_OFFLOAD_READ_SEND(mem_ctx, ev, handle, fsp,
+						fsctl, ttl, offset, to_copy);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+	tevent_req_set_callback(subreq, fruit_offload_read_done, req);
+	return req;
+}
+
+static void fruit_offload_read_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct fruit_offload_read_state *state = tevent_req_data(
+		req, struct fruit_offload_read_state);
+	NTSTATUS status;
+
+	status = SMB_VFS_NEXT_OFFLOAD_READ_RECV(subreq,
+						state->handle,
+						state,
+						&state->token);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	if (state->fsctl != FSCTL_SRV_REQUEST_RESUME_KEY) {
+		tevent_req_done(req);
+		return;
+	}
+
+	status = vfs_offload_token_ctx_init(state->fsp->conn->sconn->client,
+					    &fruit_offload_ctx);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	status = vfs_offload_token_db_store_fsp(fruit_offload_ctx,
+						state->fsp,
+						&state->token);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+
+	tevent_req_done(req);
+	return;
+}
+
+static NTSTATUS fruit_offload_read_recv(struct tevent_req *req,
+					struct vfs_handle_struct *handle,
+					TALLOC_CTX *mem_ctx,
+					DATA_BLOB *token)
+{
+	struct fruit_offload_read_state *state = tevent_req_data(
+		req, struct fruit_offload_read_state);
+	NTSTATUS status;
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+
+	token->length = state->token.length;
+	token->data = talloc_move(mem_ctx, &state->token.data);
+
+	tevent_req_received(req);
+	return NT_STATUS_OK;
+}
+
 struct fruit_copy_chunk_state {
 	struct vfs_handle_struct *handle;
 	off_t copied;
@@ -5590,6 +5698,8 @@ static struct vfs_fn_pointers vfs_fruit_fns = {
 	.fallocate_fn = fruit_fallocate,
 	.create_file_fn = fruit_create_file,
 	.readdir_attr_fn = fruit_readdir_attr,
+	.offload_read_send_fn = fruit_offload_read_send,
+	.offload_read_recv_fn = fruit_offload_read_recv,
 	.copy_chunk_send_fn = fruit_copy_chunk_send,
 	.copy_chunk_recv_fn = fruit_copy_chunk_recv,
 
diff --git a/source3/modules/vfs_full_audit.c b/source3/modules/vfs_full_audit.c
index 215cb1f3934..abf74074fbe 100644
--- a/source3/modules/vfs_full_audit.c
+++ b/source3/modules/vfs_full_audit.c
@@ -168,6 +168,8 @@ typedef enum _vfs_op_type {
 	SMB_VFS_OP_STRICT_UNLOCK,
 	SMB_VFS_OP_TRANSLATE_NAME,
 	SMB_VFS_OP_FSCTL,
+	SMB_VFS_OP_OFFLOAD_READ_SEND,
+	SMB_VFS_OP_OFFLOAD_READ_RECV,
 	SMB_VFS_OP_COPY_CHUNK_SEND,
 	SMB_VFS_OP_COPY_CHUNK_RECV,
 	SMB_VFS_OP_GET_COMPRESSION,
@@ -310,6 +312,8 @@ static struct {
 	{ SMB_VFS_OP_STRICT_UNLOCK, "strict_unlock" },
 	{ SMB_VFS_OP_TRANSLATE_NAME,	"translate_name" },
 	{ SMB_VFS_OP_FSCTL,		"fsctl" },
+	{ SMB_VFS_OP_OFFLOAD_READ_SEND,	"offload_read_send" },
+	{ SMB_VFS_OP_OFFLOAD_READ_RECV,	"offload_read_recv" },
 	{ SMB_VFS_OP_COPY_CHUNK_SEND,	"copy_chunk_send" },
 	{ SMB_VFS_OP_COPY_CHUNK_RECV,	"copy_chunk_recv" },
 	{ SMB_VFS_OP_GET_COMPRESSION,	"get_compression" },
@@ -1901,6 +1905,42 @@ static NTSTATUS smb_full_audit_fsctl(struct vfs_handle_struct *handle,
 	return result;
 }
 
+static struct tevent_req *smb_full_audit_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	struct files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+
+	req = SMB_VFS_NEXT_OFFLOAD_READ_SEND(mem_ctx, ev, handle, fsp,
+					     fsctl, ttl, offset, to_copy);
+
+	do_log(SMB_VFS_OP_OFFLOAD_READ_SEND, req, handle, "");
+
+	return req;
+}
+
+static NTSTATUS smb_full_audit_offload_read_recv(
+	struct tevent_req *req,
+	struct vfs_handle_struct *handle,
+	TALLOC_CTX *mem_ctx,
+	DATA_BLOB *_token_blob)
+{
+	NTSTATUS status;
+
+	status = SMB_VFS_NEXT_OFFLOAD_READ_RECV(req, handle, mem_ctx,
+						_token_blob);
+
+	do_log(SMB_VFS_OP_OFFLOAD_READ_RECV, NT_STATUS_IS_OK(status), handle, "");
+
+	return status;
+}
+
 static struct tevent_req *smb_full_audit_copy_chunk_send(struct vfs_handle_struct *handle,
 							 TALLOC_CTX *mem_ctx,
 							 struct tevent_context *ev,
@@ -2534,6 +2574,8 @@ static struct vfs_fn_pointers vfs_full_audit_fns = {
 	.realpath_fn = smb_full_audit_realpath,
 	.chflags_fn = smb_full_audit_chflags,
 	.file_id_create_fn = smb_full_audit_file_id_create,
+	.offload_read_send_fn = smb_full_audit_offload_read_send,
+	.offload_read_recv_fn = smb_full_audit_offload_read_recv,
 	.copy_chunk_send_fn = smb_full_audit_copy_chunk_send,
 	.copy_chunk_recv_fn = smb_full_audit_copy_chunk_recv,
 	.get_compression_fn = smb_full_audit_get_compression,
diff --git a/source3/modules/vfs_time_audit.c b/source3/modules/vfs_time_audit.c
index 2f7c3d39dea..a13adfa9225 100644
--- a/source3/modules/vfs_time_audit.c
+++ b/source3/modules/vfs_time_audit.c
@@ -1898,6 +1898,103 @@ static NTSTATUS smb_time_fset_dos_attributes(struct vfs_handle_struct *handle,
 	return result;
 }
 
+struct time_audit_offload_read_state {
+	struct vfs_handle_struct *handle;
+	struct timespec ts_send;
+	DATA_BLOB token_blob;
+};
+
+static void smb_time_audit_offload_read_done(struct tevent_req *subreq);
+
+static struct tevent_req *smb_time_audit_offload_read_send(
+	TALLOC_CTX *mem_ctx,
+	struct tevent_context *ev,
+	struct vfs_handle_struct *handle,
+	struct files_struct *fsp,
+	uint32_t fsctl,
+	uint32_t ttl,
+	off_t offset,
+	size_t to_copy)
+{
+	struct tevent_req *req = NULL;
+	struct tevent_req *subreq = NULL;
+	struct time_audit_offload_read_state *state = NULL;
+
+	req = tevent_req_create(mem_ctx, &state,
+				struct time_audit_offload_read_state);
+	if (req == NULL) {
+		return NULL;
+	}
+	state->handle = handle;
+	clock_gettime_mono(&state->ts_send);
+
+	subreq = SMB_VFS_NEXT_OFFLOAD_READ_SEND(mem_ctx, ev,
+						handle, fsp,
+						fsctl, ttl,
+						offset, to_copy);
+	if (tevent_req_nomem(subreq, req)) {
+		return tevent_req_post(req, ev);
+	}
+
+	tevent_req_set_callback(subreq, smb_time_audit_offload_read_done, req);
+	return req;
+}
+
+static void smb_time_audit_offload_read_done(struct tevent_req *subreq)
+{
+	struct tevent_req *req = tevent_req_callback_data(
+		subreq, struct tevent_req);
+	struct time_audit_offload_read_state *state = tevent_req_data(
+		req, struct time_audit_offload_read_state);
+	NTSTATUS status;
+
+	status = SMB_VFS_NEXT_OFFLOAD_READ_RECV(subreq,
+						state->handle,
+						state,
+						&state->token_blob);
+	TALLOC_FREE(subreq);
+	if (tevent_req_nterror(req, status)) {
+		return;
+	}
+	tevent_req_done(req);
+}
+
+static NTSTATUS smb_time_audit_offload_read_recv(
+	struct tevent_req *req,
+	struct vfs_handle_struct *handle,
+	TALLOC_CTX *mem_ctx,
+	DATA_BLOB *_token_blob)
+{
+	struct time_audit_offload_read_state *state = tevent_req_data(
+		req, struct time_audit_offload_read_state);
+	struct timespec ts_recv;
+	double timediff;
+	DATA_BLOB token_blob;
+	NTSTATUS status;
+
+	clock_gettime_mono(&ts_recv);
+	timediff = nsec_time_diff(&ts_recv, &state->ts_send) * 1.0e-9;
+	if (timediff > audit_timeout) {
+		smb_time_audit_log("offload_read", timediff);
+	}
+
+	if (tevent_req_is_nterror(req, &status)) {
+		tevent_req_received(req);
+		return status;
+	}
+
+	token_blob = data_blob_talloc(mem_ctx,
+				      state->token_blob.data,
+				      state->token_blob.length);
+	if (token_blob.data == NULL) {
+		tevent_req_received(req);
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	tevent_req_received(req);
+	return NT_STATUS_OK;
+}
+
 struct time_audit_cc_state {
 	struct timespec ts_send;
 	struct vfs_handle_struct *handle;
@@ -2672,6 +2769,8 @@ static struct vfs_fn_pointers vfs_time_audit_fns = {
 	.realpath_fn = smb_time_audit_realpath,
 	.chflags_fn = smb_time_audit_chflags,
 	.file_id_create_fn = smb_time_audit_file_id_create,
+	.offload_read_send_fn = smb_time_audit_offload_read_send,
+	.offload_read_recv_fn = smb_time_audit_offload_read_recv,
 	.copy_chunk_send_fn = smb_time_audit_copy_chunk_send,
 	.copy_chunk_recv_fn = smb_time_audit_copy_chunk_recv,
 	.get_compression_fn = smb_time_audit_get_compression,
diff --git a/source3/modules/wscript_build b/source3/modules/wscript_build
index a5d84075872..840fdef7757 100644
--- a/source3/modules/wscript_build
+++ b/source3/modules/wscript_build
@@ -22,10 +22,14 @@ bld.SAMBA3_SUBSYSTEM('vfs',
                     source='',
                     deps='smbd_base')
 
+bld.SAMBA3_SUBSYSTEM('OFFLOAD_TOKEN',
+                    source='offload_token.c',
+                    deps='samba-util')
+
 bld.SAMBA3_MODULE('vfs_default',
                  subsystem='vfs',
                  source='vfs_default.c',
-                 deps='samba-util NDR_DFSBLOBS',
+                 deps='samba-util NDR_DFSBLOBS OFFLOAD_TOKEN',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_default'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_default'))
@@ -89,7 +93,7 @@ bld.SAMBA3_MODULE('vfs_netatalk',
 bld.SAMBA3_MODULE('vfs_fruit',
                  subsystem='vfs',
                  source='vfs_fruit.c',
-                 deps='samba-util',
+                 deps='samba-util OFFLOAD_TOKEN',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_fruit'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_fruit'))
@@ -425,7 +429,7 @@ bld.SAMBA3_MODULE('vfs_dfs_samba4',
 bld.SAMBA3_MODULE('vfs_btrfs',
                  subsystem='vfs',
                  source='vfs_btrfs.c',
-                 deps='samba-util',
+                 deps='samba-util OFFLOAD_TOKEN',
                  init_function='',
                  internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_btrfs'),
                  enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_btrfs'))
diff --git a/source3/smbd/vfs.c b/source3/smbd/vfs.c
index 8304ef18b18..2088dbe7979 100644
--- a/source3/smbd/vfs.c
+++ b/source3/smbd/vfs.c
@@ -2350,6 +2350,30 @@ NTSTATUS smb_vfs_call_fset_dos_attributes(struct vfs_handle_struct *handle,
 	return handle->fns->fset_dos_attributes_fn(handle, fsp, dosmode);
 }
 
+struct tevent_req *smb_vfs_call_offload_read_send(TALLOC_CTX *mem_ctx,
+						  struct tevent_context *ev,
+						  struct vfs_handle_struct *handle,
+						  struct files_struct *fsp,
+						  uint32_t fsctl,
+						  uint32_t ttl,
+						  off_t offset,
+						  size_t to_copy)
+{
+	VFS_FIND(offload_read_send);
+	return handle->fns->offload_read_send_fn(mem_ctx, ev, handle,
+						 fsp, fsctl,
+						 ttl, offset, to_copy);
+}
+
+NTSTATUS smb_vfs_call_offload_read_recv(struct tevent_req *req,
+					struct vfs_handle_struct *handle,
+					TALLOC_CTX *mem_ctx,
+					DATA_BLOB *token_blob)
+{
+	VFS_FIND(offload_read_recv);
+	return handle->fns->offload_read_recv_fn(req, handle, mem_ctx, token_blob);
+}
+
 struct tevent_req *smb_vfs_call_copy_chunk_send(struct vfs_handle_struct *handle,
 						TALLOC_CTX *mem_ctx,
 						struct tevent_context *ev,