path: root/drm/nouveau/nvkm/subdev/secboot/gm200.c

/*
 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

/*
 * Secure boot is the process by which NVIDIA-signed firmware is loaded into
 * some of the falcons of a GPU. For production devices this is the only way
 * for the firmware to access useful (but sensitive) registers.
 *
 * A Falcon microprocessor supporting advanced security modes can run in one of
 * three modes:
 *
 * - Non-secure (NS). In this mode, functionality is similar to Falcon
 *   architectures before security modes were introduced (pre-Maxwell), but
 *   capability is restricted. In particular, certain registers may be
 *   inaccessible for reads and/or writes, and physical memory access may be
 *   disabled (on certain Falcon instances). This is the only possible mode that
 *   can be used if you don't have microcode cryptographically signed by NVIDIA.
 *
 * - Heavy Secure (HS). In this mode, the microprocessor is a black box - it's
 *   not possible to read or write any Falcon internal state or Falcon registers
 *   from outside the Falcon (for example, from the host system). The only way
 *   to enable this mode is by loading microcode that has been signed by NVIDIA.
 *   (The loading process involves tagging the IMEM block as secure, writing the
 *   signature into a Falcon register, and starting execution. The hardware will
 *   validate the signature, and if valid, grant HS privileges.)
 *
 * - Light Secure (LS). In this mode, the microprocessor has more privileges
 *   than NS but fewer than HS. Some of the microprocessor state is visible to
 *   host software to ease debugging. The only way to enable this mode is by HS
 *   microcode enabling LS mode. Some privileges available to HS mode are not
 *   available here. LS mode is introduced in GM20x.
 *
 * Secure boot consists in temporarily switchin a HS-capable falcon (typically
 * PMU) into HS mode in order to validate the LS firmware of managed falcons,
 * load it, and switch managed falcons into LS mode. Once secure boot completes,
 * no falcon remains in HS mode.
 *
 * Secure boot requires a write-protected memory region (WPR) which can only be
 * written by the secure falcon. On dGPU, the driver sets up the WPR region in
 * video memory. On Tegra, it is set up by the bootloader and its location and
 * size written into memory controller registers.
 *
 * The secure boot process takes place as follows:
 *
 * 1) A LS blob is constructed that contains all the LS firmwares we want to
 *    load, along with their signatures and bootloaders.
 *
 * 2) A HS blob (also called ACR) is created that contains the signed HS
 *    firmware in charge of loading the LS firmwares into their respective
 *    falcons.
 *
 * 3) The HS blob is loaded (via its own bootloader) and executed on the
 *    HS-capable falcon. It authenticates itself, switches the secure falcon to
 *    HS mode and setup the WPR region around the LS blob (dGPU) or copies the
 *    LS blob into the WPR region (Tegra).
 *
 * 4) The LS blob is now secure from all external tampering. The HS falcon
 *    checks the signatures of the LS firmwares and, if valid, switches the
 *    managed falcons to LS mode and makes them ready to run the LS firmware.
 *
 * 5) The managed falcons remain in LS mode and can be started.
 *
 */

#include "priv.h"

#include <core/gpuobj.h>
#include <core/firmware.h>
#include <subdev/fb.h>

enum {
	FALCON_DMAIDX_UCODE		= 0,
	FALCON_DMAIDX_VIRT		= 1,
	FALCON_DMAIDX_PHYS_VID		= 2,
	FALCON_DMAIDX_PHYS_SYS_COH	= 3,
	FALCON_DMAIDX_PHYS_SYS_NCOH	= 4,
};

/*
 *
 * LS blob structures
 *
 */

/**
 * struct lsf_ucode_desc - LS falcon signatures
 * @prd_keys:		signature to use when the GPU is in production mode
 * @dgb_keys:		signature to use when the GPU is in debug mode
 * @b_prd_present:	whether the production key is present
 * @b_dgb_present:	whether the debug key is present
 * @falcon_id:		ID of the falcon the ucode applies to
 *
 * Directly loaded from a signature file.
 */
struct lsf_ucode_desc {
	u8  prd_keys[2][16];
	u8  dbg_keys[2][16];
	u32 b_prd_present;
	u32 b_dbg_present;
	u32 falcon_id;
};

/**
 * struct lsf_lsb_header - LS firmware header
 * @signature:		signature to verify the firmware against
 * @ucode_off:		offset of the ucode blob in the WPR region. The ucode
 *                      blob contains the bootloader, code and data of the
 *                      LS falcon
 * @ucode_size:		size of the ucode blob, including bootloader
 * @data_size:		size of the ucode blob data
 * @bl_code_size:	size of the bootloader code
 * @bl_imem_off:	offset in imem of the bootloader
 * @bl_data_off:	offset of the bootloader data in WPR region
 * @bl_data_size:	size of the bootloader data
 * @app_code_off:	offset of the app code relative to ucode_off
 * @app_code_size:	size of the app code
 * @app_data_off:	offset of the app data relative to ucode_off
 * @app_data_size:	size of the app data
 * @flags:		flags for the secure bootloader
 *
 * This structure is written into the WPR region for each managed falcon. Each
 * instance is referenced by the lsb_offset member of the corresponding
 * lsf_wpr_header.
 */
struct lsf_lsb_header {
	struct lsf_ucode_desc signature;
	u32 ucode_off;
	u32 ucode_size;
	u32 data_size;
	u32 bl_code_size;
	u32 bl_imem_off;
	u32 bl_data_off;
	u32 bl_data_size;
	u32 app_code_off;
	u32 app_code_size;
	u32 app_data_off;
	u32 app_data_size;
	u32 flags;
#define LSF_FLAG_LOAD_CODE_AT_0		1
#define LSF_FLAG_DMACTL_REQ_CTX		4
#define LSF_FLAG_FORCE_PRIV_LOAD	8
};

/**
 * struct lsf_wpr_header - LS blob WPR Header
 * @falcon_id:		LS falcon ID
 * @lsb_offset:		offset of the lsb_lsf_header in the WPR region
 * @bootstrap_owner:	secure falcon reponsible for bootstrapping the LS falcon
 * @lazy_bootstrap:	skip bootstrapping by ACR
 * @status:		bootstrapping status
 *
 * An array of these is written at the beginning of the WPR region, one for
 * each managed falcon. The array is terminated by an instance which falcon_id
 * is LSF_FALCON_ID_INVALID.
 */
struct lsf_wpr_header {
	u32  falcon_id;
	u32  lsb_offset;
	u32  bootstrap_owner;
	u32  lazy_bootstrap;
	u32  status;
#define LSF_IMAGE_STATUS_NONE				0
#define LSF_IMAGE_STATUS_COPY				1
#define LSF_IMAGE_STATUS_VALIDATION_CODE_FAILED		2
#define LSF_IMAGE_STATUS_VALIDATION_DATA_FAILED		3
#define LSF_IMAGE_STATUS_VALIDATION_DONE		4
#define LSF_IMAGE_STATUS_VALIDATION_SKIPPED		5
#define LSF_IMAGE_STATUS_BOOTSTRAP_READY		6
};


/**
 * struct ls_ucode_desc - descriptor of firmware image
 * @descriptor_size:		size of this descriptor
 * @image_size:			size of the whole image
 * @bootloader_start_offset:	start offset of the bootloader in ucode image
 * @bootloader_size:		size of the bootloader
 * @bootloader_imem_offset:	start off set of the bootloader in IMEM
 * @bootloader_entry_point:	entry point of the bootloader in IMEM
 * @app_start_offset:		start offset of the LS firmware
 * @app_size:			size of the LS firmware's code and data
 * @app_imem_offset:		offset of the app in IMEM
 * @app_imem_entry:		entry point of the app in IMEM
 * @app_dmem_offset:		offset of the data in DMEM
 * @app_resident_code_offset:	offset of app code from app_start_offset
 * @app_resident_code_size:	size of the code
 * @app_resident_data_offset:	offset of data from app_start_offset
 * @app_resident_data_size:	size of data
 *
 * A firmware image contains the code, data, and bootloader of a given LS
 * falcon in a single blob. This structure describes where everything is.
 *
 * This can be generated from a (bootloader, code, data) set if they have
 * been loaded separately, or come directly from a file. For the later case,
 * we need to keep the fields that are unused by the code.
 */
struct ls_ucode_desc {
	u32 descriptor_size;
	u32 image_size;
	u32 tools_version;
	u32 app_version;
	char date[64];
	u32 bootloader_start_offset;
	u32 bootloader_size;
	u32 bootloader_imem_offset;
	u32 bootloader_entry_point;
	u32 app_start_offset;
	u32 app_size;
	u32 app_imem_offset;
	u32 app_imem_entry;
	u32 app_dmem_offset;
	u32 app_resident_code_offset;
	u32 app_resident_code_size;
	u32 app_resident_data_offset;
	u32 app_resident_data_size;
	u32 nb_overlays;
	struct {u32 start; u32 size; } load_ovl[32];
	u32 compressed;
};

/**
 * struct ls_ucode_img - temporary storage for loaded LS firmwares
 * @node:		to link within lsf_ucode_mgr
 * @falcon_id:		ID of the falcon this LS firmware is for
 * @ucode_desc:		loaded or generated map of ucode_data
 * @ucode_header:	header of the firmware
 * @ucode_data:		firmware payload (code and data)
 * @ucode_size:		size in bytes of data in ucode_data
 * @wpr_header:		WPR header to be written to the LS blob
 * @lsb_header:		LSB header to be written to the LS blob
 *
 * Preparing the WPR LS blob requires information about all the LS firmwares
 * (size, etc) to be known. This structure contains all the data of one LS
 * firmware.
 */
struct ls_ucode_img {
	struct list_head node;
	enum nvkm_secboot_falcon falcon_id;

	struct ls_ucode_desc ucode_desc;
	u32 *ucode_header;
	u8 *ucode_data;
	u32 ucode_size;

	struct lsf_wpr_header wpr_header;
	struct lsf_lsb_header lsb_header;
};

/**
 * struct lsf_ucode_mgr - manager for all LS falcon firmwares
 * @count:	number of managed LS falcons
 * @wpr_size:	size of the required WPR region in bytes
 * @img_list:	linked list of lsf_ucode_img
 */
struct ls_ucode_mgr {
	u16 count;
	u32 wpr_size;
	struct list_head img_list;
};

/*
 *
 * HS blob structures
 *
 */

/**
 * struct hs_bin_hdr - header of HS firmware and bootloader files
 * @bin_magic:		always 0x10de
 * @bin_ver:		version of the bin format
 * @bin_size:		entire image size including this header
 * @header_offset:	offset of the firmware/bootloader header in the file
 * @data_offset:	offset of the firmware/bootloader payload in the file
 * @data_size:		size of the payload
 *
 * This header is located at the beginning of the HS firmware and HS bootloader
 * files, to describe where the headers and data can be found.
 */
struct hsf_bin_hdr {
	u32 bin_magic;
	u32 bin_ver;
	u32 bin_size;
	u32 header_offset;
	u32 data_offset;
	u32 data_size;
};

/**
 * struct hsf_bl_desc - HS firmware bootloader descriptor
 * @bl_start_tag:		starting tag of bootloader
 * @bl_desc_dmem_load_off:	DMEM offset of flcn_bl_dmem_desc
 * @bl_code_off:		offset of code section
 * @bl_code_size:		size of code section
 * @bl_data_off:		offset of data section
 * @bl_data_size:		size of data section
 *
 * This structure is embedded in the HS bootloader firmware file at
 * hs_bin_hdr.header_offset to describe the IMEM and DMEM layout expected by the
 * HS bootloader.
 */
struct hsf_bl_desc {
	u32 bl_start_tag;
	u32 bl_desc_dmem_load_off;
	u32 bl_code_off;
	u32 bl_code_size;
	u32 bl_data_off;
	u32 bl_data_size;
};

/**
 * struct hsf_fw_header - HS firmware descriptor
 * @sig_dbg_offset:	offset of the debug signature
 * @sig_dbg_size:	size of the debug signature
 * @sig_prod_offset:	offset of the production signature
 * @sig_prod_size:	size of the production signature
 * @patch_loc:		offset of the offset (sic) of where the signature is
 * @patch_sig:		offset of the offset (sic) to add to sig_*_offset
 * @hdr_offset:		offset of the load header (see struct hs_load_header)
 * @hdr_size:		size of above header
 *
 * This structure is embedded in the HS firmware image at
 * hs_bin_hdr.header_offset.
 */
struct hsf_fw_header {
	u32 sig_dbg_offset;
	u32 sig_dbg_size;
	u32 sig_prod_offset;
	u32 sig_prod_size;
	u32 patch_loc;
	u32 patch_sig;
	u32 hdr_offset;
	u32 hdr_size;
};

/**
 * struct hsf_load_header - HS firmware loading header
 *
 * Data to be copied as-is into the struct flcn_bl_dmem_desc for the HS firmware
 */
struct hsf_load_header {
	u32 non_sec_code_off;
	u32 non_sec_code_size;
	u32 data_dma_base;
	u32 data_size;
	u32 reserved;
	u32 sec_code_off;
	u32 sec_code_size;
};

/**
 * Convenience function to duplicate a firmware file in memory and check that
 * it has the required minimum size.
 */
static void *
gm200_secboot_load_firmware(struct nvkm_device *device, const char *name,
		    size_t min_size)
{
	const struct firmware *fw;
	void *blob;
	int ret;

	ret = nvkm_firmware_get(device, name, &fw);
	if (ret)
		return ERR_PTR(ret);
	if (fw->size < min_size) {
		nvkm_firmware_put(fw);
		return ERR_PTR(-EINVAL);
	}
	blob = kmemdup(fw->data, fw->size, GFP_KERNEL);
	nvkm_firmware_put(fw);
	if (!blob)
		return ERR_PTR(-ENOMEM);

	return blob;
}


/*
 * Low-secure blob creation
 */

#define BL_DESC_BLK_SIZE 256
/**
 * Build a ucode image and descriptor from provided bootloader, code and data.
 *
 * @bl:		bootloader image, including 16-bytes descriptor
 * @code:	LS firmware code segment
 * @data:	LS firmware data segment
 * @desc:	ucode descriptor to be written
 *
 * Return: allocated ucode image with corresponding descriptor information. desc
 *         is also updated to contain the right offsets within returned image.
 */
static void *
ls_ucode_img_build(const struct firmware *bl, const struct firmware *code,
		   const struct firmware *data, struct ls_ucode_desc *desc)
{
	struct {
		u32 start_offset;
		u32 size;
		u32 imem_offset;
		u32 entry_point;
	} *bl_desc;
	u32 *bl_image;
	u32 pos = 0;
	u8 *image;

	bl_desc = (void *)bl->data;
	bl_image = (void *)(bl_desc + 1);

	desc->bootloader_start_offset = pos;
	desc->bootloader_size = ALIGN(bl_desc->size, sizeof(u32));
	desc->bootloader_imem_offset = bl_desc->imem_offset;
	desc->bootloader_entry_point = bl_desc->entry_point;

	pos = ALIGN(pos + desc->bootloader_size, BL_DESC_BLK_SIZE);
	desc->app_start_offset = pos;
	desc->app_size = ALIGN(code->size, BL_DESC_BLK_SIZE) +
			 ALIGN(data->size, BL_DESC_BLK_SIZE);
	desc->app_imem_offset = 0;
	desc->app_imem_entry = 0;
	desc->app_dmem_offset = 0;
	desc->app_resident_code_offset = 0;
	desc->app_resident_code_size = ALIGN(code->size, BL_DESC_BLK_SIZE);

	pos = ALIGN(pos + desc->app_resident_code_size, BL_DESC_BLK_SIZE);
	desc->app_resident_data_offset = pos - desc->app_start_offset;
	desc->app_resident_data_size = ALIGN(data->size, BL_DESC_BLK_SIZE);

	desc->image_size = ALIGN(bl_desc->size, BL_DESC_BLK_SIZE) +
			   desc->app_size;

	image = kzalloc(desc->image_size, GFP_KERNEL);
	if (!image)
		return ERR_PTR(-ENOMEM);

	memcpy(image + desc->bootloader_start_offset, bl_image, bl_desc->size);
	memcpy(image + desc->app_start_offset, code->data, code->size);
	memcpy(image + desc->app_start_offset + desc->app_resident_data_offset,
	       data->data, data->size);

	return image;
}

/**
 * ls_ucode_img_load_generic() - load and prepare a LS ucode image
 *
 * Load the LS microcode, bootloader and signature and pack them into a single
 * blob. Also generate the corresponding ucode descriptor.
 */
static int
ls_ucode_img_load_generic(struct nvkm_device *device,
			  struct ls_ucode_img *img, const char *falcon_name,
			  const u32 falcon_id)
{
	const struct firmware *bl, *code, *data;
	struct lsf_ucode_desc *lsf_desc;
	char f[64];
	int ret;

	img->ucode_header = NULL;

	snprintf(f, sizeof(f), "%s_bl", falcon_name);
	ret = nvkm_firmware_get(device, f, &bl);
	if (ret)
		goto error;

	snprintf(f, sizeof(f), "%s_inst", falcon_name);
	ret = nvkm_firmware_get(device, f, &code);
	if (ret)
		goto free_bl;

	snprintf(f, sizeof(f), "%s_data", falcon_name);
	ret = nvkm_firmware_get(device, f, &data);
	if (ret)
		goto free_inst;

	img->ucode_data = ls_ucode_img_build(bl, code, data,
					     &img->ucode_desc);
	if (IS_ERR(img->ucode_data)) {
		ret = PTR_ERR(img->ucode_data);
		goto free_data;
	}
	img->ucode_size = img->ucode_desc.image_size;

	snprintf(f, sizeof(f), "%s_sig", falcon_name);
	lsf_desc = gm200_secboot_load_firmware(device, f, sizeof(*lsf_desc));
	if (IS_ERR(lsf_desc)) {
		ret = PTR_ERR(lsf_desc);
		goto free_image;
	}
	/* not needed? the signature should already have the right value */
	lsf_desc->falcon_id = falcon_id;
	memcpy(&img->lsb_header.signature, lsf_desc, sizeof(*lsf_desc));
	img->falcon_id = lsf_desc->falcon_id;
	kfree(lsf_desc);

	/* success path - only free requested firmware files */
	goto free_data;

free_image:
	kfree(img->ucode_data);
free_data:
	nvkm_firmware_put(data);
free_inst:
	nvkm_firmware_put(code);
free_bl:
	nvkm_firmware_put(bl);
error:
	return ret;
}

static int
ls_ucode_img_load_fecs(struct nvkm_device *device, struct ls_ucode_img *img)
{
	return ls_ucode_img_load_generic(device, img, "fecs",
					 NVKM_SECBOOT_FALCON_FECS);
}

static int
ls_ucode_img_load_gpccs(struct nvkm_device *device, struct ls_ucode_img *img)
{
	return ls_ucode_img_load_generic(device, img, "gpccs",
					 NVKM_SECBOOT_FALCON_GPCCS);
}

/**
 * ls_ucode_img_populate_bl_desc() - populate a DMEM BL descriptor for LS image
 * @img:	ucode image to generate against
 * @desc:	descriptor to populate
 * @sb:		secure boot state to use for base addresses
 *
 * Populate the DMEM BL descriptor with the information contained in a
 * ls_ucode_desc.
 *
 */
static void
ls_ucode_img_populate_bl_desc(struct ls_ucode_img *img, u64 wpr_addr,
			      struct gm200_flcn_bl_desc *desc)
{
	struct ls_ucode_desc *pdesc = &img->ucode_desc;
	u64 addr_base;

	addr_base = wpr_addr + img->lsb_header.ucode_off +
		    pdesc->app_start_offset;

	memset(desc, 0, sizeof(*desc));
	desc->ctx_dma = FALCON_DMAIDX_UCODE;
	desc->code_dma_base.lo = lower_32_bits(
		(addr_base + pdesc->app_resident_code_offset));
	desc->code_dma_base.hi = upper_32_bits(
		(addr_base + pdesc->app_resident_code_offset));
	desc->non_sec_code_size = pdesc->app_resident_code_size;
	desc->data_dma_base.lo = lower_32_bits(
		(addr_base + pdesc->app_resident_data_offset));
	desc->data_dma_base.hi = upper_32_bits(
		(addr_base + pdesc->app_resident_data_offset));
	desc->data_size = pdesc->app_resident_data_size;
	desc->code_entry_point = pdesc->app_imem_entry;
}

typedef int (*lsf_load_func)(struct nvkm_device *, struct ls_ucode_img *);

/**
 * ls_ucode_img_load() - create a lsf_ucode_img and load it
 */
static struct ls_ucode_img *
ls_ucode_img_load(struct nvkm_device *device, lsf_load_func load_func)
{
	struct ls_ucode_img *img;
	int ret;

	img = kzalloc(sizeof(*img), GFP_KERNEL);
	if (!img)
		return ERR_PTR(-ENOMEM);

	ret = load_func(device, img);
	if (ret) {
		kfree(img);
		return ERR_PTR(ret);
	}

	return img;
}

static const lsf_load_func lsf_load_funcs[] = {
	[NVKM_SECBOOT_FALCON_END] = NULL, /* reserve enough space */
	[NVKM_SECBOOT_FALCON_FECS] = ls_ucode_img_load_fecs,
	[NVKM_SECBOOT_FALCON_GPCCS] = ls_ucode_img_load_gpccs,
};

#define LSF_LSB_HEADER_ALIGN 256
#define LSF_BL_DATA_ALIGN 256
#define LSF_BL_DATA_SIZE_ALIGN 256
#define LSF_BL_CODE_SIZE_ALIGN 256
#define LSF_UCODE_DATA_ALIGN 4096

/**
 * ls_ucode_img_fill_headers - fill the WPR and LSB headers of an image
 * @gsb:	secure boot device used
 * @img:	image to generate for
 * @offset:	offset in the WPR region where this image starts
 *
 * Allocate space in the WPR area from offset and write the WPR and LSB headers
 * accordingly.
 *
 * Return: offset at the end of this image.
 */
static u32
ls_ucode_img_fill_headers(struct gm200_secboot *gsb, struct ls_ucode_img *img,
			  u32 offset)
{
	struct lsf_wpr_header *whdr = &img->wpr_header;
	struct lsf_lsb_header *lhdr = &img->lsb_header;
	struct ls_ucode_desc *desc = &img->ucode_desc;

	if (img->ucode_header) {
		nvkm_fatal(&gsb->base.subdev,
			    "images withough loader are not supported yet!\n");
		return offset;
	}

	/* Fill WPR header */
	whdr->falcon_id = img->falcon_id;
	whdr->bootstrap_owner = gsb->base.func->boot_falcon;
	whdr->status = LSF_IMAGE_STATUS_COPY;

	/* Align, save off, and include an LSB header size */
	offset = ALIGN(offset, LSF_LSB_HEADER_ALIGN);
	whdr->lsb_offset = offset;
	offset += sizeof(struct lsf_lsb_header);

	/*
	 * Align, save off, and include the original (static) ucode
	 * image size
	 */
	offset = ALIGN(offset, LSF_UCODE_DATA_ALIGN);
	lhdr->ucode_off = offset;
	offset += img->ucode_size;

	/*
	 * For falcons that use a boot loader (BL), we append a loader
	 * desc structure on the end of the ucode image and consider
	 * this the boot loader data. The host will then copy the loader
	 * desc args to this space within the WPR region (before locking
	 * down) and the HS bin will then copy them to DMEM 0 for the
	 * loader.
	 */
	lhdr->bl_code_size = ALIGN(desc->bootloader_size,
				   LSF_BL_CODE_SIZE_ALIGN);
	lhdr->ucode_size = ALIGN(desc->app_resident_data_offset,
				 LSF_BL_CODE_SIZE_ALIGN) + lhdr->bl_code_size;
	lhdr->data_size = ALIGN(desc->app_size, LSF_BL_CODE_SIZE_ALIGN) +
				lhdr->bl_code_size - lhdr->ucode_size;
	/*
	 * Though the BL is located at 0th offset of the image, the VA
	 * is different to make sure that it doesn't collide the actual
	 * OS VA range
	 */
	lhdr->bl_imem_off = desc->bootloader_imem_offset;
	lhdr->app_code_off = desc->app_start_offset +
			     desc->app_resident_code_offset;
	lhdr->app_code_size = desc->app_resident_code_size;
	lhdr->app_data_off = desc->app_start_offset +
			     desc->app_resident_data_offset;
	lhdr->app_data_size = desc->app_resident_data_size;

	lhdr->flags = 0;
	if (img->falcon_id == gsb->base.func->boot_falcon)
		lhdr->flags = LSF_FLAG_DMACTL_REQ_CTX;

	/* GPCCS will be loaded using PRI */
	if (img->falcon_id == NVKM_SECBOOT_FALCON_GPCCS)
		lhdr->flags |= LSF_FLAG_FORCE_PRIV_LOAD;

	/* Align (size bloat) and save off BL descriptor size */
	lhdr->bl_data_size = ALIGN(sizeof(struct gm200_flcn_bl_desc),
				   LSF_BL_DATA_SIZE_ALIGN);
	/*
	 * Align, save off, and include the additional BL data
	 */
	offset = ALIGN(offset, LSF_BL_DATA_ALIGN);
	lhdr->bl_data_off = offset;
	offset += lhdr->bl_data_size;

	return offset;
}

static void
ls_ucode_mgr_init(struct ls_ucode_mgr *mgr)
{
	memset(mgr, 0, sizeof(*mgr));
	INIT_LIST_HEAD(&mgr->img_list);
}

static void
ls_ucode_mgr_cleanup(struct ls_ucode_mgr *mgr)
{
	struct ls_ucode_img *img, *t;

	list_for_each_entry_safe(img, t, &mgr->img_list, node) {
		kfree(img->ucode_data);
		kfree(img->ucode_header);
		kfree(img);
	}
}

static void
ls_ucode_mgr_add_img(struct ls_ucode_mgr *mgr, struct ls_ucode_img *img)
{
	mgr->count++;
	list_add_tail(&img->node, &mgr->img_list);
}

/**
 * ls_ucode_mgr_fill_headers - fill WPR and LSB headers of all managed images
 */
static void
ls_ucode_mgr_fill_headers(struct gm200_secboot *gsb, struct ls_ucode_mgr *mgr)
{
	struct ls_ucode_img *img;
	u32 offset;

	/*
	 * Start with an array of WPR headers at the base of the WPR.
	 * The expectation here is that the secure falcon will do a single DMA
	 * read of this array and cache it internally so it's ok to pack these.
	 * Also, we add 1 to the falcon count to indicate the end of the array.
	 */
	offset = sizeof(struct lsf_wpr_header) * (mgr->count + 1);

	/*
	 * Walk the managed falcons, accounting for the LSB structs
	 * as well as the ucode images.
	 */
	list_for_each_entry(img, &mgr->img_list, node) {
		offset = ls_ucode_img_fill_headers(gsb, img, offset);
	}

	mgr->wpr_size = offset;
}

/**
 * ls_ucode_mgr_write_wpr - write the WPR blob contents
 */
static int
ls_ucode_mgr_write_wpr(struct gm200_secboot *gsb, struct ls_ucode_mgr *mgr,
		       struct nvkm_gpuobj *wpr_blob)
{
	struct ls_ucode_img *img;
	u32 pos = 0;

	nvkm_kmap(wpr_blob);

	list_for_each_entry(img, &mgr->img_list, node) {
		nvkm_gpuobj_memcpy_to(wpr_blob, pos, &img->wpr_header,
				      sizeof(img->wpr_header));

		nvkm_gpuobj_memcpy_to(wpr_blob, img->wpr_header.lsb_offset,
				     &img->lsb_header, sizeof(img->lsb_header));

		/* Generate and write BL descriptor */
		if (!img->ucode_header) {
			u8 desc[gsb->bl_desc_size];
			struct gm200_flcn_bl_desc gdesc;

			ls_ucode_img_populate_bl_desc(img, gsb->wpr_addr,
						      &gdesc);
			gsb->fixup_bl_desc(&gdesc, &desc);
			nvkm_gpuobj_memcpy_to(wpr_blob,
					      img->lsb_header.bl_data_off,
					      &desc, gsb->bl_desc_size);
		}

		/* Copy ucode */
		nvkm_gpuobj_memcpy_to(wpr_blob, img->lsb_header.ucode_off,
				      img->ucode_data, img->ucode_size);

		pos += sizeof(img->wpr_header);
	}

	nvkm_wo32(wpr_blob, pos, NVKM_SECBOOT_FALCON_INVALID);

	nvkm_done(wpr_blob);

	return 0;
}

/* Both size and address of WPR need to be 128K-aligned */
#define WPR_ALIGNMENT	0x20000
/**
 * gm200_secboot_prepare_ls_blob() - prepare the LS blob
 *
 * For each securely managed falcon, load the FW, signatures and bootloaders and
 * prepare a ucode blob. Then, compute the offsets in the WPR region for each
 * blob, and finally write the headers and ucode blobs into a GPU object that
 * will be copied into the WPR region by the HS firmware.
 */
static int
gm200_secboot_prepare_ls_blob(struct gm200_secboot *gsb)
{
	struct nvkm_secboot *sb = &gsb->base;
	struct nvkm_device *device = sb->subdev.device;
	struct ls_ucode_mgr mgr;
	int falcon_id;
	int ret;

	ls_ucode_mgr_init(&mgr);

	/* Load all LS blobs */
	for_each_set_bit(falcon_id, &gsb->base.func->managed_falcons,
			 NVKM_SECBOOT_FALCON_END) {
		struct ls_ucode_img *img;

		img = ls_ucode_img_load(device, lsf_load_funcs[falcon_id]);

		if (IS_ERR(img)) {
			ret = PTR_ERR(img);
			goto cleanup;
		}
		ls_ucode_mgr_add_img(&mgr, img);
	}

	/*
	 * Fill the WPR and LSF headers with the right offsets and compute
	 * required WPR size
	 */
	ls_ucode_mgr_fill_headers(gsb, &mgr);
	mgr.wpr_size = ALIGN(mgr.wpr_size, WPR_ALIGNMENT);

	/* Allocate GPU object that will contain the WPR region */
	ret = nvkm_gpuobj_new(device, mgr.wpr_size, WPR_ALIGNMENT, false, NULL,
			      &gsb->ls_blob);
	if (ret)
		goto cleanup;

	nvkm_debug(&sb->subdev, "%d managed LS falcons, WPR size is %d bytes\n",
		    mgr.count, mgr.wpr_size);

	/* If WPR address and size are not fixed, set them to fit the LS blob */
	if (!gsb->wpr_size) {
		gsb->wpr_addr = gsb->ls_blob->addr;
		gsb->wpr_size = gsb->ls_blob->size;
	}

	/* Write LS blob */
	ret = ls_ucode_mgr_write_wpr(gsb, &mgr, gsb->ls_blob);

cleanup:
	ls_ucode_mgr_cleanup(&mgr);

	return ret;
}

/*
 * High-secure blob creation
 */

/**
 * gm200_secboot_hsf_patch_signature() - patch HS blob with correct signature
 */
static void
gm200_secboot_hsf_patch_signature(struct gm200_secboot *gsb, void *acr_image)
{
	struct nvkm_secboot *sb = &gsb->base;
	struct hsf_bin_hdr *hsbin_hdr = acr_image;
	struct hsf_fw_header *fw_hdr = acr_image + hsbin_hdr->header_offset;
	void *hs_data = acr_image + hsbin_hdr->data_offset;
	u32 patch_loc;
	u32 patch_sig;
	void *sig;
	u32 sig_size;

	patch_loc = *(u32 *)(acr_image + fw_hdr->patch_loc);
	patch_sig = *(u32 *)(acr_image + fw_hdr->patch_sig);

	/* Falcon in debug or production mode? */
	if ((nvkm_rd32(sb->subdev.device, sb->base + 0xc08) >> 20) & 0x1) {
		sig = acr_image + fw_hdr->sig_dbg_offset;
		sig_size = fw_hdr->sig_dbg_size;
	} else {
		sig = acr_image + fw_hdr->sig_prod_offset;
		sig_size = fw_hdr->sig_prod_size;
	}

	/* Patch signature */
	memcpy(hs_data + patch_loc, sig + patch_sig, sig_size);
}

static void
gm200_secboot_fixup_hs_desc(struct gm200_secboot *gsb,
			    struct hsflcn_acr_desc *desc)
{
	desc->ucode_blob_base = gsb->ls_blob->addr;
	desc->ucode_blob_size = gsb->ls_blob->size;

	desc->wpr_offset = 0;

	/* WPR region information for the HS binary to set up */
	desc->wpr_region_id = 1;
	desc->regions.no_regions = 1;
	desc->regions.region_props[0].region_id = 1;
	desc->regions.region_props[0].start_addr = gsb->wpr_addr >> 8;
	desc->regions.region_props[0].end_addr =
		(gsb->wpr_addr + gsb->wpr_size) >> 8;
}

/**
 * gm200_secboot_populate_hsf_bl_desc() - populate BL descriptor for HS image
 */
static void
gm200_secboot_populate_hsf_bl_desc(void *acr_image,
				   struct gm200_flcn_bl_desc *bl_desc)
{
	struct hsf_bin_hdr *hsbin_hdr = acr_image;
	struct hsf_fw_header *fw_hdr = acr_image + hsbin_hdr->header_offset;
	struct hsf_load_header *load_hdr = acr_image + fw_hdr->hdr_offset;

	/*
	 * Descriptor for the bootloader that will load the ACR image into
	 * IMEM/DMEM memory.
	 */
	fw_hdr = acr_image + hsbin_hdr->header_offset;
	load_hdr = acr_image + fw_hdr->hdr_offset;
	memset(bl_desc, 0, sizeof(*bl_desc));
	bl_desc->ctx_dma = FALCON_DMAIDX_VIRT;
	bl_desc->non_sec_code_off = load_hdr->non_sec_code_off;
	bl_desc->non_sec_code_size = load_hdr->non_sec_code_size;
	bl_desc->sec_code_off = load_hdr->sec_code_off;
	bl_desc->sec_code_size = load_hdr->sec_code_size;
	bl_desc->code_entry_point = 0;
	/*
	 * We need to set code_dma_base to the virtual address of the acr_blob,
	 * and add this address to data_dma_base before writing it into DMEM
	 */
	bl_desc->code_dma_base.lo = 0;
	bl_desc->data_dma_base.lo = load_hdr->data_dma_base;
	bl_desc->data_size = load_hdr->data_size;
}

static int
gm200_secboot_prepare_hs_blob(struct gm200_secboot *gsb)
{
	struct nvkm_device *device = gsb->base.subdev.device;
	void *acr_image;
	struct hsf_bin_hdr *hsbin_hdr;
	struct hsf_fw_header *fw_hdr;
	struct hsf_load_header *load_hdr;
	struct hsflcn_acr_desc *desc;
	u32 img_size;
	int ret;

	acr_image = gm200_secboot_load_firmware(device, "acr_ucode_load", 0);
	if (IS_ERR(acr_image))
		return PTR_ERR(acr_image);
	hsbin_hdr = acr_image;

	/* Patch signature */
	gm200_secboot_hsf_patch_signature(gsb, acr_image);

	/* Patch descriptor */
	fw_hdr = acr_image + hsbin_hdr->header_offset;
	load_hdr = acr_image + fw_hdr->hdr_offset;
	desc = acr_image + hsbin_hdr->data_offset + load_hdr->data_dma_base;
	gsb->fixup_hs_desc(gsb, desc);

	/* Generate HS BL descriptor */
	gm200_secboot_populate_hsf_bl_desc(acr_image, &gsb->acr_bl_desc);

	/* Create ACR blob and copy HS data to it */
	img_size = ALIGN(hsbin_hdr->data_size, 256);
	ret = nvkm_gpuobj_new(device, img_size, 0x1000, false, NULL,
			      &gsb->acr_blob);
	if (ret)
		goto cleanup;

	nvkm_kmap(gsb->acr_blob);
	nvkm_gpuobj_memcpy_to(gsb->acr_blob, 0,
			      acr_image + hsbin_hdr->data_offset, img_size);
	nvkm_done(gsb->acr_blob);

cleanup:
	kfree(acr_image);

	return ret;
}

/*
 * High-secure bootloader blob creation
 */

static int
gm200_secboot_prepare_hsbl_blob(struct gm200_secboot *gsb)
{
	struct nvkm_device *device = gsb->base.subdev.device;

	gsb->hsbl_blob = gm200_secboot_load_firmware(device, "acr_bl", 0);
	if (IS_ERR(gsb->hsbl_blob)) {
		int ret = PTR_ERR(gsb->hsbl_blob);

		gsb->hsbl_blob = NULL;
		return ret;
	}

	return 0;
}

int
gm200_secboot_prepare_blobs(struct nvkm_secboot *sb)
{
	struct gm200_secboot *gsb = gm200_secboot(sb);
	int ret;

	/* Load and prepare the managed falcon's firmwares */
	ret = gm200_secboot_prepare_ls_blob(gsb);
	if (ret)
		return ret;

	/* Load the HS firmware for the performing falcon */
	ret = gm200_secboot_prepare_hs_blob(gsb);
	if (ret)
		return ret;

	/* Load the HS firmware bootloader */
	ret = gm200_secboot_prepare_hsbl_blob(gsb);
	if (ret)
		return ret;

	return 0;
}



/*
 * Secure Boot Execution
 */

/**
 * gm200_secboot_load_hs_bl() - load HS bootloader into DMEM and IMEM
 */
static void
gm200_secboot_load_hs_bl(struct nvkm_secboot *sb, void *data, u32 data_size)
{
	struct nvkm_device *device = sb->subdev.device;
	struct gm200_secboot *gsb = gm200_secboot(sb);
	struct hsf_bin_hdr *hdr = gsb->hsbl_blob;
	struct hsf_bl_desc *hsbl_desc = gsb->hsbl_blob + hdr->header_offset;
	void *hsbl_code = gsb->hsbl_blob + hdr->data_offset;
	u32 code_size = ALIGN(hsbl_desc->bl_code_size, 256);
	u32 blk;
	u32 tag;
	int i;

	/*
	 * Copy HS bootloader interface structure where the HS descriptor
	 * expects it to be
	 */
	nvkm_wr32(device, sb->base + 0x1c0,
		  (hsbl_desc->bl_desc_dmem_load_off | (0x1 << 24)));
	for (i = 0; i < data_size / 4; i++)
		nvkm_wr32(device, sb->base + 0x1c4, ((u32 *)data)[i]);

	/* Copy HS bootloader code to end of IMEM */
	blk = (nvkm_rd32(device, sb->base + 0x108) & 0x1ff) - (code_size >> 8);
	tag = hsbl_desc->bl_start_tag;
	nvkm_wr32(device, sb->base + 0x180, ((blk & 0xff) << 8) | (0x1 << 24));
	for (i = 0; i < code_size / 4; i++) {
		/* write new tag every 256B */
		if ((i & 0x3f) == 0) {
			nvkm_wr32(device, sb->base + 0x188, tag & 0xffff);
			tag++;
		}
		nvkm_wr32(device, sb->base + 0x184, ((u32 *)hsbl_code)[i]);
	}
	nvkm_wr32(device, sb->base + 0x188, 0);
}

/**
 * gm200_secboot_setup_falcon() - set up the secure falcon for secure boot
 */
static int
gm200_secboot_setup_falcon(struct nvkm_secboot *sb)
{
	struct nvkm_device *device = sb->subdev.device;
	struct gm200_secboot *gsb = gm200_secboot(sb);
	struct hsf_bin_hdr *hdr = gsb->hsbl_blob;
	struct hsf_bl_desc *hsbl_desc = gsb->hsbl_blob + hdr->header_offset;
	/* virtual start address for boot vector */
	u32 virt_addr = hsbl_desc->bl_start_tag << 8;
	const u32 reg_base = sb->base + 0xe00;
	u32 inst_loc;
	int ret;

	ret = nvkm_secboot_falcon_reset(sb);
	if (ret)
		return ret;

	/* setup apertures - virtual */
	nvkm_wr32(device, reg_base + 4 * (FALCON_DMAIDX_UCODE), 0x4);
	nvkm_wr32(device, reg_base + 4 * (FALCON_DMAIDX_VIRT), 0x0);
	/* setup apertures - physical */
	nvkm_wr32(device, reg_base + 4 * (FALCON_DMAIDX_PHYS_VID), 0x4);
	nvkm_wr32(device, reg_base + 4 * (FALCON_DMAIDX_PHYS_SYS_COH),
		  0x4 | 0x1);
	nvkm_wr32(device, reg_base + 4 * (FALCON_DMAIDX_PHYS_SYS_NCOH),
		  0x4 | 0x2);

	/* Set context */
	if (device->fb->ram)
		inst_loc = 0x0; /* FB */
	else
		inst_loc = 0x3; /* Non-coherent sysmem */

	nvkm_mask(device, sb->base + 0x048, 0x1, 0x1);
	nvkm_wr32(device, sb->base + 0x480,
		  ((gsb->inst->addr >> 12) & 0xfffffff) |
		  (inst_loc << 28) | (1 << 30));

	/* Set boot vector to code's starting virtual address */
	nvkm_wr32(device, sb->base + 0x104, virt_addr);

	return 0;
}

/*
 * gm200_secboot_run() - execute secure boot from the prepared state
 *
 * Load the HS bootloader and ask the falcon to run it. This will in turn
 * load the HS firmware and run it, so once the falcon stops all the managed
 * falcons should have their LS firmware loaded and be ready to run.
 */
int
gm200_secboot_run(struct nvkm_secboot *sb)
{
	struct gm200_secboot *gsb = gm200_secboot(sb);
	struct gm200_flcn_bl_desc *desc = &gsb->acr_bl_desc;
	struct nvkm_vma *vma = &gsb->acr_blob_vma;
	u64 vma_addr;
	u8 bl_desc[gsb->bl_desc_size];
	int ret;

	/* Map the HS firmware so the HS bootloader can see it */
	ret = nvkm_gpuobj_map(gsb->acr_blob, gsb->vm, NV_MEM_ACCESS_RW,
			      &gsb->acr_blob_vma);
	if (ret)
		return ret;

	/* Add the mapping address to the DMA bases */
	vma_addr = flcn64_to_u64(desc->code_dma_base) + vma->offset;
	desc->code_dma_base.lo = lower_32_bits(vma_addr);
	desc->code_dma_base.hi = upper_32_bits(vma_addr);
	vma_addr = flcn64_to_u64(desc->data_dma_base) + vma->offset;
	desc->data_dma_base.lo = lower_32_bits(vma_addr);
	desc->data_dma_base.hi = upper_32_bits(vma_addr);

	/* Fixup the BL header */
	gsb->fixup_bl_desc(&gsb->acr_bl_desc, &bl_desc);

	/* Reset the falcon and make it ready to run the HS bootloader */
	ret = gm200_secboot_setup_falcon(sb);
	if (ret)
		goto done;

	/* Load the HS bootloader into the falcon's IMEM/DMEM */
	gm200_secboot_load_hs_bl(sb, &bl_desc, gsb->bl_desc_size);

	/* Start the HS bootloader */
	ret = nvkm_secboot_falcon_run(sb);
	if (ret)
		goto done;

done:
	/* Restore the original DMA addresses */
	vma_addr = flcn64_to_u64(desc->code_dma_base) - vma->offset;
	desc->code_dma_base.lo = lower_32_bits(vma_addr);
	desc->code_dma_base.hi = upper_32_bits(vma_addr);
	vma_addr = flcn64_to_u64(desc->data_dma_base) - vma->offset;
	desc->data_dma_base.lo = lower_32_bits(vma_addr);
	desc->data_dma_base.hi = upper_32_bits(vma_addr);

	/* We don't need the ACR firmware anymore */
	nvkm_gpuobj_unmap(&gsb->acr_blob_vma);

	return ret;
}



int
gm200_secboot_init(struct nvkm_secboot *sb)
{
	struct gm200_secboot *gsb = gm200_secboot(sb);
	struct nvkm_device *device = sb->subdev.device;
	struct nvkm_vm *vm;
	const u64 vm_area_len = 600 * 1024;
	int ret;

	/* Allocate instance block and VM */
	ret = nvkm_gpuobj_new(device, 0x1000, 0, true, NULL, &gsb->inst);
	if (ret)
		return ret;

	ret = nvkm_gpuobj_new(device, 0x8000, 0, true, NULL, &gsb->pgd);
	if (ret)
		return ret;

	ret = nvkm_vm_new(device, 0, vm_area_len, 0, NULL, &vm);
	if (ret)
		return ret;

	atomic_inc(&vm->engref[NVKM_SUBDEV_PMU]);

	ret = nvkm_vm_ref(vm, &gsb->vm, gsb->pgd);
	nvkm_vm_ref(NULL, &vm, NULL);
	if (ret)
		return ret;

	nvkm_kmap(gsb->inst);
	nvkm_wo32(gsb->inst, 0x200, lower_32_bits(gsb->pgd->addr));
	nvkm_wo32(gsb->inst, 0x204, upper_32_bits(gsb->pgd->addr));
	nvkm_wo32(gsb->inst, 0x208, lower_32_bits(vm_area_len - 1));
	nvkm_wo32(gsb->inst, 0x20c, upper_32_bits(vm_area_len - 1));
	nvkm_done(gsb->inst);

	return 0;
}

void *
gm200_secboot_dtor(struct nvkm_secboot *sb)
{
	struct gm200_secboot *gsb = gm200_secboot(sb);

	kfree(gsb->hsbl_blob);
	nvkm_gpuobj_del(&gsb->acr_blob);
	nvkm_gpuobj_del(&gsb->ls_blob);

	nvkm_vm_ref(NULL, &gsb->vm, gsb->pgd);
	nvkm_gpuobj_del(&gsb->pgd);
	nvkm_gpuobj_del(&gsb->inst);

	return gsb;
}


static const struct nvkm_secboot_func
gm200_secboot = {
	.dtor = gm200_secboot_dtor,
	.init = gm200_secboot_init,
	.prepare_blobs = gm200_secboot_prepare_blobs,
	.run = gm200_secboot_run,
	.managed_falcons = BIT(NVKM_SECBOOT_FALCON_FECS) |
			   BIT(NVKM_SECBOOT_FALCON_GPCCS),
	.boot_falcon = NVKM_SECBOOT_FALCON_PMU,
};

/**
 * gm200_fixup_bl_desc - just copy the BL descriptor
 *
 * Use the GM200 descriptor format by default.
 */
static void
gm200_fixup_bl_desc(const struct gm200_flcn_bl_desc *desc, void *ret)
{
	memcpy(ret, desc, sizeof(*desc));
}

int
gm200_secboot_new(struct nvkm_device *device, int index,
		  struct nvkm_secboot **psb)
{
	int ret;
	struct gm200_secboot *gsb;

	gsb = kzalloc(sizeof(*gsb), GFP_KERNEL);
	if (!gsb) {
		psb = NULL;
		return -ENOMEM;
	}
	*psb = &gsb->base;

	ret = nvkm_secboot_ctor(&gm200_secboot, device, index, &gsb->base);
	if (ret)
		return ret;

	gsb->bl_desc_size = sizeof(struct gm200_flcn_bl_desc);
	gsb->fixup_bl_desc = gm200_fixup_bl_desc;
	gsb->fixup_hs_desc = gm200_secboot_fixup_hs_desc;

	return 0;
}