diff options
Diffstat (limited to 'liboffloadmic/runtime/offload_host.cpp')
| -rw-r--r-- | liboffloadmic/runtime/offload_host.cpp | 4402 |
1 files changed, 4402 insertions, 0 deletions
diff --git a/liboffloadmic/runtime/offload_host.cpp b/liboffloadmic/runtime/offload_host.cpp new file mode 100644 index 00000000000..23a873f3886 --- /dev/null +++ b/liboffloadmic/runtime/offload_host.cpp @@ -0,0 +1,4402 @@ +/* + Copyright (c) 2014 Intel Corporation. All Rights Reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +// Forward declaration as the following 2 functions are declared as friend in offload_engine.h +// CLANG does not like static to been after friend declaration. +static void __offload_init_library_once(void); +static void __offload_fini_library(void); + +#include "offload_host.h" +#ifdef MYO_SUPPORT +#include "offload_myo_host.h" +#endif + +#include <malloc.h> +#ifndef TARGET_WINNT +#include <alloca.h> +#include <elf.h> +#endif // TARGET_WINNT +#include <errno.h> +#include <fcntl.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include <algorithm> +#include <bitset> + +#if defined(HOST_WINNT) +#define PATH_SEPARATOR ";" +#else +#define PATH_SEPARATOR ":" +#endif + +#define GET_OFFLOAD_NUMBER(timer_data) \ + timer_data? timer_data->offload_number : 0 + +#ifdef TARGET_WINNT +// Small subset of ELF declarations for Windows which is needed to compile +// this file. ELF header is used to understand what binary type is contained +// in the target image - shared library or executable. + +typedef uint16_t Elf64_Half; +typedef uint32_t Elf64_Word; +typedef uint64_t Elf64_Addr; +typedef uint64_t Elf64_Off; + +#define EI_NIDENT 16 + +#define ET_EXEC 2 +#define ET_DYN 3 + +typedef struct +{ + unsigned char e_ident[EI_NIDENT]; + Elf64_Half e_type; + Elf64_Half e_machine; + Elf64_Word e_version; + Elf64_Addr e_entry; + Elf64_Off e_phoff; + Elf64_Off e_shoff; + Elf64_Word e_flags; + Elf64_Half e_ehsize; + Elf64_Half e_phentsize; + Elf64_Half e_phnum; + Elf64_Half e_shentsize; + Elf64_Half e_shnum; + Elf64_Half e_shstrndx; +} Elf64_Ehdr; +#endif // TARGET_WINNT + +// Host console and file logging +const char *prefix; +int console_enabled = 0; +int offload_number = 0; + +static const char *htrace_envname = "H_TRACE"; +static const char *offload_report_envname = "OFFLOAD_REPORT"; +static char *timer_envname = "H_TIME"; + +// Trace information +static const char* vardesc_direction_as_string[] = { + "NOCOPY", + "IN", + "OUT", + "INOUT" +}; +static const char* vardesc_type_as_string[] = { + "unknown", + "data", + "data_ptr", + "func_ptr", + "void_ptr", + "string_ptr", + "dv", + "dv_data", + "dv_data_slice", + "dv_ptr", + "dv_ptr_data", + "dv_ptr_data_slice", + "cean_var", + "cean_var_ptr", + "c_data_ptr_array", + "c_func_ptr_array", + "c_void_ptr_array", + "c_string_ptr_array" +}; + +Engine* mic_engines = 0; +uint32_t mic_engines_total = 0; +pthread_key_t mic_thread_key; +MicEnvVar mic_env_vars; +uint64_t cpu_frequency = 0; + +// MIC_STACKSIZE +uint32_t mic_stack_size = 12 * 1024 * 1024; + +// MIC_BUFFERSIZE +uint64_t mic_buffer_size = 0; + +// MIC_LD_LIBRARY_PATH +char* mic_library_path = 0; + +// MIC_PROXY_IO +bool mic_proxy_io = true; + +// MIC_PROXY_FS_ROOT +char* mic_proxy_fs_root = 0; + +// Threshold for creating buffers with large pages. Buffer is created +// with large pages hint if its size exceeds the threshold value. +// By default large pages are disabled right now (by setting default +// value for threshold to MAX) due to HSD 4114629. +uint64_t __offload_use_2mb_buffers = 0xffffffffffffffffULL; +static const char *mic_use_2mb_buffers_envname = + "MIC_USE_2MB_BUFFERS"; + +static uint64_t __offload_use_async_buffer_write = 2 * 1024 * 1024; +static const char *mic_use_async_buffer_write_envname = + "MIC_USE_ASYNC_BUFFER_WRITE"; + +static uint64_t __offload_use_async_buffer_read = 2 * 1024 * 1024; +static const char *mic_use_async_buffer_read_envname = + "MIC_USE_ASYNC_BUFFER_READ"; + +// device initialization type +OffloadInitType __offload_init_type = c_init_on_offload_all; +static const char *offload_init_envname = "OFFLOAD_INIT"; + +// active wait +static bool __offload_active_wait = true; +static const char *offload_active_wait_envname = "OFFLOAD_ACTIVE_WAIT"; + +// OMP_DEFAULT_DEVICE +int __omp_device_num = 0; +static const char *omp_device_num_envname = "OMP_DEFAULT_DEVICE"; + +// The list of pending target libraries +static bool __target_libs; +static TargetImageList __target_libs_list; +static mutex_t __target_libs_lock; +static mutex_t stack_alloc_lock; + +// Target executable +TargetImage* __target_exe; + +static char * offload_get_src_base(void * ptr, uint8_t type) +{ + char *base; + if (VAR_TYPE_IS_PTR(type)) { + base = *static_cast<char**>(ptr); + } + else if (VAR_TYPE_IS_SCALAR(type)) { + base = static_cast<char*>(ptr); + } + else if (VAR_TYPE_IS_DV_DATA_SLICE(type) || VAR_TYPE_IS_DV_DATA(type)) { + ArrDesc *dvp; + if (VAR_TYPE_IS_DV_DATA_SLICE(type)) { + const arr_desc *ap = static_cast<const arr_desc*>(ptr); + dvp = (type == c_dv_data_slice) ? + reinterpret_cast<ArrDesc*>(ap->base) : + *reinterpret_cast<ArrDesc**>(ap->base); + } + else { + dvp = (type == c_dv_data) ? + static_cast<ArrDesc*>(ptr) : + *static_cast<ArrDesc**>(ptr); + } + base = reinterpret_cast<char*>(dvp->Base); + } + else { + base = NULL; + } + return base; +} + +void OffloadDescriptor::report_coi_error(error_types msg, COIRESULT res) +{ + // special case for the 'process died' error + if (res == COI_PROCESS_DIED) { + m_device.fini_process(true); + } + else { + switch (msg) { + case c_buf_create: + if (res == COI_OUT_OF_MEMORY) { + msg = c_buf_create_out_of_mem; + } + /* fallthru */ + + case c_buf_create_from_mem: + case c_buf_get_address: + case c_pipeline_create: + case c_pipeline_run_func: + LIBOFFLOAD_ERROR(msg, m_device.get_logical_index(), res); + break; + + case c_buf_read: + case c_buf_write: + case c_buf_copy: + case c_buf_map: + case c_buf_unmap: + case c_buf_destroy: + case c_buf_set_state: + LIBOFFLOAD_ERROR(msg, res); + break; + + default: + break; + } + } + + exit(1); +} + +_Offload_result OffloadDescriptor::translate_coi_error(COIRESULT res) const +{ + switch (res) { + case COI_SUCCESS: + return OFFLOAD_SUCCESS; + + case COI_PROCESS_DIED: + return OFFLOAD_PROCESS_DIED; + + case COI_OUT_OF_MEMORY: + return OFFLOAD_OUT_OF_MEMORY; + + default: + return OFFLOAD_ERROR; + } +} + +bool OffloadDescriptor::alloc_ptr_data( + PtrData* &ptr_data, + void *base, + int64_t disp, + int64_t size, + int64_t alloc_disp, + int align +) +{ + // total length of base + int64_t length = disp + size; + bool is_new; + + OFFLOAD_TRACE(3, "Creating association for data: addr %p, length %lld\n", + base, length); + + // add new entry + ptr_data = m_device.insert_ptr_data(base, length, is_new); + if (is_new) { + + OFFLOAD_TRACE(3, "Added new association\n"); + + if (length > 0) { + OffloadTimer timer(get_timer_data(), c_offload_host_alloc_buffers); + COIRESULT res; + + // align should be a power of 2 + if (align > 0 && (align & (align - 1)) == 0) { + // offset within mic_buffer. Can do offset optimization + // only when source address alignment satisfies requested + // alignment on the target (cq172736). + if ((reinterpret_cast<intptr_t>(base) & (align - 1)) == 0) { + ptr_data->mic_offset = reinterpret_cast<intptr_t>(base) & 4095; + } + } + + // buffer size and flags + uint64_t buffer_size = length + ptr_data->mic_offset; + uint32_t buffer_flags = 0; + + // create buffer with large pages if data length exceeds + // large page threshold + if (length >= __offload_use_2mb_buffers) { + buffer_flags = COI_OPTIMIZE_HUGE_PAGE_SIZE; + } + + // create CPU buffer + OFFLOAD_DEBUG_TRACE_1(3, + GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_create_buf_host, + "Creating buffer from source memory %p, " + "length %lld\n", base, length); + + // result is not checked because we can continue without cpu + // buffer. In this case we will use COIBufferRead/Write instead + // of COIBufferCopy. + COI::BufferCreateFromMemory(length, + COI_BUFFER_NORMAL, + 0, + base, + 1, + &m_device.get_process(), + &ptr_data->cpu_buf); + + OFFLOAD_DEBUG_TRACE_1(3, + GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_create_buf_mic, + "Creating buffer for sink: size %lld, offset %d, " + "flags =0x%x\n", buffer_size - alloc_disp, + ptr_data->mic_offset, buffer_flags); + + // create MIC buffer + res = COI::BufferCreate(buffer_size - alloc_disp, + COI_BUFFER_NORMAL, + buffer_flags, + 0, + 1, + &m_device.get_process(), + &ptr_data->mic_buf); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_create, res); + } + ptr_data->alloc_ptr_data_lock.unlock(); + return false; + } + + // make buffer valid on the device. + res = COI::BufferSetState(ptr_data->mic_buf, + m_device.get_process(), + COI_BUFFER_VALID, + COI_BUFFER_NO_MOVE, + 0, 0, 0); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_set_state, res); + } + ptr_data->alloc_ptr_data_lock.unlock(); + return false; + } + + res = COI::BufferSetState(ptr_data->mic_buf, + COI_PROCESS_SOURCE, + COI_BUFFER_INVALID, + COI_BUFFER_NO_MOVE, + 0, 0, 0); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_set_state, res); + } + ptr_data->alloc_ptr_data_lock.unlock(); + return false; + } + } + + ptr_data->alloc_disp = alloc_disp; + ptr_data->alloc_ptr_data_lock.unlock(); + } + else { + mutex_locker_t locker(ptr_data->alloc_ptr_data_lock); + + OFFLOAD_TRACE(3, "Found existing association: addr %p, length %lld, " + "is_static %d\n", + ptr_data->cpu_addr.start(), ptr_data->cpu_addr.length(), + ptr_data->is_static); + + // This is not a new entry. Make sure that provided address range fits + // into existing one. + MemRange addr_range(base, length - ptr_data->alloc_disp); + if (!ptr_data->cpu_addr.contains(addr_range)) { + LIBOFFLOAD_ERROR(c_bad_ptr_mem_range); + exit(1); + } + + // if the entry is associated with static data it may not have buffers + // created because they are created on demand. + if (ptr_data->is_static && !init_static_ptr_data(ptr_data)) { + return false; + } + } + + return true; +} + +bool OffloadDescriptor::find_ptr_data( + PtrData* &ptr_data, + void *base, + int64_t disp, + int64_t size, + bool report_error +) +{ + // total length of base + int64_t length = disp + size; + + OFFLOAD_TRACE(3, "Looking for association for data: addr %p, " + "length %lld\n", base, length); + + // find existing association in pointer table + ptr_data = m_device.find_ptr_data(base); + if (ptr_data == 0) { + if (report_error) { + LIBOFFLOAD_ERROR(c_no_ptr_data, base); + exit(1); + } + OFFLOAD_TRACE(3, "Association does not exist\n"); + return true; + } + + OFFLOAD_TRACE(3, "Found association: base %p, length %lld, is_static %d\n", + ptr_data->cpu_addr.start(), ptr_data->cpu_addr.length(), + ptr_data->is_static); + + // make sure that provided address range fits into existing one + MemRange addr_range(base, length); + if (!ptr_data->cpu_addr.contains(addr_range)) { + if (report_error) { + LIBOFFLOAD_ERROR(c_bad_ptr_mem_range); + exit(1); + } + OFFLOAD_TRACE(3, "Existing association partially overlaps with " + "data address range\n"); + ptr_data = 0; + return true; + } + + // if the entry is associated with static data it may not have buffers + // created because they are created on demand. + if (ptr_data->is_static && !init_static_ptr_data(ptr_data)) { + return false; + } + + return true; +} + +bool OffloadDescriptor::init_static_ptr_data(PtrData *ptr_data) +{ + OffloadTimer timer(get_timer_data(), c_offload_host_alloc_buffers); + + if (ptr_data->cpu_buf == 0) { + OFFLOAD_TRACE(3, "Creating buffer from source memory %llx\n", + ptr_data->cpu_addr.start()); + + COIRESULT res = COI::BufferCreateFromMemory( + ptr_data->cpu_addr.length(), + COI_BUFFER_NORMAL, + 0, + const_cast<void*>(ptr_data->cpu_addr.start()), + 1, &m_device.get_process(), + &ptr_data->cpu_buf); + + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_create_from_mem, res); + } + } + + if (ptr_data->mic_buf == 0) { + OFFLOAD_TRACE(3, "Creating buffer from sink memory %llx\n", + ptr_data->mic_addr); + + COIRESULT res = COI::BufferCreateFromMemory( + ptr_data->cpu_addr.length(), + COI_BUFFER_NORMAL, + COI_SINK_MEMORY, + reinterpret_cast<void*>(ptr_data->mic_addr), + 1, &m_device.get_process(), + &ptr_data->mic_buf); + + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_create_from_mem, res); + } + } + + return true; +} + +bool OffloadDescriptor::init_mic_address(PtrData *ptr_data) +{ + if (ptr_data->mic_buf != 0 && ptr_data->mic_addr == 0) { + COIRESULT res = COI::BufferGetSinkAddress(ptr_data->mic_buf, + &ptr_data->mic_addr); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_get_address, res); + } + return false; + } + } + return true; +} + +bool OffloadDescriptor::nullify_target_stack( + COIBUFFER targ_buf, + uint64_t size +) +{ + char * ptr = (char*)malloc(size); + if (ptr == NULL) + LIBOFFLOAD_ERROR(c_malloc); + COIRESULT res; + + memset(ptr, 0, size); + res = COI::BufferWrite( + targ_buf, + 0, + ptr, + size, + COI_COPY_UNSPECIFIED, + 0, 0, 0); + free(ptr); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_write, res); + } + return true; +} + +bool OffloadDescriptor::offload_stack_memory_manager( + const void * stack_begin, + int routine_id, + int buf_size, + int align, + bool *is_new) +{ + mutex_locker_t locker(stack_alloc_lock); + + PersistData * new_el; + PersistDataList::iterator it_begin = m_device.m_persist_list.begin(); + PersistDataList::iterator it_end; + int erase = 0; + + *is_new = false; + + for (PersistDataList::iterator it = m_device.m_persist_list.begin(); + it != m_device.m_persist_list.end(); it++) { + PersistData cur_el = *it; + + if (stack_begin > it->stack_cpu_addr) { + // this stack data must be destroyed + m_destroy_stack.push_front(cur_el.stack_ptr_data); + it_end = it; + erase++; + } + else if (stack_begin == it->stack_cpu_addr) { + if (routine_id != it-> routine_id) { + // this stack data must be destroyed + m_destroy_stack.push_front(cur_el.stack_ptr_data); + it_end = it; + erase++; + break; + } + else { + // stack data is reused + m_stack_ptr_data = it->stack_ptr_data; + if (erase > 0) { + // all obsolete stack sections must be erased from the list + m_device.m_persist_list.erase(it_begin, ++it_end); + + m_in_datalen += + erase * sizeof(new_el->stack_ptr_data->mic_addr); + } + OFFLOAD_TRACE(3, "Reuse of stack buffer with addr %p\n", + m_stack_ptr_data->mic_addr); + return true; + } + } + else if (stack_begin < it->stack_cpu_addr) { + break; + } + } + + if (erase > 0) { + // all obsolete stack sections must be erased from the list + m_device.m_persist_list.erase(it_begin, ++it_end); + m_in_datalen += erase * sizeof(new_el->stack_ptr_data->mic_addr); + } + // new stack table is created + new_el = new PersistData(stack_begin, routine_id, buf_size); + // create MIC buffer + COIRESULT res; + uint32_t buffer_flags = 0; + + // create buffer with large pages if data length exceeds + // large page threshold + if (buf_size >= __offload_use_2mb_buffers) { + buffer_flags = COI_OPTIMIZE_HUGE_PAGE_SIZE; + } + res = COI::BufferCreate(buf_size, + COI_BUFFER_NORMAL, + buffer_flags, + 0, + 1, + &m_device.get_process(), + &new_el->stack_ptr_data->mic_buf); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_create, res); + } + return false; + } + // make buffer valid on the device. + res = COI::BufferSetState(new_el->stack_ptr_data->mic_buf, + m_device.get_process(), + COI_BUFFER_VALID, + COI_BUFFER_NO_MOVE, + 0, 0, 0); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_set_state, res); + } + return false; + } + res = COI::BufferSetState(new_el->stack_ptr_data->mic_buf, + COI_PROCESS_SOURCE, + COI_BUFFER_INVALID, + COI_BUFFER_NO_MOVE, + 0, 0, 0); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + } + else if (m_is_mandatory) { + report_coi_error(c_buf_set_state, res); + } + return false; + } + // persistence algorithm requires target stack initialy to be nullified + if (!nullify_target_stack(new_el->stack_ptr_data->mic_buf, buf_size)) { + return false; + } + + m_stack_ptr_data = new_el->stack_ptr_data; + init_mic_address(m_stack_ptr_data); + OFFLOAD_TRACE(3, "Allocating stack buffer with addr %p\n", + m_stack_ptr_data->mic_addr); + m_device.m_persist_list.push_front(*new_el); + init_mic_address(new_el->stack_ptr_data); + *is_new = true; + return true; +} + +bool OffloadDescriptor::setup_descriptors( + VarDesc *vars, + VarDesc2 *vars2, + int vars_total, + int entry_id, + const void *stack_addr +) +{ + COIRESULT res; + + OffloadTimer timer(get_timer_data(), c_offload_host_setup_buffers); + + // make a copy of variable descriptors + m_vars_total = vars_total; + if (vars_total > 0) { + m_vars = (VarDesc*) malloc(m_vars_total * sizeof(VarDesc)); + if (m_vars == NULL) + LIBOFFLOAD_ERROR(c_malloc); + memcpy(m_vars, vars, m_vars_total * sizeof(VarDesc)); + m_vars_extra = (VarExtra*) malloc(m_vars_total * sizeof(VarExtra)); + if (m_vars_extra == NULL) + LIBOFFLOAD_ERROR(c_malloc); + } + + // dependencies + m_in_deps = (COIEVENT*) malloc(sizeof(COIEVENT) * (m_vars_total + 1)); + if (m_in_deps == NULL) + LIBOFFLOAD_ERROR(c_malloc); + if (m_vars_total > 0) { + m_out_deps = (COIEVENT*) malloc(sizeof(COIEVENT) * m_vars_total); + if (m_out_deps == NULL) + LIBOFFLOAD_ERROR(c_malloc); + } + + // copyin/copyout data length + m_in_datalen = 0; + m_out_datalen = 0; + + // First pass over variable descriptors + // - Calculate size of the input and output non-pointer data + // - Allocate buffers for input and output pointers + for (int i = 0; i < m_vars_total; i++) { + void* alloc_base = NULL; + int64_t alloc_disp = 0; + int64_t alloc_size; + bool src_is_for_mic = (m_vars[i].direction.out || + m_vars[i].into == NULL); + + const char *var_sname = ""; + if (vars2 != NULL && i < vars_total) { + if (vars2[i].sname != NULL) { + var_sname = vars2[i].sname; + } + } + OFFLOAD_TRACE(2, " VarDesc %d, var=%s, %s, %s\n", + i, var_sname, + vardesc_direction_as_string[m_vars[i].direction.bits], + vardesc_type_as_string[m_vars[i].type.src]); + if (vars2 != NULL && i < vars_total && vars2[i].dname != NULL) { + OFFLOAD_TRACE(2, " into=%s, %s\n", vars2[i].dname, + vardesc_type_as_string[m_vars[i].type.dst]); + } + OFFLOAD_TRACE(2, + " type_src=%d, type_dstn=%d, direction=%d, " + "alloc_if=%d, free_if=%d, align=%d, mic_offset=%d, flags=0x%x, " + "offset=%lld, size=%lld, count/disp=%lld, ptr=%p, into=%p\n", + m_vars[i].type.src, + m_vars[i].type.dst, + m_vars[i].direction.bits, + m_vars[i].alloc_if, + m_vars[i].free_if, + m_vars[i].align, + m_vars[i].mic_offset, + m_vars[i].flags.bits, + m_vars[i].offset, + m_vars[i].size, + m_vars[i].count, + m_vars[i].ptr, + m_vars[i].into); + + if (m_vars[i].alloc != NULL) { + // array descriptor + const arr_desc *ap = + static_cast<const arr_desc*>(m_vars[i].alloc); + + // debug dump + __arr_desc_dump(" ", "ALLOC", ap, 0); + + __arr_data_offset_and_length(ap, alloc_disp, alloc_size); + + alloc_base = reinterpret_cast<void*>(ap->base); + } + + m_vars_extra[i].cpu_disp = 0; + m_vars_extra[i].cpu_offset = 0; + m_vars_extra[i].src_data = 0; + m_vars_extra[i].read_rng_src = 0; + m_vars_extra[i].read_rng_dst = 0; + // flag is_arr_ptr_el is 1 only for var_descs generated + // for c_data_ptr_array type + if (i < vars_total) { + m_vars_extra[i].is_arr_ptr_el = 0; + } + + switch (m_vars[i].type.src) { + case c_data_ptr_array: + { + const arr_desc *ap; + const VarDesc3 *vd3 = + static_cast<const VarDesc3*>(m_vars[i].ptr); + int flags = vd3->array_fields; + OFFLOAD_TRACE(2, + " pointer array flags = %04x\n", flags); + OFFLOAD_TRACE(2, + " pointer array type is %s\n", + vardesc_type_as_string[flags & 0x3f]); + ap = static_cast<const arr_desc*>(vd3->ptr_array); + __arr_desc_dump(" ", "ptr array", ap, 0); + if (m_vars[i].into) { + ap = static_cast<const arr_desc*>(m_vars[i].into); + __arr_desc_dump( + " ", "into array", ap, 0); + } + if ((flags & (1<<flag_align_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->align_array); + __arr_desc_dump( + " ", "align array", ap, 0); + } + if ((flags & (1<<flag_alloc_if_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->alloc_if_array); + __arr_desc_dump( + " ", "alloc_if array", ap, 0); + } + if ((flags & (1<<flag_free_if_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->free_if_array); + __arr_desc_dump( + " ", "free_if array", ap, 0); + } + if ((flags & (1<<flag_extent_start_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->extent_start); + __arr_desc_dump( + " ", "extent_start array", ap, 0); + } else if ((flags & + (1<<flag_extent_start_is_scalar)) != 0) { + OFFLOAD_TRACE(2, + " extent_start scalar = %d\n", + (int64_t)vd3->extent_start); + } + if ((flags & (1<<flag_extent_elements_is_array)) != 0) { + ap = static_cast<const arr_desc*> + (vd3->extent_elements); + __arr_desc_dump( + " ", "extent_elements array", ap, 0); + } else if ((flags & + (1<<flag_extent_elements_is_scalar)) != 0) { + OFFLOAD_TRACE(2, + " extent_elements scalar = %d\n", + (int64_t)vd3->extent_elements); + } + if ((flags & (1<<flag_into_start_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->into_start); + __arr_desc_dump( + " ", "into_start array", ap, 0); + } else if ((flags & + (1<<flag_into_start_is_scalar)) != 0) { + OFFLOAD_TRACE(2, + " into_start scalar = %d\n", + (int64_t)vd3->into_start); + } + if ((flags & (1<<flag_into_elements_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->into_elements); + __arr_desc_dump( + " ", "into_elements array", ap, 0); + } else if ((flags & + (1<<flag_into_elements_is_scalar)) != 0) { + OFFLOAD_TRACE(2, + " into_elements scalar = %d\n", + (int64_t)vd3->into_elements); + } + if ((flags & (1<<flag_alloc_start_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->alloc_start); + __arr_desc_dump( + " ", "alloc_start array", ap, 0); + } else if ((flags & + (1<<flag_alloc_start_is_scalar)) != 0) { + OFFLOAD_TRACE(2, + " alloc_start scalar = %d\n", + (int64_t)vd3->alloc_start); + } + if ((flags & (1<<flag_alloc_elements_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->alloc_elements); + __arr_desc_dump( + " ", "alloc_elements array", ap, 0); + } else if ((flags & + (1<<flag_alloc_elements_is_scalar)) != 0) { + OFFLOAD_TRACE(2, + " alloc_elements scalar = %d\n", + (int64_t)vd3->alloc_elements); + } + } + if (!gen_var_descs_for_pointer_array(i)) { + return false; + } + break; + + case c_data: + case c_void_ptr: + case c_cean_var: + // In all uses later + // VarDesc.size will have the length of the data to be + // transferred + // VarDesc.disp will have an offset from base + if (m_vars[i].type.src == c_cean_var) { + // array descriptor + const arr_desc *ap = + static_cast<const arr_desc*>(m_vars[i].ptr); + + // debug dump + __arr_desc_dump("", "IN/OUT", ap, 0); + + // offset and length are derived from the array descriptor + __arr_data_offset_and_length(ap, m_vars[i].disp, + m_vars[i].size); + if (!is_arr_desc_contiguous(ap)) { + m_vars[i].flags.is_noncont_src = 1; + m_vars_extra[i].read_rng_src = + init_read_ranges_arr_desc(ap); + } + // all necessary information about length and offset is + // transferred in var descriptor. There is no need to send + // array descriptor to the target side. + m_vars[i].ptr = reinterpret_cast<void*>(ap->base); + } + else { + m_vars[i].size *= m_vars[i].count; + m_vars[i].disp = 0; + } + + if (m_vars[i].direction.bits) { + // make sure that transfer size > 0 + if (m_vars[i].size <= 0) { + LIBOFFLOAD_ERROR(c_zero_or_neg_transfer_size); + exit(1); + } + + if (m_vars[i].flags.is_static) { + PtrData *ptr_data; + + // find data associated with variable + if (!find_ptr_data(ptr_data, + m_vars[i].ptr, + m_vars[i].disp, + m_vars[i].size, + false)) { + return false; + } + + if (ptr_data != 0) { + // offset to base from the beginning of the buffer + // memory + m_vars[i].offset = + (char*) m_vars[i].ptr - + (char*) ptr_data->cpu_addr.start(); + } + else { + m_vars[i].flags.is_static = false; + if (m_vars[i].into == NULL) { + m_vars[i].flags.is_static_dstn = false; + } + } + m_vars_extra[i].src_data = ptr_data; + } + + if (m_is_openmp) { + if (m_vars[i].flags.is_static) { + // Static data is transferred only by omp target + // update construct which passes zeros for + // alloc_if and free_if. + if (m_vars[i].alloc_if || m_vars[i].free_if) { + m_vars[i].direction.bits = c_parameter_nocopy; + } + } + else { + AutoData *auto_data; + if (m_vars[i].alloc_if) { + auto_data = m_device.insert_auto_data( + m_vars[i].ptr, m_vars[i].size); + auto_data->add_reference(); + } + else { + // TODO: what should be done if var is not in + // the table? + auto_data = m_device.find_auto_data( + m_vars[i].ptr); + } + + // For automatic variables data is transferred + // only if alloc_if == 0 && free_if == 0 + // or reference count is 1 + if ((m_vars[i].alloc_if || m_vars[i].free_if) && + auto_data != 0 && + auto_data->get_reference() != 1) { + m_vars[i].direction.bits = c_parameter_nocopy; + } + + // save data for later use + m_vars_extra[i].auto_data = auto_data; + } + } + + if (m_vars[i].direction.in && + !m_vars[i].flags.is_static) { + m_in_datalen += m_vars[i].size; + + // for non-static target destination defined as CEAN + // expression we pass to target its size and dist + if (m_vars[i].into == NULL && + m_vars[i].type.src == c_cean_var) { + m_in_datalen += 2 * sizeof(uint64_t); + } + m_need_runfunction = true; + } + if (m_vars[i].direction.out && + !m_vars[i].flags.is_static) { + m_out_datalen += m_vars[i].size; + m_need_runfunction = true; + } + } + break; + + case c_dv: + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + ArrDesc *dvp = static_cast<ArrDesc*>(m_vars[i].ptr); + + // debug dump + __dv_desc_dump("IN/OUT", dvp); + + // send dope vector contents excluding base + m_in_datalen += m_vars[i].size - sizeof(uint64_t); + m_need_runfunction = true; + } + break; + + case c_string_ptr: + if ((m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) && + m_vars[i].size == 0) { + m_vars[i].size = 1; + m_vars[i].count = + strlen(*static_cast<char**>(m_vars[i].ptr)) + 1; + } + /* fallthru */ + + case c_data_ptr: + if (m_vars[i].flags.is_stack_buf && + !m_vars[i].direction.bits && + m_vars[i].alloc_if) { + // this var_desc is for stack buffer + bool is_new; + + if (!offload_stack_memory_manager( + stack_addr, entry_id, + m_vars[i].count, m_vars[i].align, &is_new)) { + return false; + } + if (is_new) { + m_compute_buffers.push_back( + m_stack_ptr_data->mic_buf); + m_device.m_persist_list.front().cpu_stack_addr = + static_cast<char*>(m_vars[i].ptr); + } + else { + m_vars[i].flags.sink_addr = 1; + m_in_datalen += sizeof(m_stack_ptr_data->mic_addr); + } + m_vars[i].size = m_destroy_stack.size(); + m_vars_extra[i].src_data = m_stack_ptr_data; + // need to add reference for buffer + m_need_runfunction = true; + break; + } + /* fallthru */ + + case c_cean_var_ptr: + case c_dv_ptr: + if (m_vars[i].type.src == c_cean_var_ptr) { + // array descriptor + const arr_desc *ap = + static_cast<const arr_desc*>(m_vars[i].ptr); + + // debug dump + __arr_desc_dump("", "IN/OUT", ap, 1); + + // offset and length are derived from the array descriptor + __arr_data_offset_and_length(ap, m_vars[i].disp, + m_vars[i].size); + + if (!is_arr_desc_contiguous(ap)) { + m_vars[i].flags.is_noncont_src = 1; + m_vars_extra[i].read_rng_src = + init_read_ranges_arr_desc(ap); + } + // all necessary information about length and offset is + // transferred in var descriptor. There is no need to send + // array descriptor to the target side. + m_vars[i].ptr = reinterpret_cast<void*>(ap->base); + } + else if (m_vars[i].type.src == c_dv_ptr) { + // need to send DV to the device unless it is 'nocopy' + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + ArrDesc *dvp = *static_cast<ArrDesc**>(m_vars[i].ptr); + + // debug dump + __dv_desc_dump("IN/OUT", dvp); + + m_vars[i].direction.bits = c_parameter_in; + } + + // no displacement + m_vars[i].disp = 0; + } + else { + // c_data_ptr or c_string_ptr + m_vars[i].size *= m_vars[i].count; + m_vars[i].disp = 0; + } + + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + PtrData *ptr_data; + + // check that buffer length >= 0 + if (m_vars[i].alloc_if && + m_vars[i].disp + m_vars[i].size < 0) { + LIBOFFLOAD_ERROR(c_zero_or_neg_ptr_len); + exit(1); + } + + // base address + void *base = *static_cast<void**>(m_vars[i].ptr); + + // allocate buffer if we have no INTO and don't need + // allocation for the ptr at target + if (src_is_for_mic) { + if (m_vars[i].flags.is_stack_buf) { + // for stack persistent objects ptr data is created + // by var_desc with number 0. + // Its ptr_data is stored at m_stack_ptr_data + ptr_data = m_stack_ptr_data; + m_vars[i].flags.sink_addr = 1; + } + else if (m_vars[i].alloc_if) { + // add new entry + if (!alloc_ptr_data( + ptr_data, + base, + (alloc_base != NULL) ? + alloc_disp : m_vars[i].disp, + (alloc_base != NULL) ? + alloc_size : m_vars[i].size, + alloc_disp, + (alloc_base != NULL) ? + 0 : m_vars[i].align)) { + return false; + } + + if (ptr_data->add_reference() == 0 && + ptr_data->mic_buf != 0) { + // add buffer to the list of buffers that + // are passed to dispatch call + m_compute_buffers.push_back( + ptr_data->mic_buf); + } + else { + // will send buffer address to device + m_vars[i].flags.sink_addr = 1; + } + + if (!ptr_data->is_static) { + // need to add reference for buffer + m_need_runfunction = true; + } + } + else { + bool error_if_not_found = true; + if (m_is_openmp) { + // For omp target update variable is ignored + // if it does not exist. + if (!m_vars[i].alloc_if && + !m_vars[i].free_if) { + error_if_not_found = false; + } + } + + // use existing association from pointer table + if (!find_ptr_data(ptr_data, + base, + m_vars[i].disp, + m_vars[i].size, + error_if_not_found)) { + return false; + } + + if (m_is_openmp) { + // make var nocopy if it does not exist + if (ptr_data == 0) { + m_vars[i].direction.bits = + c_parameter_nocopy; + } + } + + if (ptr_data != 0) { + m_vars[i].flags.sink_addr = 1; + } + } + + if (ptr_data != 0) { + if (m_is_openmp) { + // data is transferred only if + // alloc_if == 0 && free_if == 0 + // or reference count is 1 + if ((m_vars[i].alloc_if || + m_vars[i].free_if) && + ptr_data->get_reference() != 1) { + m_vars[i].direction.bits = + c_parameter_nocopy; + } + } + + if (ptr_data->alloc_disp != 0) { + m_vars[i].flags.alloc_disp = 1; + m_in_datalen += sizeof(alloc_disp); + } + + if (m_vars[i].flags.sink_addr) { + // get buffers's address on the sink + if (!init_mic_address(ptr_data)) { + return false; + } + + m_in_datalen += sizeof(ptr_data->mic_addr); + } + + if (!ptr_data->is_static && m_vars[i].free_if) { + // need to decrement buffer reference on target + m_need_runfunction = true; + } + + // offset to base from the beginning of the buffer + // memory + m_vars[i].offset = (char*) base - + (char*) ptr_data->cpu_addr.start(); + + // copy other pointer properties to var descriptor + m_vars[i].mic_offset = ptr_data->mic_offset; + m_vars[i].flags.is_static = ptr_data->is_static; + } + } + else { + if (!find_ptr_data(ptr_data, + base, + m_vars[i].disp, + m_vars[i].size, + false)) { + return false; + } + if (ptr_data) { + m_vars[i].offset = + (char*) base - + (char*) ptr_data->cpu_addr.start(); + } + } + + // save pointer data + m_vars_extra[i].src_data = ptr_data; + } + break; + + case c_func_ptr: + if (m_vars[i].direction.in) { + m_in_datalen += __offload_funcs.max_name_length(); + } + if (m_vars[i].direction.out) { + m_out_datalen += __offload_funcs.max_name_length(); + } + m_need_runfunction = true; + break; + + case c_dv_data: + case c_dv_ptr_data: + case c_dv_data_slice: + case c_dv_ptr_data_slice: + ArrDesc *dvp; + if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src)) { + const arr_desc *ap; + ap = static_cast<const arr_desc*>(m_vars[i].ptr); + + dvp = (m_vars[i].type.src == c_dv_data_slice) ? + reinterpret_cast<ArrDesc*>(ap->base) : + *reinterpret_cast<ArrDesc**>(ap->base); + } + else { + dvp = (m_vars[i].type.src == c_dv_data) ? + static_cast<ArrDesc*>(m_vars[i].ptr) : + *static_cast<ArrDesc**>(m_vars[i].ptr); + } + + // if allocatable dope vector isn't allocated don't + // transfer its data + if (!__dv_is_allocated(dvp)) { + m_vars[i].direction.bits = c_parameter_nocopy; + m_vars[i].alloc_if = 0; + m_vars[i].free_if = 0; + } + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + const arr_desc *ap; + + if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src)) { + ap = static_cast<const arr_desc*>(m_vars[i].ptr); + + // debug dump + __arr_desc_dump("", "IN/OUT", ap, 0); + } + if (!__dv_is_contiguous(dvp)) { + m_vars[i].flags.is_noncont_src = 1; + m_vars_extra[i].read_rng_src = + init_read_ranges_dv(dvp); + } + + // size and displacement + if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src)) { + // offset and length are derived from the + // array descriptor + __arr_data_offset_and_length(ap, + m_vars[i].disp, + m_vars[i].size); + if (m_vars[i].direction.bits) { + if (!is_arr_desc_contiguous(ap)) { + if (m_vars[i].flags.is_noncont_src) { + LIBOFFLOAD_ERROR(c_slice_of_noncont_array); + return false; + } + m_vars[i].flags.is_noncont_src = 1; + m_vars_extra[i].read_rng_src = + init_read_ranges_arr_desc(ap); + } + } + } + else { + if (m_vars[i].flags.has_length) { + m_vars[i].size = + __dv_data_length(dvp, m_vars[i].count); + } + else { + m_vars[i].size = __dv_data_length(dvp); + } + m_vars[i].disp = 0; + } + + // check that length >= 0 + if (m_vars[i].alloc_if && + (m_vars[i].disp + m_vars[i].size < 0)) { + LIBOFFLOAD_ERROR(c_zero_or_neg_ptr_len); + exit(1); + } + + // base address + void *base = reinterpret_cast<void*>(dvp->Base); + PtrData *ptr_data; + + // allocate buffer if we have no INTO and don't need + // allocation for the ptr at target + if (src_is_for_mic) { + if (m_vars[i].alloc_if) { + // add new entry + if (!alloc_ptr_data( + ptr_data, + base, + (alloc_base != NULL) ? + alloc_disp : m_vars[i].disp, + (alloc_base != NULL) ? + alloc_size : m_vars[i].size, + alloc_disp, + (alloc_base != NULL) ? + 0 : m_vars[i].align)) { + return false; + } + + if (ptr_data->add_reference() == 0 && + ptr_data->mic_buf != 0) { + // add buffer to the list of buffers + // that are passed to dispatch call + m_compute_buffers.push_back( + ptr_data->mic_buf); + } + else { + // will send buffer address to device + m_vars[i].flags.sink_addr = 1; + } + + if (!ptr_data->is_static) { + // need to add reference for buffer + m_need_runfunction = true; + } + } + else { + bool error_if_not_found = true; + if (m_is_openmp) { + // For omp target update variable is ignored + // if it does not exist. + if (!m_vars[i].alloc_if && + !m_vars[i].free_if) { + error_if_not_found = false; + } + } + + // use existing association from pointer table + if (!find_ptr_data(ptr_data, + base, + m_vars[i].disp, + m_vars[i].size, + error_if_not_found)) { + return false; + } + + if (m_is_openmp) { + // make var nocopy if it does not exist + if (ptr_data == 0) { + m_vars[i].direction.bits = + c_parameter_nocopy; + } + } + + if (ptr_data != 0) { + // need to update base in dope vector on device + m_vars[i].flags.sink_addr = 1; + } + } + + if (ptr_data != 0) { + if (m_is_openmp) { + // data is transferred only if + // alloc_if == 0 && free_if == 0 + // or reference count is 1 + if ((m_vars[i].alloc_if || + m_vars[i].free_if) && + ptr_data->get_reference() != 1) { + m_vars[i].direction.bits = + c_parameter_nocopy; + } + } + + if (ptr_data->alloc_disp != 0) { + m_vars[i].flags.alloc_disp = 1; + m_in_datalen += sizeof(alloc_disp); + } + + if (m_vars[i].flags.sink_addr) { + // get buffers's address on the sink + if (!init_mic_address(ptr_data)) { + return false; + } + + m_in_datalen += sizeof(ptr_data->mic_addr); + } + + if (!ptr_data->is_static && m_vars[i].free_if) { + // need to decrement buffer reference on target + m_need_runfunction = true; + } + + // offset to base from the beginning of the buffer + // memory + m_vars[i].offset = + (char*) base - + (char*) ptr_data->cpu_addr.start(); + + // copy other pointer properties to var descriptor + m_vars[i].mic_offset = ptr_data->mic_offset; + m_vars[i].flags.is_static = ptr_data->is_static; + } + } + else { // !src_is_for_mic + if (!find_ptr_data(ptr_data, + base, + m_vars[i].disp, + m_vars[i].size, + false)) { + return false; + } + m_vars[i].offset = !ptr_data ? 0 : + (char*) base - + (char*) ptr_data->cpu_addr.start(); + } + + // save pointer data + m_vars_extra[i].src_data = ptr_data; + } + break; + + default: + LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars[i].type.src); + LIBOFFLOAD_ABORT; + } + if (m_vars[i].type.src == c_data_ptr_array) { + continue; + } + + if (src_is_for_mic && m_vars[i].flags.is_stack_buf) { + m_vars[i].offset = static_cast<char*>(m_vars[i].ptr) - + m_device.m_persist_list.front().cpu_stack_addr; + } + // if source is used at CPU save its offset and disp + if (m_vars[i].into == NULL || m_vars[i].direction.in) { + m_vars_extra[i].cpu_offset = m_vars[i].offset; + m_vars_extra[i].cpu_disp = m_vars[i].disp; + } + + // If "into" is define we need to do the similar work for it + if (!m_vars[i].into) { + continue; + } + + int64_t into_disp =0, into_offset = 0; + + switch (m_vars[i].type.dst) { + case c_data_ptr_array: + break; + case c_data: + case c_void_ptr: + case c_cean_var: { + int64_t size = m_vars[i].size; + + if (m_vars[i].type.dst == c_cean_var) { + // array descriptor + const arr_desc *ap = + static_cast<const arr_desc*>(m_vars[i].into); + + // debug dump + __arr_desc_dump(" ", "INTO", ap, 0); + + // offset and length are derived from the array descriptor + __arr_data_offset_and_length(ap, into_disp, size); + + if (!is_arr_desc_contiguous(ap)) { + m_vars[i].flags.is_noncont_dst = 1; + m_vars_extra[i].read_rng_dst = + init_read_ranges_arr_desc(ap); + if (!cean_ranges_match( + m_vars_extra[i].read_rng_src, + m_vars_extra[i].read_rng_dst)) { + LIBOFFLOAD_ERROR(c_ranges_dont_match); + exit(1); + } + } + m_vars[i].into = reinterpret_cast<void*>(ap->base); + } + + int64_t size_src = m_vars_extra[i].read_rng_src ? + cean_get_transf_size(m_vars_extra[i].read_rng_src) : + m_vars[i].size; + int64_t size_dst = m_vars_extra[i].read_rng_dst ? + cean_get_transf_size(m_vars_extra[i].read_rng_dst) : + size; + // It's supposed that "into" size must be not less + // than src size + if (size_src > size_dst) { + LIBOFFLOAD_ERROR(c_different_src_and_dstn_sizes, + size_src, size_dst); + exit(1); + } + + if (m_vars[i].direction.bits) { + if (m_vars[i].flags.is_static_dstn) { + PtrData *ptr_data; + + // find data associated with variable + if (!find_ptr_data(ptr_data, m_vars[i].into, + into_disp, size, false)) { + return false; + } + if (ptr_data != 0) { + // offset to base from the beginning of the buffer + // memory + into_offset = + (char*) m_vars[i].into - + (char*) ptr_data->cpu_addr.start(); + } + else { + m_vars[i].flags.is_static_dstn = false; + } + m_vars_extra[i].dst_data = ptr_data; + } + } + + if (m_vars[i].direction.in && + !m_vars[i].flags.is_static_dstn) { + m_in_datalen += m_vars[i].size; + + // for non-static target destination defined as CEAN + // expression we pass to target its size and dist + if (m_vars[i].type.dst == c_cean_var) { + m_in_datalen += 2 * sizeof(uint64_t); + } + m_need_runfunction = true; + } + break; + } + + case c_dv: + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + ArrDesc *dvp = static_cast<ArrDesc*>(m_vars[i].into); + + // debug dump + __dv_desc_dump("INTO", dvp); + + // send dope vector contents excluding base + m_in_datalen += m_vars[i].size - sizeof(uint64_t); + m_need_runfunction = true; + } + break; + + case c_string_ptr: + case c_data_ptr: + case c_cean_var_ptr: + case c_dv_ptr: { + int64_t size = m_vars[i].size; + + if (m_vars[i].type.dst == c_cean_var_ptr) { + // array descriptor + const arr_desc *ap = + static_cast<const arr_desc*>(m_vars[i].into); + + // debug dump + __arr_desc_dump(" ", "INTO", ap, 1); + + // offset and length are derived from the array descriptor + __arr_data_offset_and_length(ap, into_disp, size); + + if (!is_arr_desc_contiguous(ap)) { + m_vars[i].flags.is_noncont_src = 1; + m_vars_extra[i].read_rng_dst = + init_read_ranges_arr_desc(ap); + if (!cean_ranges_match( + m_vars_extra[i].read_rng_src, + m_vars_extra[i].read_rng_dst)) { + LIBOFFLOAD_ERROR(c_ranges_dont_match); + } + } + m_vars[i].into = reinterpret_cast<char**>(ap->base); + } + else if (m_vars[i].type.dst == c_dv_ptr) { + // need to send DV to the device unless it is 'nocopy' + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + ArrDesc *dvp = *static_cast<ArrDesc**>(m_vars[i].into); + + // debug dump + __dv_desc_dump("INTO", dvp); + + m_vars[i].direction.bits = c_parameter_in; + } + } + + int64_t size_src = m_vars_extra[i].read_rng_src ? + cean_get_transf_size(m_vars_extra[i].read_rng_src) : + m_vars[i].size; + int64_t size_dst = m_vars_extra[i].read_rng_dst ? + cean_get_transf_size(m_vars_extra[i].read_rng_dst) : + size; + // It's supposed that "into" size must be not less than + // src size + if (size_src > size_dst) { + LIBOFFLOAD_ERROR(c_different_src_and_dstn_sizes, + size_src, size_dst); + exit(1); + } + + if (m_vars[i].direction.bits) { + PtrData *ptr_data; + + // base address + void *base = *static_cast<void**>(m_vars[i].into); + + if (m_vars[i].direction.in) { + // allocate buffer + if (m_vars[i].flags.is_stack_buf) { + // for stack persistent objects ptr data is created + // by var_desc with number 0. + // Its ptr_data is stored at m_stack_ptr_data + ptr_data = m_stack_ptr_data; + m_vars[i].flags.sink_addr = 1; + } + else if (m_vars[i].alloc_if) { + // add new entry + if (!alloc_ptr_data( + ptr_data, + base, + (alloc_base != NULL) ? + alloc_disp : into_disp, + (alloc_base != NULL) ? + alloc_size : size, + alloc_disp, + (alloc_base != NULL) ? + 0 : m_vars[i].align)) { + return false; + } + + if (ptr_data->add_reference() == 0 && + ptr_data->mic_buf != 0) { + // add buffer to the list of buffers that + // are passed to dispatch call + m_compute_buffers.push_back( + ptr_data->mic_buf); + } + else { + // will send buffer address to device + m_vars[i].flags.sink_addr = 1; + } + + if (!ptr_data->is_static) { + // need to add reference for buffer + m_need_runfunction = true; + } + } + else { + // use existing association from pointer table + if (!find_ptr_data(ptr_data, base, into_disp, size)) { + return false; + } + m_vars[i].flags.sink_addr = 1; + } + + if (ptr_data->alloc_disp != 0) { + m_vars[i].flags.alloc_disp = 1; + m_in_datalen += sizeof(alloc_disp); + } + + if (m_vars[i].flags.sink_addr) { + // get buffers's address on the sink + if (!init_mic_address(ptr_data)) { + return false; + } + + m_in_datalen += sizeof(ptr_data->mic_addr); + } + + if (!ptr_data->is_static && m_vars[i].free_if) { + // need to decrement buffer reference on target + m_need_runfunction = true; + } + + // copy other pointer properties to var descriptor + m_vars[i].mic_offset = ptr_data->mic_offset; + m_vars[i].flags.is_static_dstn = ptr_data->is_static; + } + else { + if (!find_ptr_data(ptr_data, + base, + into_disp, + m_vars[i].size, + false)) { + return false; + } + } + if (ptr_data) { + into_offset = ptr_data ? + (char*) base - + (char*) ptr_data->cpu_addr.start() : + 0; + } + // save pointer data + m_vars_extra[i].dst_data = ptr_data; + } + break; + } + + case c_func_ptr: + break; + + case c_dv_data: + case c_dv_ptr_data: + case c_dv_data_slice: + case c_dv_ptr_data_slice: + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + const arr_desc *ap; + ArrDesc *dvp; + PtrData *ptr_data; + int64_t disp; + int64_t size; + + if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.dst)) { + ap = static_cast<const arr_desc*>(m_vars[i].into); + + // debug dump + __arr_desc_dump(" ", "INTO", ap, 0); + + dvp = (m_vars[i].type.dst == c_dv_data_slice) ? + reinterpret_cast<ArrDesc*>(ap->base) : + *reinterpret_cast<ArrDesc**>(ap->base); + } + else { + dvp = (m_vars[i].type.dst == c_dv_data) ? + static_cast<ArrDesc*>(m_vars[i].into) : + *static_cast<ArrDesc**>(m_vars[i].into); + } + if (!__dv_is_contiguous(dvp)) { + m_vars[i].flags.is_noncont_dst = 1; + m_vars_extra[i].read_rng_dst = + init_read_ranges_dv(dvp); + } + // size and displacement + if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.dst)) { + // offset and length are derived from the array + // descriptor + __arr_data_offset_and_length(ap, into_disp, size); + if (m_vars[i].direction.bits) { + if (!is_arr_desc_contiguous(ap)) { + if (m_vars[i].flags.is_noncont_dst) { + LIBOFFLOAD_ERROR(c_slice_of_noncont_array); + return false; + } + m_vars[i].flags.is_noncont_dst = 1; + m_vars_extra[i].read_rng_dst = + init_read_ranges_arr_desc(ap); + if (!cean_ranges_match( + m_vars_extra[i].read_rng_src, + m_vars_extra[i].read_rng_dst)) { + LIBOFFLOAD_ERROR(c_ranges_dont_match); + } + } + } + } + else { + if (m_vars[i].flags.has_length) { + size = __dv_data_length(dvp, m_vars[i].count); + } + else { + size = __dv_data_length(dvp); + } + disp = 0; + } + + int64_t size_src = + m_vars_extra[i].read_rng_src ? + cean_get_transf_size(m_vars_extra[i].read_rng_src) : + m_vars[i].size; + int64_t size_dst = + m_vars_extra[i].read_rng_dst ? + cean_get_transf_size(m_vars_extra[i].read_rng_dst) : + size; + // It's supposed that "into" size must be not less + // than src size + if (size_src > size_dst) { + LIBOFFLOAD_ERROR(c_different_src_and_dstn_sizes, + size_src, size_dst); + exit(1); + } + + // base address + void *base = reinterpret_cast<void*>(dvp->Base); + + // allocate buffer + if (m_vars[i].direction.in) { + if (m_vars[i].alloc_if) { + // add new entry + if (!alloc_ptr_data( + ptr_data, + base, + (alloc_base != NULL) ? + alloc_disp : into_disp, + (alloc_base != NULL) ? + alloc_size : size, + alloc_disp, + (alloc_base != NULL) ? + 0 : m_vars[i].align)) { + return false; + } + if (ptr_data->add_reference() == 0 && + ptr_data->mic_buf !=0) { + // add buffer to the list of buffers + // that are passed to dispatch call + m_compute_buffers.push_back( + ptr_data->mic_buf); + } + else { + // will send buffer address to device + m_vars[i].flags.sink_addr = 1; + } + + if (!ptr_data->is_static) { + // need to add reference for buffer + m_need_runfunction = true; + } + } + else { + // use existing association from pointer table + if (!find_ptr_data(ptr_data, base, into_disp, size)) { + return false; + } + + // need to update base in dope vector on device + m_vars[i].flags.sink_addr = 1; + } + + if (ptr_data->alloc_disp != 0) { + m_vars[i].flags.alloc_disp = 1; + m_in_datalen += sizeof(alloc_disp); + } + + if (m_vars[i].flags.sink_addr) { + // get buffers's address on the sink + if (!init_mic_address(ptr_data)) { + return false; + } + m_in_datalen += sizeof(ptr_data->mic_addr); + } + + if (!ptr_data->is_static && m_vars[i].free_if) { + // need to decrement buffer reference on target + m_need_runfunction = true; + } + + // offset to base from the beginning of the buffer + // memory + into_offset = + (char*) base - (char*) ptr_data->cpu_addr.start(); + + // copy other pointer properties to var descriptor + m_vars[i].mic_offset = ptr_data->mic_offset; + m_vars[i].flags.is_static_dstn = ptr_data->is_static; + } + else { // src_is_for_mic + if (!find_ptr_data(ptr_data, + base, + into_disp, + size, + false)) { + return false; + } + into_offset = !ptr_data ? + 0 : + (char*) base - (char*) ptr_data->cpu_addr.start(); + } + + // save pointer data + m_vars_extra[i].dst_data = ptr_data; + } + break; + + default: + LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars[i].type.src); + LIBOFFLOAD_ABORT; + } + // if into is used at CPU save its offset and disp + if (m_vars[i].direction.out) { + m_vars_extra[i].cpu_offset = into_offset; + m_vars_extra[i].cpu_disp = into_disp; + } + else { + if (m_vars[i].flags.is_stack_buf) { + into_offset = static_cast<char*>(m_vars[i].into) - + m_device.m_persist_list.front().cpu_stack_addr; + } + m_vars[i].offset = into_offset; + m_vars[i].disp = into_disp; + } + } + + return true; +} + +bool OffloadDescriptor::setup_misc_data(const char *name) +{ + OffloadTimer timer(get_timer_data(), c_offload_host_setup_misc_data); + + // we can skip run functon call together with wait if offloaded + // region is empty and there is no user defined non-pointer IN/OUT data + if (m_need_runfunction) { + // variable descriptors are sent as input data + m_in_datalen += m_vars_total * sizeof(VarDesc); + + // timer data is sent as a part of the output data + m_out_datalen += OFFLOAD_TIMER_DATALEN(); + + // max from input data and output data length + uint64_t data_len = m_in_datalen > m_out_datalen ? m_in_datalen : + m_out_datalen; + + // Misc data has the following layout + // <Function Descriptor> + // <Function Name> + // <In/Out Data> (optional) + // + // We can transfer copyin/copyout data in misc/return data which can + // be passed to run function call if its size does not exceed + // COI_PIPELINE_MAX_IN_MISC_DATA_LEN. Otherwise we have to allocate + // buffer for it. + + m_func_desc_size = sizeof(FunctionDescriptor) + strlen(name) + 1; + m_func_desc_size = (m_func_desc_size + 7) & ~7; + + int misc_data_offset = 0; + int misc_data_size = 0; + if (data_len > 0) { + if (m_func_desc_size + + m_in_datalen <= COI_PIPELINE_MAX_IN_MISC_DATA_LEN && + m_out_datalen <= COI_PIPELINE_MAX_IN_MISC_DATA_LEN) { + // use misc/return data for copyin/copyout + misc_data_offset = m_func_desc_size; + misc_data_size = data_len; + } + else { + OffloadTimer timer_buf(get_timer_data(), + c_offload_host_alloc_data_buffer); + + // send/receive data using buffer + COIRESULT res = COI::BufferCreate(data_len, + COI_BUFFER_NORMAL, + 0, 0, + 1, &m_device.get_process(), + &m_inout_buf); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_create, res); + } + + m_compute_buffers.push_back(m_inout_buf); + m_destroy_buffers.push_back(m_inout_buf); + } + } + + // initialize function descriptor + m_func_desc = (FunctionDescriptor*) malloc(m_func_desc_size + + misc_data_size); + if (m_func_desc == NULL) + LIBOFFLOAD_ERROR(c_malloc); + m_func_desc->console_enabled = console_enabled; + m_func_desc->timer_enabled = + timer_enabled || (offload_report_level && offload_report_enabled); + m_func_desc->offload_report_level = offload_report_level; + m_func_desc->offload_number = GET_OFFLOAD_NUMBER(get_timer_data()); + m_func_desc->in_datalen = m_in_datalen; + m_func_desc->out_datalen = m_out_datalen; + m_func_desc->vars_num = m_vars_total; + m_func_desc->data_offset = misc_data_offset; + + // append entry name + strcpy(m_func_desc->data, name); + } + + return true; +} + +bool OffloadDescriptor::wait_dependencies( + const void **waits, + int num_waits +) +{ + OffloadTimer timer(get_timer_data(), c_offload_host_wait_deps); + bool ret = true; + + for (int i = 0; i < num_waits; i++) { + + OffloadDescriptor *task = m_device.find_signal(waits[i], true); + if (task == 0) { + LIBOFFLOAD_ERROR(c_offload1, m_device.get_logical_index(), + waits[i]); + LIBOFFLOAD_ABORT; + } + + if (!task->offload_finish()) { + ret = false; + } + + task->cleanup(); + delete task; + } + + return ret; +} + +bool OffloadDescriptor::offload( + const char *name, + bool is_empty, + VarDesc *vars, + VarDesc2 *vars2, + int vars_total, + const void **waits, + int num_waits, + const void **signal, + int entry_id, + const void *stack_addr +) +{ + if (signal == 0) { + OFFLOAD_DEBUG_TRACE_1(1, + GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_init_func, + "Offload function %s, is_empty=%d, #varDescs=%d, " + "#waits=%d, signal=none\n", + name, is_empty, vars_total, num_waits); + OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_sent_pointer_data, + "#Wait : %d \n", num_waits); + OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_signal, + "none %d\n", 0); + } + else { + OFFLOAD_DEBUG_TRACE_1(1, + GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_init_func, + "Offload function %s, is_empty=%d, #varDescs=%d, " + "#waits=%d, signal=%p\n", + name, is_empty, vars_total, num_waits, + *signal); + + OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_signal, + "%d\n", signal); + } + OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_wait, + "#Wait : %d %p\n", num_waits, waits); + + if (m_status != 0) { + m_status->result = OFFLOAD_SUCCESS; + m_status->device_number = m_device.get_logical_index(); + } + + m_need_runfunction = !is_empty; + + // wait for dependencies to finish + if (!wait_dependencies(waits, num_waits)) { + cleanup(); + return false; + } + + // setup buffers + if (!setup_descriptors(vars, vars2, vars_total, entry_id, stack_addr)) { + cleanup(); + return false; + } + + // initiate send for pointers. Want to do it as early as possible. + if (!send_pointer_data(signal != 0)) { + cleanup(); + return false; + } + + // setup misc data for run function + if (!setup_misc_data(name)) { + cleanup(); + return false; + } + + // gather copyin data into buffer + if (!gather_copyin_data()) { + cleanup(); + return false; + } + + // Start the computation + if (!compute()) { + cleanup(); + return false; + } + + // initiate receive for pointers + if (!receive_pointer_data(signal != 0)) { + cleanup(); + return false; + } + + // if there is a signal save descriptor for the later use. + if (signal != 0) { + m_device.add_signal(*signal, this); + return true; + } + + // wait for the offload to finish. + if (!offload_finish()) { + cleanup(); + return false; + } + + cleanup(); + return true; +} + +bool OffloadDescriptor::offload_finish() +{ + COIRESULT res; + + // wait for compute dependencies to become signaled + if (m_in_deps_total > 0) { + OffloadTimer timer(get_timer_data(), c_offload_host_wait_compute); + + if (__offload_active_wait) { + // keep CPU busy + do { + res = COI::EventWait(m_in_deps_total, m_in_deps, 0, 1, 0, 0); + } + while (res == COI_TIME_OUT_REACHED); + } + else { + res = COI::EventWait(m_in_deps_total, m_in_deps, -1, 1, 0, 0); + } + + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_event_wait, res); + } + } + + // scatter copyout data received from target + if (!scatter_copyout_data()) { + return false; + } + // wait for receive dependencies to become signaled + if (m_out_deps_total > 0) { + OffloadTimer timer(get_timer_data(), c_offload_host_wait_buffers_reads); + + if (__offload_active_wait) { + // keep CPU busy + do { + res = COI::EventWait(m_out_deps_total, m_out_deps, 0, 1, 0, 0); + } + while (res == COI_TIME_OUT_REACHED); + } + else { + res = COI::EventWait(m_out_deps_total, m_out_deps, -1, 1, 0, 0); + } + + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_event_wait, res); + } + } + + // destroy buffers + { + OffloadTimer timer(get_timer_data(), c_offload_host_destroy_buffers); + + for (BufferList::const_iterator it = m_destroy_buffers.begin(); + it != m_destroy_buffers.end(); it++) { + res = COI::BufferDestroy(*it); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_destroy, res); + } + } + } + + return true; +} + +void OffloadDescriptor::cleanup() +{ + // release device in orsl + ORSL::release(m_device.get_logical_index()); + + OFFLOAD_TIMER_STOP(get_timer_data(), c_offload_host_total_offload); + + // report stuff + Offload_Report_Epilog(get_timer_data()); +} + +bool OffloadDescriptor::is_signaled() +{ + bool signaled = true; + COIRESULT res; + + // check compute and receive dependencies + if (m_in_deps_total > 0) { + res = COI::EventWait(m_in_deps_total, m_in_deps, 0, 1, 0, 0); + signaled = signaled && (res == COI_SUCCESS); + } + if (m_out_deps_total > 0) { + res = COI::EventWait(m_out_deps_total, m_out_deps, 0, 1, 0, 0); + signaled = signaled && (res == COI_SUCCESS); + } + + return signaled; +} + +// Send pointer data if source or destination or both of them are +// noncontiguous. There is guarantee that length of destination enough for +// transfered data. +bool OffloadDescriptor::send_noncontiguous_pointer_data( + int i, + PtrData* src_data, + PtrData* dst_data, + COIEVENT *event + ) +{ + int64_t offset_src, offset_dst; + int64_t length_src, length_dst; + int64_t length_src_cur, length_dst_cur; + int64_t send_size, data_sent = 0; + COIRESULT res; + bool dst_is_empty = true; + bool src_is_empty = true; + + // Set length_src and length_dst + length_src = (m_vars_extra[i].read_rng_src) ? + m_vars_extra[i].read_rng_src->range_size : m_vars[i].size; + length_dst = !m_vars[i].into ? length_src : + (m_vars_extra[i].read_rng_dst) ? + m_vars_extra[i].read_rng_dst->range_size : m_vars[i].size; + send_size = (length_src < length_dst) ? length_src : length_dst; + + // consequently get contiguous ranges, + // define corresponded destination offset and send data + do { + if (src_is_empty) { + if (m_vars_extra[i].read_rng_src) { + if (!get_next_range(m_vars_extra[i].read_rng_src, + &offset_src)) { + // source ranges are over - nothing to send + break; + } + } + else if (data_sent == 0) { + offset_src = m_vars_extra[i].cpu_disp; + } + else { + break; + } + length_src_cur = length_src; + } + else { + // if source is contiguous or its contiguous range is greater + // than destination one + offset_src += send_size; + } + length_src_cur -= send_size; + src_is_empty = length_src_cur == 0; + + if (dst_is_empty) { + if (m_vars[i].into) { + if (m_vars_extra[i].read_rng_dst) { + if (!get_next_range(m_vars_extra[i].read_rng_dst, + &offset_dst)) { + // destination ranges are over + LIBOFFLOAD_ERROR(c_destination_is_over); + return false; + } + } + // into is contiguous. + else { + offset_dst = m_vars[i].disp; + } + length_dst_cur = length_dst; + } + // same as source + else { + offset_dst = offset_src; + length_dst_cur = length_src; + } + } + else { + // if destination is contiguous or its contiguous range is greater + // than source one + offset_dst += send_size; + } + length_dst_cur -= send_size; + dst_is_empty = length_dst_cur == 0; + + if (src_data != 0 && src_data->cpu_buf != 0) { + res = COI::BufferCopy( + dst_data->mic_buf, + src_data->cpu_buf, + m_vars[i].mic_offset - dst_data->alloc_disp + + m_vars[i].offset + offset_dst, + m_vars_extra[i].cpu_offset + offset_src, + send_size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + char *base = offload_get_src_base(m_vars[i].ptr, + m_vars[i].type.src); + + res = COI::BufferWrite( + dst_data->mic_buf, + m_vars[i].mic_offset - dst_data->alloc_disp + + m_vars[i].offset + offset_dst, + base + offset_src, + send_size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_write, res); + } + } + data_sent += length_src; + } + while (true); + return true; +} + +bool OffloadDescriptor::send_pointer_data(bool is_async) +{ + OffloadTimer timer(get_timer_data(), c_offload_host_send_pointers); + + uint64_t ptr_sent = 0; + COIRESULT res; + + // Initiate send for pointer data + for (int i = 0; i < m_vars_total; i++) { + switch (m_vars[i].type.dst) { + case c_data_ptr_array: + break; + case c_data: + case c_void_ptr: + case c_cean_var: + if (m_vars[i].direction.in && + m_vars[i].flags.is_static_dstn) { + COIEVENT *event = + (is_async || + m_vars[i].size >= __offload_use_async_buffer_write) ? + &m_in_deps[m_in_deps_total++] : 0; + PtrData* dst_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + PtrData* src_data = + VAR_TYPE_IS_PTR(m_vars[i].type.src) || + VAR_TYPE_IS_SCALAR(m_vars[i].type.src) && + m_vars[i].flags.is_static ? + m_vars_extra[i].src_data : 0; + + if (m_vars[i].flags.is_noncont_src || + m_vars[i].flags.is_noncont_dst) { + if (!send_noncontiguous_pointer_data( + i, src_data, dst_data, event)) { + return false; + } + } + else if (src_data != 0 && src_data->cpu_buf != 0) { + res = COI::BufferCopy( + dst_data->mic_buf, + src_data->cpu_buf, + m_vars[i].mic_offset - dst_data->alloc_disp + + m_vars[i].offset + m_vars[i].disp, + m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + char *base = offload_get_src_base(m_vars[i].ptr, + m_vars[i].type.src); + res = COI::BufferWrite( + dst_data->mic_buf, + m_vars[i].mic_offset - dst_data->alloc_disp + + m_vars[i].offset + m_vars[i].disp, + base + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_write, res); + } + } + ptr_sent += m_vars[i].size; + } + break; + + case c_string_ptr: + case c_data_ptr: + case c_cean_var_ptr: + case c_dv_ptr: + if (m_vars[i].direction.in && m_vars[i].size > 0) { + COIEVENT *event = + (is_async || + m_vars[i].size >= __offload_use_async_buffer_write) ? + &m_in_deps[m_in_deps_total++] : 0; + PtrData* dst_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + PtrData* src_data = + VAR_TYPE_IS_PTR(m_vars[i].type.src) || + VAR_TYPE_IS_SCALAR(m_vars[i].type.src) && + m_vars[i].flags.is_static ? + m_vars_extra[i].src_data : 0; + + if (m_vars[i].flags.is_noncont_src || + m_vars[i].flags.is_noncont_dst) { + send_noncontiguous_pointer_data( + i, src_data, dst_data, event); + } + else if (src_data != 0 && src_data->cpu_buf != 0) { + res = COI::BufferCopy( + dst_data->mic_buf, + src_data->cpu_buf, + m_vars[i].mic_offset - dst_data->alloc_disp + + m_vars[i].offset + m_vars[i].disp, + m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + char *base = offload_get_src_base(m_vars[i].ptr, + m_vars[i].type.src); + res = COI::BufferWrite( + dst_data->mic_buf, + m_vars[i].mic_offset - dst_data->alloc_disp + + m_vars[i].offset + m_vars[i].disp, + base + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_write, res); + } + } + + ptr_sent += m_vars[i].size; + } + break; + + case c_dv_data: + case c_dv_ptr_data: + if (m_vars[i].direction.in && + m_vars[i].size > 0) { + PtrData *ptr_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + PtrData* src_data = m_vars_extra[i].src_data; + + COIEVENT *event = + (is_async || + m_vars[i].size >= __offload_use_async_buffer_write) ? + &m_in_deps[m_in_deps_total++] : 0; + + if (m_vars[i].flags.is_noncont_src || + m_vars[i].flags.is_noncont_dst) { + send_noncontiguous_pointer_data( + i, src_data, ptr_data, event); + } + else if (src_data && src_data->cpu_buf != 0) { + res = COI::BufferCopy( + ptr_data->mic_buf, + src_data->cpu_buf, + m_vars[i].offset + ptr_data->mic_offset - + ptr_data->alloc_disp + + m_vars[i].disp, + m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + char *base = offload_get_src_base(m_vars[i].ptr, + m_vars[i].type.src); + res = COI::BufferWrite( + ptr_data->mic_buf, + ptr_data->mic_offset - ptr_data->alloc_disp + + m_vars[i].offset + m_vars[i].disp, + base + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_write, res); + } + } + ptr_sent += m_vars[i].size; + } + break; + + case c_dv_data_slice: + case c_dv_ptr_data_slice: + if (m_vars[i].direction.in && + m_vars[i].size > 0) { + PtrData *dst_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + PtrData* src_data = + (VAR_TYPE_IS_PTR(m_vars[i].type.src) || + VAR_TYPE_IS_DV_DATA(m_vars[i].type.src) || + VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src) || + VAR_TYPE_IS_SCALAR(m_vars[i].type.src) && + m_vars[i].flags.is_static) ? + m_vars_extra[i].src_data : 0; + COIEVENT *event = + (is_async || + m_vars[i].size >= __offload_use_async_buffer_write) ? + &m_in_deps[m_in_deps_total++] : 0; + if (m_vars[i].flags.is_noncont_src || + m_vars[i].flags.is_noncont_dst) { + send_noncontiguous_pointer_data( + i, src_data, dst_data, event); + } + else if (src_data && src_data->cpu_buf != 0) { + res = COI::BufferCopy( + dst_data->mic_buf, + src_data->cpu_buf, + m_vars[i].offset - dst_data->alloc_disp + + dst_data->mic_offset + + m_vars[i].disp, + m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + char *base = offload_get_src_base(m_vars[i].ptr, + m_vars[i].type.src); + res = COI::BufferWrite( + dst_data->mic_buf, + dst_data->mic_offset - dst_data->alloc_disp + + m_vars[i].offset + m_vars[i].disp, + base + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + 0, 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_write, res); + } + } + + ptr_sent += m_vars[i].size; + } + break; + + default: + break; + } + + // alloc field isn't used at target. + // We can reuse it for offset of array pointers. + if (m_vars_extra[i].is_arr_ptr_el) { + m_vars[i].ptr_arr_offset = m_vars_extra[i].ptr_arr_offset; + } + } + + if (m_status) { + m_status->data_sent += ptr_sent; + } + + OFFLOAD_TIMER_HOST_SDATA(get_timer_data(), ptr_sent); + OFFLOAD_DEBUG_TRACE_1(1, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_sent_pointer_data, + "Total pointer data sent to target: [%lld] bytes\n", + ptr_sent); + + return true; +} + +bool OffloadDescriptor::gather_copyin_data() +{ + OffloadTimer timer(get_timer_data(), c_offload_host_gather_inputs); + + if (m_need_runfunction && m_in_datalen > 0) { + COIMAPINSTANCE map_inst; + char *data; + + // init marshaller + if (m_inout_buf != 0) { + OffloadTimer timer_map(get_timer_data(), + c_offload_host_map_in_data_buffer); + + COIRESULT res = COI::BufferMap(m_inout_buf, 0, m_in_datalen, + COI_MAP_WRITE_ENTIRE_BUFFER, + 0, 0, 0, &map_inst, + reinterpret_cast<void**>(&data)); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_map, res); + } + } + else { + data = (char*) m_func_desc + m_func_desc->data_offset; + } + + // send variable descriptors + memcpy(data, m_vars, m_vars_total * sizeof(VarDesc)); + data += m_vars_total * sizeof(VarDesc); + + // init marshaller + m_in.init_buffer(data, m_in_datalen); + + // Gather copy data into buffer + for (int i = 0; i < m_vars_total; i++) { + bool src_is_for_mic = (m_vars[i].direction.out || + m_vars[i].into == NULL); + PtrData* ptr_data = src_is_for_mic ? + m_vars_extra[i].src_data : + m_vars_extra[i].dst_data; + if (m_vars[i].flags.alloc_disp) { + m_in.send_data(&ptr_data->alloc_disp, + sizeof(ptr_data->alloc_disp)); + } + + // send sink address to the target + if (m_vars[i].flags.sink_addr) { + m_in.send_data(&ptr_data->mic_addr, + sizeof(ptr_data->mic_addr)); + } + + switch (m_vars[i].type.dst) { + case c_data_ptr_array: + break; + case c_data: + case c_void_ptr: + case c_cean_var: + if (m_vars[i].direction.in && + !m_vars[i].flags.is_static_dstn) { + + char *ptr = offload_get_src_base(m_vars[i].ptr, + m_vars[i].type.src); + if (m_vars[i].type.dst == c_cean_var) { + // offset and length are derived from the array + // descriptor + int64_t size = m_vars[i].size; + int64_t disp = m_vars[i].disp; + m_in.send_data(reinterpret_cast<char*>(&size), + sizeof(int64_t)); + m_in.send_data(reinterpret_cast<char*>(&disp), + sizeof(int64_t)); + } + + m_in.send_data(ptr + m_vars_extra[i].cpu_disp, + m_vars[i].size); + } + break; + + case c_dv: + if (m_vars[i].direction.bits || + m_vars[i].alloc_if || + m_vars[i].free_if) { + // send dope vector excluding base + char *ptr = static_cast<char*>(m_vars[i].ptr); + m_in.send_data(ptr + sizeof(uint64_t), + m_vars[i].size - sizeof(uint64_t)); + } + break; + + case c_data_ptr: + // send to target addresses of obsolete + // stacks to be released + if (m_vars[i].flags.is_stack_buf && + !m_vars[i].direction.bits && + m_vars[i].alloc_if && + m_vars[i].size != 0) { + for (PtrDataList::iterator it = + m_destroy_stack.begin(); + it != m_destroy_stack.end(); it++) { + PtrData * ptr_data = *it; + m_in.send_data(&(ptr_data->mic_addr), + sizeof(ptr_data->mic_addr)); + } + } + break; + case c_func_ptr: + if (m_vars[i].direction.in) { + m_in.send_func_ptr(*((const void**) m_vars[i].ptr)); + } + break; + + default: + break; + } + } + + if (m_status) { + m_status->data_sent += m_in.get_tfr_size(); + } + + if (m_func_desc->data_offset == 0) { + OffloadTimer timer_unmap(get_timer_data(), + c_offload_host_unmap_in_data_buffer); + COIRESULT res = COI::BufferUnmap(map_inst, 0, 0, 0); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_unmap, res); + } + } + } + + OFFLOAD_TIMER_HOST_SDATA(get_timer_data(), m_in.get_tfr_size()); + OFFLOAD_DEBUG_TRACE_1(1, + GET_OFFLOAD_NUMBER(get_timer_data()), c_offload_copyin_data, + "Total copyin data sent to target: [%lld] bytes\n", + m_in.get_tfr_size()); + + return true; +} + +bool OffloadDescriptor::compute() +{ + OffloadTimer timer(get_timer_data(), c_offload_host_start_compute); + + if (m_need_runfunction) { + OFFLOAD_DEBUG_TRACE_1(2, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_compute, "Compute task on MIC\n"); + + void* misc = m_func_desc; + int misc_len = m_func_desc_size; + void* ret = 0; + int ret_len = 0; + + if (m_func_desc->data_offset != 0) { + misc_len += m_in_datalen; + + if (m_out_datalen > 0) { + ret = (char*) m_func_desc + m_func_desc->data_offset; + ret_len = m_out_datalen; + } + } + + // dispatch task + COIRESULT res; + COIEVENT event; + res = m_device.compute(m_compute_buffers, + misc, misc_len, + ret, ret_len, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + &event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_pipeline_run_func, res); + } + + m_in_deps_total = 1; + m_in_deps[0] = event; + } + + return true; +} + +// recieve pointer data if source or destination or both of them are +// noncontiguous. There is guarantee that length of destination enough for +// transfered data. +bool OffloadDescriptor::recieve_noncontiguous_pointer_data( + int i, + char* base, + COIBUFFER dst_buf, + COIEVENT *event +) +{ + int64_t offset_src, offset_dst; + int64_t length_src, length_dst; + int64_t length_src_cur, length_dst_cur; + int64_t recieve_size, data_recieved = 0; + COIRESULT res; + bool dst_is_empty = true; + bool src_is_empty = true; + + // Set length_src and length_dst + length_src = (m_vars_extra[i].read_rng_src) ? + m_vars_extra[i].read_rng_src->range_size : m_vars[i].size; + length_dst = !m_vars[i].into ? length_src : + (m_vars_extra[i].read_rng_dst) ? + m_vars_extra[i].read_rng_dst->range_size : m_vars[i].size; + recieve_size = (length_src < length_dst) ? length_src : length_dst; + + // consequently get contiguous ranges, + // define corresponded destination offset and recieve data + do { + // get sorce offset + if (src_is_empty) { + if (m_vars_extra[i].read_rng_src) { + if (!get_next_range(m_vars_extra[i].read_rng_src, + &offset_src)) { + // source ranges are over - nothing to send + break; + } + } + else if (data_recieved == 0) { + offset_src = 0; + } + else { + break; + } + length_src_cur = length_src; + } + else { + // if source is contiguous or its contiguous range is greater + // than destination one + offset_src += recieve_size; + } + length_src_cur -= recieve_size; + src_is_empty = length_src_cur == 0; + + // get destination offset + if (dst_is_empty) { + if (m_vars[i].into) { + if (m_vars_extra[i].read_rng_dst) { + if (!get_next_range(m_vars_extra[i].read_rng_dst, + &offset_dst)) { + // destination ranges are over + LIBOFFLOAD_ERROR(c_destination_is_over); + return false; + } + } + // destination is contiguous. + else { + offset_dst = m_vars_extra[i].cpu_disp; + } + length_dst_cur = length_dst; + } + // same as source + else { + offset_dst = offset_src; + length_dst_cur = length_src; + } + } + else { + // if destination is contiguous or its contiguous range is greater + // than source one + offset_dst += recieve_size; + } + length_dst_cur -= recieve_size; + dst_is_empty = length_dst_cur == 0; + + if (dst_buf != 0) { + res = COI::BufferCopy( + dst_buf, + m_vars_extra[i].src_data->mic_buf, + m_vars_extra[i].cpu_offset + offset_dst, + m_vars[i].offset + offset_src + + m_vars[i].mic_offset - + m_vars_extra[i].src_data->alloc_disp, + recieve_size, + COI_COPY_UNSPECIFIED, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + res = COI::BufferRead( + m_vars_extra[i].src_data->mic_buf, + m_vars[i].offset + offset_src + + m_vars[i].mic_offset - + m_vars_extra[i].src_data->alloc_disp, + base + offset_dst, + recieve_size, + COI_COPY_UNSPECIFIED, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_read, res); + } + } + data_recieved += recieve_size; + } + while (true); + return true; +} + +bool OffloadDescriptor::receive_pointer_data(bool is_async) +{ + OffloadTimer timer(get_timer_data(), c_offload_host_start_buffers_reads); + + uint64_t ptr_received = 0; + COIRESULT res; + + for (int i = 0; i < m_vars_total; i++) { + switch (m_vars[i].type.src) { + case c_data_ptr_array: + break; + case c_data: + case c_void_ptr: + case c_cean_var: + if (m_vars[i].direction.out && + m_vars[i].flags.is_static) { + COIEVENT *event = + (is_async || + m_in_deps_total > 0 || + m_vars[i].size >= __offload_use_async_buffer_read) ? + &m_out_deps[m_out_deps_total++] : 0; + PtrData *ptr_data = NULL; + COIBUFFER dst_buf = NULL; // buffer at host + char *base; + + if (VAR_TYPE_IS_PTR(m_vars[i].type.dst)) { + ptr_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + } + else if (VAR_TYPE_IS_SCALAR(m_vars[i].type.dst)) { + if (m_vars[i].flags.is_static_dstn) { + ptr_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + } + } + dst_buf = ptr_data ? ptr_data->cpu_buf : NULL; + if (dst_buf == NULL) { + base = offload_get_src_base( + m_vars[i].into ? + static_cast<char*>(m_vars[i].into) : + static_cast<char*>(m_vars[i].ptr), + m_vars[i].type.dst); + } + + if (m_vars[i].flags.is_noncont_src || + m_vars[i].flags.is_noncont_dst) { + recieve_noncontiguous_pointer_data( + i, base, dst_buf, event); + } + else if (dst_buf != 0) { + res = COI::BufferCopy( + dst_buf, + m_vars_extra[i].src_data->mic_buf, + m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp, + m_vars[i].offset + m_vars[i].disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + res = COI::BufferRead( + m_vars_extra[i].src_data->mic_buf, + m_vars[i].offset + m_vars[i].disp, + base + m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_read, res); + } + } + ptr_received += m_vars[i].size; + } + break; + + case c_string_ptr: + case c_data_ptr: + case c_cean_var_ptr: + case c_dv_data: + case c_dv_ptr_data: + case c_dv_data_slice: + case c_dv_ptr_data_slice: + case c_dv_ptr: { + COIBUFFER dst_buf = NULL; // buffer on host + if (m_vars[i].direction.out && m_vars[i].size > 0) { + COIEVENT *event = + (is_async || + m_in_deps_total > 0 || + m_vars[i].size >= __offload_use_async_buffer_read) ? + &m_out_deps[m_out_deps_total++] : 0; + + uint64_t dst_offset = 0; + char *base = static_cast<char*>(m_vars[i].ptr); + + if (VAR_TYPE_IS_PTR(m_vars[i].type.dst)) { + PtrData *ptr_data = m_vars[i].into ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + dst_buf = ptr_data ? ptr_data->cpu_buf : NULL; + if (dst_buf == NULL) { + base = m_vars[i].into ? + *static_cast<char**>(m_vars[i].into) : + *static_cast<char**>(m_vars[i].ptr); + } + dst_offset = m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp; + } + else if (VAR_TYPE_IS_SCALAR(m_vars[i].type.dst)) { + if (m_vars[i].flags.is_static_dstn) { + dst_buf = m_vars[i].into ? + m_vars_extra[i].dst_data->cpu_buf : + m_vars_extra[i].src_data->cpu_buf; + } + if (dst_buf == NULL) { + base = offload_get_src_base( + m_vars[i].into ? + static_cast<char*>(m_vars[i].into) : + static_cast<char*>(m_vars[i].ptr), + m_vars[i].type.dst); + } + dst_offset = m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp; + } + else if (VAR_TYPE_IS_DV_DATA(m_vars[i].type.dst) || + VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.dst)) { + PtrData *ptr_data = m_vars[i].into != 0 ? + m_vars_extra[i].dst_data : + m_vars_extra[i].src_data; + dst_buf = ptr_data != 0 ? ptr_data->cpu_buf : 0; + if (dst_buf == NULL) { + base = offload_get_src_base( + m_vars[i].into ? + static_cast<char*>(m_vars[i].into) : + static_cast<char*>(m_vars[i].ptr), + m_vars[i].type.dst); + + } + dst_offset = m_vars_extra[i].cpu_offset + + m_vars_extra[i].cpu_disp; + } + + if (m_vars[i].flags.is_noncont_src || + m_vars[i].flags.is_noncont_dst) { + recieve_noncontiguous_pointer_data( + i, base, dst_buf, event); + } + else if (dst_buf != 0) { + res = COI::BufferCopy( + dst_buf, + m_vars_extra[i].src_data->mic_buf, + dst_offset, + m_vars[i].offset + m_vars[i].disp + + m_vars[i].mic_offset - + m_vars_extra[i].src_data->alloc_disp, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_copy, res); + } + } + else { + res = COI::BufferRead( + m_vars_extra[i].src_data->mic_buf, + m_vars[i].offset + m_vars[i].disp + + m_vars[i].mic_offset - + m_vars_extra[i].src_data->alloc_disp, + base + dst_offset, + m_vars[i].size, + COI_COPY_UNSPECIFIED, + m_in_deps_total, + m_in_deps_total > 0 ? m_in_deps : 0, + event); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_read, res); + } + } + ptr_received += m_vars[i].size; + } + break; + } + + default: + break; + } + + // destroy buffers for obsolete stacks + if (m_destroy_stack.size() != 0) { + for (PtrDataList::iterator it = m_destroy_stack.begin(); + it != m_destroy_stack.end(); it++) { + PtrData *ptr_data = *it; + m_destroy_buffers.push_back(ptr_data->mic_buf); + OFFLOAD_TRACE(3, "Removing stack buffer with addr %p\n", + ptr_data->mic_addr); + } + m_destroy_stack.clear(); + } + if (m_vars[i].free_if) { + // remove association for automatic variables + if (m_is_openmp && !m_vars[i].flags.is_static && + (m_vars[i].type.src == c_data || + m_vars[i].type.src == c_void_ptr || + m_vars[i].type.src == c_cean_var)) { + AutoData *auto_data = m_vars_extra[i].auto_data; + if (auto_data != 0 && auto_data->remove_reference() == 0) { + m_device.remove_auto_data(auto_data->cpu_addr.start()); + } + } + + // destroy buffers + if (m_vars[i].direction.out || m_vars[i].into == NULL) { + if (!VAR_TYPE_IS_PTR(m_vars[i].type.src) && + !VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src) && + !VAR_TYPE_IS_DV_DATA(m_vars[i].type.src)) { + continue; + } + + PtrData *ptr_data = m_vars_extra[i].src_data; + if (ptr_data->remove_reference() == 0) { + // destroy buffers + if (ptr_data->cpu_buf != 0) { + m_destroy_buffers.push_back(ptr_data->cpu_buf); + } + if (ptr_data->mic_buf != 0) { + m_destroy_buffers.push_back(ptr_data->mic_buf); + } + OFFLOAD_TRACE(3, "Removing association for addr %p\n", + ptr_data->cpu_addr.start()); + + // remove association from map + m_device.remove_ptr_data(ptr_data->cpu_addr.start()); + } + } + else if (VAR_TYPE_IS_PTR(m_vars[i].type.dst) || + VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.dst) || + VAR_TYPE_IS_DV_DATA(m_vars[i].type.dst)) { + PtrData *ptr_data = m_vars_extra[i].dst_data; + if (ptr_data->remove_reference() == 0) { + // destroy buffers + if (ptr_data->cpu_buf != 0) { + m_destroy_buffers.push_back(ptr_data->cpu_buf); + } + if (ptr_data->mic_buf != 0) { + m_destroy_buffers.push_back(ptr_data->mic_buf); + } + OFFLOAD_TRACE(3, "Removing association for addr %p\n", + ptr_data->cpu_addr.start()); + + // remove association from map + m_device.remove_ptr_data(ptr_data->cpu_addr.start()); + } + } + } + } + + if (m_status) { + m_status->data_received += ptr_received; + } + + OFFLOAD_TIMER_HOST_RDATA(get_timer_data(), ptr_received); + OFFLOAD_DEBUG_TRACE_1(1, GET_OFFLOAD_NUMBER(get_timer_data()), + c_offload_received_pointer_data, + "Total pointer data received from target: [%lld] bytes\n", + ptr_received); + + return true; +} + +bool OffloadDescriptor::scatter_copyout_data() +{ + OffloadTimer timer(get_timer_data(), c_offload_host_scatter_outputs); + + if (m_need_runfunction && m_out_datalen > 0) { + + // total size that need to be transferred from target to host + COIMAPINSTANCE map_inst; + COIRESULT res; + char *data; + + // output data buffer + if (m_func_desc->data_offset == 0) { + OffloadTimer timer_map(get_timer_data(), + c_offload_host_map_out_data_buffer); + + COIRESULT res = COI::BufferMap(m_inout_buf, 0, m_out_datalen, + COI_MAP_READ_ONLY, 0, 0, 0, + &map_inst, + reinterpret_cast<void**>(&data)); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_map, res); + } + } + else { + data = (char*) m_func_desc + m_func_desc->data_offset; + } + + // get timing data + OFFLOAD_TIMER_TARGET_DATA(get_timer_data(), data); + data += OFFLOAD_TIMER_DATALEN(); + + // initialize output marshaller + m_out.init_buffer(data, m_out_datalen); + + for (int i = 0; i < m_vars_total; i++) { + switch (m_vars[i].type.src) { + case c_data_ptr_array: + break; + case c_data: + case c_void_ptr: + case c_cean_var: + if (m_vars[i].direction.out && + !m_vars[i].flags.is_static) { + + if (m_vars[i].into) { + char *ptr = offload_get_src_base( + static_cast<char*>(m_vars[i].into), + m_vars[i].type.dst); + m_out.receive_data(ptr + m_vars_extra[i].cpu_disp, + m_vars[i].size); + } + else { + m_out.receive_data( + static_cast<char*>(m_vars[i].ptr) + + m_vars_extra[i].cpu_disp, + m_vars[i].size); + } + } + break; + + case c_func_ptr: + if (m_vars[i].direction.out) { + m_out.receive_func_ptr((const void**) m_vars[i].ptr); + } + break; + + default: + break; + } + } + + if (m_status) { + m_status->data_received += m_out.get_tfr_size(); + } + + if (m_func_desc->data_offset == 0) { + OffloadTimer timer_unmap(get_timer_data(), + c_offload_host_unmap_out_data_buffer); + + COIRESULT res = COI::BufferUnmap(map_inst, 0, 0, 0); + if (res != COI_SUCCESS) { + if (m_status != 0) { + m_status->result = translate_coi_error(res); + return false; + } + report_coi_error(c_buf_unmap, res); + } + } + } + + OFFLOAD_TIMER_HOST_RDATA(get_timer_data(), m_out.get_tfr_size()); + OFFLOAD_TRACE(1, "Total copyout data received from target: [%lld] bytes\n", + m_out.get_tfr_size()); + + return true; +} + +void get_arr_desc_numbers( + const arr_desc *ap, + int64_t el_size, + int64_t &offset, + int64_t &size, + int &el_number, + CeanReadRanges* &ptr_ranges +) +{ + if (is_arr_desc_contiguous(ap)) { + ptr_ranges = NULL; + __arr_data_offset_and_length(ap, offset, size); + el_number = size / el_size; + } + else { + ptr_ranges = init_read_ranges_arr_desc(ap); + el_number = (ptr_ranges->range_size / el_size) * + ptr_ranges->range_max_number; + size = ptr_ranges->range_size; + } +} + +arr_desc * make_arr_desc( + void* ptr_val, + int64_t extent_start_val, + int64_t extent_elements_val, + int64_t size +) +{ + arr_desc *res; + res = (arr_desc *)malloc(sizeof(arr_desc)); + if (res == NULL) + LIBOFFLOAD_ERROR(c_malloc); + res->base = reinterpret_cast<int64_t>(ptr_val); + res->rank = 1; + res->dim[0].size = size; + res->dim[0].lindex = 0; + res->dim[0].lower = extent_start_val; + res->dim[0].upper = extent_elements_val + extent_start_val - 1; + res->dim[0].stride = 1; + return res; +} + +bool OffloadDescriptor::gen_var_descs_for_pointer_array(int i) +{ + int pointers_number; + int tmp_val; + int new_index = m_vars_total; + const arr_desc *ap; + const VarDesc3 *vd3 = static_cast<const VarDesc3*>(m_vars[i].ptr); + int flags = vd3->array_fields; + bool src_is_for_mic = (m_vars[i].direction.out || + m_vars[i].into == NULL); + + ReadArrElements<void *> ptr; + ReadArrElements<void *> into; + ReadArrElements<int64_t> ext_start; + ReadArrElements<int64_t> ext_elements; + ReadArrElements<int64_t> align; + ReadArrElements<int64_t> alloc_if; + ReadArrElements<int64_t> free_if; + ReadArrElements<int64_t> into_start; + ReadArrElements<int64_t> into_elem; + ReadArrElements<int64_t> alloc_start; + ReadArrElements<int64_t> alloc_elem; + + + ap = static_cast<const arr_desc*>(vd3->ptr_array); + + // "pointers_number" for total number of transfered pointers. + // For each of them we create new var_desc and put it at the bottom + // of the var_desc's array + get_arr_desc_numbers(ap, sizeof(void *), ptr.offset, ptr.size, + pointers_number, ptr.ranges); + ptr.base = reinterpret_cast<char*>(ap->base); + + // 2. prepare memory for new var_descs + m_vars_total += pointers_number; + m_vars = (VarDesc*)realloc(m_vars, m_vars_total * sizeof(VarDesc)); + if (m_vars == NULL) + LIBOFFLOAD_ERROR(c_malloc); + m_vars_extra = + (VarExtra*)realloc(m_vars_extra, m_vars_total * sizeof(VarExtra)); + if (m_vars_extra == NULL) + LIBOFFLOAD_ERROR(c_malloc); + m_in_deps = + (COIEVENT*)realloc(m_in_deps, sizeof(COIEVENT) * (m_vars_total + 1)); + if (m_in_deps == NULL) + LIBOFFLOAD_ERROR(c_malloc); + m_out_deps = + (COIEVENT*)realloc(m_out_deps, sizeof(COIEVENT) * m_vars_total); + if (m_out_deps == NULL) + LIBOFFLOAD_ERROR(c_malloc); + + // 3. Prepare for reading new var_desc's fields + // EXTENT START + if ((flags & (1<<flag_extent_start_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->extent_start); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, ext_start.offset, + ext_start.size, tmp_val, ext_start.ranges); + ext_start.base = reinterpret_cast<char*>(ap->base); + ext_start.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "extent start"); + return false; + } + } + else if ((flags & (1<<flag_extent_start_is_scalar)) != 0) { + ext_start.val = (int64_t)vd3->extent_start; + } + else { + ext_start.val = 0; + } + + // EXTENT ELEMENTS NUMBER + if ((flags & (1<<flag_extent_elements_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->extent_elements); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, + ext_elements.offset, ext_elements.size, + tmp_val, ext_elements.ranges); + ext_elements.base = reinterpret_cast<char*>(ap->base); + ext_elements.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "extent elements"); + return false; + } + } + else if ((flags & (1<<flag_extent_elements_is_scalar)) != 0) { + ext_elements.val = (int64_t)vd3->extent_elements; + } + else { + ext_elements.val = m_vars[i].count; + } + + // ALLOC_IF + if ((flags & (1<<flag_alloc_if_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->alloc_if_array); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, alloc_if.offset, + alloc_if.size, tmp_val, alloc_if.ranges); + alloc_if.base = reinterpret_cast<char*>(ap->base); + alloc_if.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "alloc_if"); + return false; + } + } + else { + alloc_if.val = m_vars[i].count; + } + + // FREE_IF + if ((flags & (1<<flag_free_if_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->free_if_array); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, free_if.offset, + free_if.size, tmp_val, free_if.ranges); + free_if.base = reinterpret_cast<char*>(ap->base); + free_if.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "free_if"); + return false; + } + } + else { + free_if.val = m_vars[i].count; + } + + // ALIGN + + if ((flags & (1<<flag_align_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->align_array); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, align.offset, + align.size, tmp_val, align.ranges); + align.base = reinterpret_cast<char*>(ap->base); + align.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "align"); + return false; + } + } + else { + align.val = m_vars[i].align; + } + + // 3.1 INTO + + if (m_vars[i].into) { + ap = static_cast<const arr_desc*>(m_vars[i].into); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, into.offset, + into.size, tmp_val, into.ranges); + into.base = reinterpret_cast<char*>(ap->base); + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "into"); + return false; + } + } + + // 3.2 INTO_START + + if ((flags & (1<<flag_into_start_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->into_start); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, into_start.offset, + into_start.size, tmp_val, into_start.ranges); + into_start.base = reinterpret_cast<char*>(ap->base); + into_start.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "into_extent start"); + return false; + } + } + else if ((flags & (1<<flag_into_start_is_scalar)) != 0) { + into_start.val = (int64_t)vd3->into_start; + } + else { + into_start.val = 0; + } + + // 3.3 INTO_ELEMENTS + + if ((flags & (1<<flag_into_elements_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->into_elements); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, into_elem.offset, + into_elem.size, tmp_val, into_elem.ranges); + into_elem.base = reinterpret_cast<char*>(ap->base); + into_elem.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "into_extent elements"); + return false; + } + } + else if ((flags & (1<<flag_into_elements_is_scalar)) != 0) { + into_elem.val = (int64_t)vd3->into_elements; + } + else { + into_elem.val = m_vars[i].count; + } + + // alloc_start + + if ((flags & (1<<flag_alloc_start_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->alloc_start); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, + alloc_start.offset, alloc_start.size, tmp_val, + alloc_start.ranges); + alloc_start.base = reinterpret_cast<char*>(ap->base); + alloc_start.el_size = ap->dim[ap->rank - 1].size; + + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "alloc_extent start"); + return false; + } + } + else if ((flags & (1<<flag_alloc_start_is_scalar)) != 0) { + alloc_start.val = (int64_t)vd3->alloc_start; + } + else { + alloc_start.val = 0; + } + + // alloc_elem + + if ((flags & (1<<flag_alloc_elements_is_array)) != 0) { + ap = static_cast<const arr_desc*>(vd3->alloc_elements); + get_arr_desc_numbers(ap, ap->dim[ap->rank - 1].size, alloc_elem.offset, + alloc_elem.size, tmp_val, alloc_elem.ranges); + alloc_elem.base = reinterpret_cast<char*>(ap->base); + alloc_elem.el_size = ap->dim[ap->rank - 1].size; + if (tmp_val < pointers_number) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, + "alloc_extent elements"); + return false; + } + } + else if ((flags & (1<<flag_alloc_elements_is_scalar)) != 0) { + alloc_elem.val = (int64_t)vd3->alloc_elements; + } + else { + alloc_elem.val = 0; + } + + for (int k = 0; k < pointers_number; k++) { + int type = flags & 0x3f; + int type_src, type_dst; + // Get new values + // type_src, type_dst + type_src = type_dst = (type == c_data_ptr_array) ? + c_data_ptr : (type == c_func_ptr_array) ? + c_func_ptr : (type == c_void_ptr_array) ? + c_void_ptr : (type == c_string_ptr_array) ? + c_string_ptr : 0; + + // Get ptr val + if (!ptr.read_next(true)) { + break; + } + else { + ptr.val = (void*)(ptr.base + ptr.offset); + } + + // !!! If we got error at phase of reading - it's an internal + // !!! error, as we must detect mismatch before + + // Get into val + if (m_vars[i].into) { + if (!into.read_next(true)) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "into"); + LIBOFFLOAD_ABORT; + } + else { + into.val = (void*)(into.base + into.offset); + } + } + + // Get other components of the clause + if (!ext_start.read_next(flags & (1<<flag_extent_start_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "extent start"); + LIBOFFLOAD_ABORT; + } + if (!ext_elements.read_next( + flags & (1<<flag_extent_elements_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "extent elements"); + LIBOFFLOAD_ABORT; + } + if (!alloc_if.read_next(flags & (1<<flag_alloc_if_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "alloc_if"); + LIBOFFLOAD_ABORT; + } + if (!free_if.read_next(flags & (1<<flag_free_if_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "free_if"); + LIBOFFLOAD_ABORT; + } + if (!align.read_next(flags & (1<<flag_align_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "align"); + LIBOFFLOAD_ABORT; + } + if (!into_start.read_next(flags & (1<<flag_into_start_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "into_extent start"); + LIBOFFLOAD_ABORT; + } + if (!into_elem.read_next(flags & (1<<flag_into_elements_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "into_extent elements"); + LIBOFFLOAD_ABORT; + } + if (!alloc_start.read_next(flags & (1<<flag_alloc_start_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "alloc_extent start"); + LIBOFFLOAD_ABORT; + } + if (!alloc_elem.read_next( + flags & (1<<flag_alloc_elements_is_array))) { + LIBOFFLOAD_ERROR(c_pointer_array_mismatch, "alloc_extent elements"); + LIBOFFLOAD_ABORT; + } + + m_vars[new_index + k].direction.bits = m_vars[i].direction.bits; + m_vars[new_index + k].alloc_if = alloc_if.val; + m_vars[new_index + k].free_if = free_if.val; + m_vars[new_index + k].align = align.val; + m_vars[new_index + k].mic_offset = 0; + m_vars[new_index + k].flags.bits = m_vars[i].flags.bits; + m_vars[new_index + k].offset = 0; + m_vars[new_index + k].size = m_vars[i].size; + + if (ext_start.val == 0) { + m_vars[new_index + k].count = ext_elements.val; + m_vars[new_index + k].ptr = ptr.val; + if (type_src == c_string_ptr) { + m_vars[new_index + k].size = 0; + } + } + else { + m_vars[new_index + k].count = 0; + m_vars[new_index + k].ptr = + static_cast<void*>(make_arr_desc( + ptr.val, + ext_start.val, + ext_elements.val, + m_vars[i].size)); + + type_src = type_src == c_data_ptr ? c_cean_var_ptr : + c_string_ptr ? c_cean_var_ptr : + type_src; + if (!m_vars[i].into) { + type_dst = type_src; + } + } + + if (m_vars[i].into && into_elem.val != 0) { + m_vars[new_index + k].into = + static_cast<void*>(make_arr_desc( + into.val, + into_start.val, + into_elem.val, + m_vars[i].size)); + type_dst = (type == c_data_ptr_array) ? c_cean_var_ptr : + (type == c_string_ptr_array) ? c_cean_var_ptr : + type_src; + } + else { + m_vars[new_index + k].into = NULL; + } + + if (alloc_elem.val != 0) { + m_vars[new_index + k].alloc = + static_cast<void*>(make_arr_desc( + ptr.val, + alloc_start.val, + alloc_elem.val, + m_vars[i].size)); + } + else { + m_vars[new_index + k].alloc = NULL; + } + + m_vars[new_index + k].type.src = type_src; + m_vars[new_index + k].type.dst = type_dst; + + m_vars_extra[new_index + k].is_arr_ptr_el = 1; + m_vars_extra[new_index + k].ptr_arr_offset = + src_is_for_mic ? ptr.offset : into.offset; + } + // count and alloc fields are useless at target. They can be reused + // for pointer arrays. + m_vars[i].count = pointers_number; + m_vars[i].ptr_arr_offset = new_index; + return true; +} + +static void __offload_fini_library(void) +{ + OFFLOAD_DEBUG_TRACE(2, "Cleanup offload library ...\n"); + if (mic_engines_total > 0) { + delete[] mic_engines; + + if (mic_proxy_fs_root != 0) { + free(mic_proxy_fs_root); + mic_proxy_fs_root = 0; + } + + if (mic_library_path != 0) { + free(mic_library_path); + mic_library_path = 0; + } + + // destroy thread key + thread_key_delete(mic_thread_key); + } + + // unload COI library + if (COI::is_available) { + COI::fini(); + } + + OFFLOAD_DEBUG_TRACE(2, "Cleanup offload library ... done\n"); +} + +static void __offload_init_library_once(void) +{ + COIRESULT res; + uint32_t num_devices; + std::bitset<MIC_ENGINES_MAX> devices; + + prefix = report_get_message_str(c_report_host); + + // initialize trace + const char *env_var = getenv(htrace_envname); + if (env_var != 0 && *env_var != '\0') { + int64_t new_val; + if (__offload_parse_int_string(env_var, new_val)) { + console_enabled = new_val & 0x0f; + } + } + + env_var = getenv(offload_report_envname); + if (env_var != 0 && *env_var != '\0') { + int64_t env_val; + if (__offload_parse_int_string(env_var, env_val)) { + if (env_val == OFFLOAD_REPORT_1 || + env_val == OFFLOAD_REPORT_2 || + env_val == OFFLOAD_REPORT_3) { + offload_report_level = env_val; + } + else { + LIBOFFLOAD_ERROR(c_invalid_env_report_value, + offload_report_envname); + } + } + else { + LIBOFFLOAD_ERROR(c_invalid_env_var_int_value, + offload_report_envname); + } + } + else if (!offload_report_level) { + env_var = getenv(timer_envname); + if (env_var != 0 && *env_var != '\0') { + timer_enabled = atoi(env_var); + } + } + + // initialize COI + if (!COI::init()) { + return; + } + + // get number of devices installed in the system + res = COI::EngineGetCount(COI_ISA_KNC, &num_devices); + if (res != COI_SUCCESS) { + return; + } + + if (num_devices > MIC_ENGINES_MAX) { + num_devices = MIC_ENGINES_MAX; + } + + // fill in the list of devices that can be used for offloading + env_var = getenv("OFFLOAD_DEVICES"); + if (env_var != 0) { + if (strcasecmp(env_var, "none") != 0) { + // value is composed of comma separated physical device indexes + char *buf = strdup(env_var); + char *str, *ptr; + for (str = strtok_r(buf, ",", &ptr); str != 0; + str = strtok_r(0, ",", &ptr)) { + // convert string to an int + int64_t num; + if (!__offload_parse_int_string(str, num)) { + LIBOFFLOAD_ERROR(c_mic_init5); + + // fallback to using all installed devices + devices.reset(); + for (int i = 0; i < num_devices; i++) { + devices.set(i); + } + break; + } + if (num < 0 || num >= num_devices) { + LIBOFFLOAD_ERROR(c_mic_init6, num); + continue; + } + devices.set(num); + } + free(buf); + } + } + else { + // use all available devices + for (int i = 0; i < num_devices; i++) { + COIENGINE engine; + res = COI::EngineGetHandle(COI_ISA_KNC, i, &engine); + if (res == COI_SUCCESS) { + devices.set(i); + } + } + } + + mic_engines_total = devices.count(); + + // no need to continue if there are no devices to offload to + if (mic_engines_total <= 0) { + return; + } + + // initialize indexes for available devices + mic_engines = new Engine[mic_engines_total]; + for (int p_idx = 0, l_idx = 0; p_idx < num_devices; p_idx++) { + if (devices[p_idx]) { + mic_engines[l_idx].set_indexes(l_idx, p_idx); + l_idx++; + } + } + + // library search path for device binaries + env_var = getenv("MIC_LD_LIBRARY_PATH"); + if (env_var != 0) { + mic_library_path = strdup(env_var); + } + + // memory size reserved for COI buffers + env_var = getenv("MIC_BUFFERSIZE"); + if (env_var != 0) { + uint64_t new_size; + if (__offload_parse_size_string(env_var, new_size)) { + mic_buffer_size = new_size; + } + else { + LIBOFFLOAD_ERROR(c_invalid_env_var_value, "MIC_BUFFERSIZE"); + } + } + + // determine stacksize for the pipeline on the device + env_var = getenv("MIC_STACKSIZE"); + if (env_var != 0 && *env_var != '\0') { + uint64_t new_size; + if (__offload_parse_size_string(env_var, new_size) && + (new_size >= 16384) && ((new_size & 4095) == 0)) { + mic_stack_size = new_size; + } + else { + LIBOFFLOAD_ERROR(c_mic_init3); + } + } + + // proxy I/O + env_var = getenv("MIC_PROXY_IO"); + if (env_var != 0 && *env_var != '\0') { + int64_t new_val; + if (__offload_parse_int_string(env_var, new_val)) { + mic_proxy_io = new_val; + } + else { + LIBOFFLOAD_ERROR(c_invalid_env_var_int_value, "MIC_PROXY_IO"); + } + } + env_var = getenv("MIC_PROXY_FS_ROOT"); + if (env_var != 0 && *env_var != '\0') { + mic_proxy_fs_root = strdup(env_var); + } + + // Prepare environment for the target process using the following + // rules + // - If MIC_ENV_PREFIX is set then any environment variable on the + // host which has that prefix are copied to the device without + // the prefix. + // All other host environment variables are ignored. + // - If MIC_ENV_PREFIX is not set or if MIC_ENV_PREFIX="" then host + // environment is duplicated. + env_var = getenv("MIC_ENV_PREFIX"); + if (env_var != 0 && *env_var != '\0') { + mic_env_vars.set_prefix(env_var); + + int len = strlen(env_var); + for (int i = 0; environ[i] != 0; i++) { + if (strncmp(environ[i], env_var, len) == 0 && + strncmp(environ[i], "MIC_LD_LIBRARY_PATH", 19) != 0 && + environ[i][len] != '=') { + mic_env_vars.analyze_env_var(environ[i]); + } + } + } + + // create key for thread data + if (thread_key_create(&mic_thread_key, Engine::destroy_thread_data)) { + LIBOFFLOAD_ERROR(c_mic_init4, errno); + return; + } + + // cpu frequency + cpu_frequency = COI::PerfGetCycleFrequency(); + + env_var = getenv(mic_use_2mb_buffers_envname); + if (env_var != 0 && *env_var != '\0') { + uint64_t new_size; + if (__offload_parse_size_string(env_var, new_size)) { + __offload_use_2mb_buffers = new_size; + } + else { + LIBOFFLOAD_ERROR(c_invalid_env_var_value, + mic_use_2mb_buffers_envname); + } + } + + env_var = getenv(mic_use_async_buffer_write_envname); + if (env_var != 0 && *env_var != '\0') { + uint64_t new_size; + if (__offload_parse_size_string(env_var, new_size)) { + __offload_use_async_buffer_write = new_size; + } + } + + env_var = getenv(mic_use_async_buffer_read_envname); + if (env_var != 0 && *env_var != '\0') { + uint64_t new_size; + if (__offload_parse_size_string(env_var, new_size)) { + __offload_use_async_buffer_read = new_size; + } + } + + // mic initialization type + env_var = getenv(offload_init_envname); + if (env_var != 0 && *env_var != '\0') { + if (strcmp(env_var, "on_offload") == 0) { + __offload_init_type = c_init_on_offload; + } + else if (strcmp(env_var, "on_offload_all") == 0) { + __offload_init_type = c_init_on_offload_all; + } +#ifndef TARGET_WINNT + else if (strcmp(env_var, "on_start") == 0) { + __offload_init_type = c_init_on_start; + } +#endif // TARGET_WINNT + else { + LIBOFFLOAD_ERROR(c_invalid_env_var_value, offload_init_envname); + } + } + + // active wait + env_var = getenv(offload_active_wait_envname); + if (env_var != 0 && *env_var != '\0') { + int64_t new_val; + if (__offload_parse_int_string(env_var, new_val)) { + __offload_active_wait = new_val; + } + else { + LIBOFFLOAD_ERROR(c_invalid_env_var_int_value, + offload_active_wait_envname); + } + } + + // omp device num + env_var = getenv(omp_device_num_envname); + if (env_var != 0 && *env_var != '\0') { + int64_t new_val; + if (__offload_parse_int_string(env_var, new_val) && new_val >= 0) { + __omp_device_num = new_val; + } + else { + LIBOFFLOAD_ERROR(c_omp_invalid_device_num_env, + omp_device_num_envname); + } + } + + // init ORSL + ORSL::init(); +} + +extern int __offload_init_library(void) +{ + // do one time intialization + static OffloadOnceControl ctrl = OFFLOAD_ONCE_CONTROL_INIT; + __offload_run_once(&ctrl, __offload_init_library_once); + + // offload is available if COI is available and the number of devices > 0 + bool is_available = COI::is_available && (mic_engines_total > 0); + + // register pending libraries if there are any + if (is_available && __target_libs) { + mutex_locker_t locker(__target_libs_lock); + + for (TargetImageList::iterator it = __target_libs_list.begin(); + it != __target_libs_list.end(); it++) { + // Register library in COI + COI::ProcessRegisterLibraries(1, &it->data, &it->size, + &it->origin, &it->offset); + + // add lib to all engines + for (int i = 0; i < mic_engines_total; i++) { + mic_engines[i].add_lib(*it); + } + } + + __target_libs = false; + __target_libs_list.clear(); + } + + return is_available; +} + +extern "C" void __offload_register_image(const void *target_image) +{ + const struct Image *image = static_cast<const struct Image*>(target_image); + + // decode image + const char *name = image->data; + const void *data = image->data + strlen(image->data) + 1; + uint64_t size = image->size; + const char *origin = 0; + uint64_t offset = 0; + + // our actions depend on the image type + const Elf64_Ehdr *hdr = static_cast<const Elf64_Ehdr*>(data); + switch (hdr->e_type) { + case ET_EXEC: + // Each offload application is supposed to have only one target + // image representing target executable. + // No thread synchronization is required here as the initialization + // code is always executed in a single thread. + if (__target_exe != 0) { + LIBOFFLOAD_ERROR(c_multiple_target_exes); + exit(1); + } + __target_exe = new TargetImage(name, data, size, origin, offset); + + // Registration code for execs is always called from the context + // of main and thus we can safely call any function here, + // including LoadLibrary API on windows. This is the place where + // we do the offload library initialization. + if (__offload_init_library()) { + // initialize engine if init_type is on_start + if (__offload_init_type == c_init_on_start) { + for (int i = 0; i < mic_engines_total; i++) { + mic_engines[i].init(); + } + } + } + break; + + case ET_DYN: + // Registration code for libraries is called from the DllMain + // context (on windows) and thus we cannot do anything usefull + // here. So we just add it to the list of pending libraries for + // the later use. + __target_libs_lock.lock(); + __target_libs = true; + __target_libs_list.push_back(TargetImage(name, data, size, + origin, offset)); + __target_libs_lock.unlock(); + break; + + default: + // something is definitely wrong, issue an error and exit + LIBOFFLOAD_ERROR(c_unknown_binary_type); + exit(1); + } +} + +extern "C" void __offload_unregister_image(const void *target_image) +{ + // Target image is packed as follows: + // 8 bytes - size of the target binary + // null-terminated string - binary name + // <size> bytes - binary contents + const struct Image { + int64_t size; + char data[]; + } *image = static_cast<const struct Image*>(target_image); + + // decode image + const char *name = image->data; + const void *data = image->data + strlen(image->data) + 1; + + // our actions depend on the image type + const Elf64_Ehdr *hdr = static_cast<const Elf64_Ehdr*>(data); + if (hdr->e_type == ET_EXEC) { + // We are executing exec's desctructors. + // It is time to do a library cleanup. + if (timer_enabled) { + Offload_Timer_Print(); + } + +#ifdef MYO_SUPPORT + __offload_myoFini(); +#endif // MYO_SUPPORT + + __offload_fini_library(); + } +} + +// Runtime trace interface for user programs + +void __offload_console_trace(int level) +{ + console_enabled = level; +} + +// User-visible offload API + +int _Offload_number_of_devices(void) +{ + __offload_init_library(); + return mic_engines_total; +} + +int _Offload_get_device_number(void) +{ + return -1; +} + +int _Offload_get_physical_device_number(void) +{ + return -1; +} + +int _Offload_signaled(int index, void *signal) +{ + __offload_init_library(); + + // check index value + if (index < 0 || mic_engines_total <= 0) { + LIBOFFLOAD_ERROR(c_offload_signaled1, index); + LIBOFFLOAD_ABORT; + } + + // find associated async task + OffloadDescriptor *task = + mic_engines[index % mic_engines_total].find_signal(signal, false); + if (task == 0) { + LIBOFFLOAD_ERROR(c_offload_signaled2, signal); + LIBOFFLOAD_ABORT; + } + + return task->is_signaled(); +} + +void _Offload_report(int val) +{ + if (val == OFFLOAD_REPORT_ON || + val == OFFLOAD_REPORT_OFF) { + offload_report_enabled = val; + } +} + +// IDB support +int __dbg_is_attached = 0; +int __dbg_target_id = -1; +pid_t __dbg_target_so_pid = -1; +char __dbg_target_exe_name[MAX_TARGET_NAME] = {0}; +const int __dbg_api_major_version = 1; +const int __dbg_api_minor_version = 0; + +void __dbg_target_so_loaded() +{ +} +void __dbg_target_so_unloaded() +{ +} |