diff options
Diffstat (limited to 'altera-socfpga/hardware-handoff/hps_hps_0/sequencer.c')
| -rw-r--r-- | altera-socfpga/hardware-handoff/hps_hps_0/sequencer.c | 10782 |
1 files changed, 10782 insertions, 0 deletions
diff --git a/altera-socfpga/hardware-handoff/hps_hps_0/sequencer.c b/altera-socfpga/hardware-handoff/hps_hps_0/sequencer.c new file mode 100644 index 0000000..ab277f0 --- /dev/null +++ b/altera-socfpga/hardware-handoff/hps_hps_0/sequencer.c @@ -0,0 +1,10782 @@ +/* +* Copyright Altera Corporation (C) 2012-2014. All rights reserved +* +* SPDX-License-Identifier: BSD-3-Clause +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are met: +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* * Neither the name of Altera Corporation nor the +* names of its contributors may be used to endorse or promote products +* derived from this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL ALTERA CORPORATION BE LIABLE FOR ANY +* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +#include "sequencer_defines.h" + +#include "alt_types.h" +#include "system.h" +#if HPS_HW +#include "sdram_io.h" +#else +#include "io.h" +#endif +#include "sequencer.h" +#include "tclrpt.h" +#include "sequencer_auto.h" + +#if HHP_HPS_SIMULATION +#include "hps_controller.h" +#endif + + +/****************************************************************************** + ****************************************************************************** + ** NOTE: Special Rules for Globale Variables ** + ** ** + ** All global variables that are explicitly initialized (including ** + ** explicitly initialized to zero), are only initialized once, during ** + ** configuration time, and not again on reset. This means that they ** + ** preserve their current contents across resets, which is needed for some ** + ** special cases involving communication with external modules. In ** + ** addition, this avoids paying the price to have the memory initialized, ** + ** even for zeroed data, provided it is explicitly set to zero in the code, ** + ** and doesn't rely on implicit initialization. ** + ****************************************************************************** + ******************************************************************************/ + +#ifndef ARMCOMPILER +#if ARRIAV +// Temporary workaround to place the initial stack pointer at a safe offset from end +#define STRINGIFY(s) STRINGIFY_STR(s) +#define STRINGIFY_STR(s) #s +asm(".global __alt_stack_pointer"); +asm("__alt_stack_pointer = " STRINGIFY(STACK_POINTER)); +#endif + +#if CYCLONEV +// Temporary workaround to place the initial stack pointer at a safe offset from end +#define STRINGIFY(s) STRINGIFY_STR(s) +#define STRINGIFY_STR(s) #s +asm(".global __alt_stack_pointer"); +asm("__alt_stack_pointer = " STRINGIFY(STACK_POINTER)); +#endif +#endif + +#if ENABLE_PRINTF_LOG +#include <stdio.h> +#include <string.h> + +typedef struct { + alt_u32 v; + alt_u32 p; + alt_u32 d; + alt_u32 ps; +} dqs_pos_t; + +/* +The parameters that were previously here are now supplied by generation, until the new data manager is working. +*/ + +struct { + const char *stage; + + alt_u32 vfifo_idx; + + dqs_pos_t gwrite_pos[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + + dqs_pos_t dqs_enable_left_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH]; + dqs_pos_t dqs_enable_right_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH]; + dqs_pos_t dqs_enable_mid[RW_MGR_MEM_IF_READ_DQS_WIDTH]; + + dqs_pos_t dqs_wlevel_left_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + dqs_pos_t dqs_wlevel_right_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + dqs_pos_t dqs_wlevel_mid[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + + alt_32 dq_read_left_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH][RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 dq_read_right_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH][RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 dq_write_left_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 dq_write_right_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 dm_left_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_NUM_DM_PER_WRITE_GROUP]; + alt_32 dm_right_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_NUM_DM_PER_WRITE_GROUP]; +} bfm_gbl; + +#endif + +#if HPS_HW +#include <sdram.h> +#endif // HPS_HW + +#if BFM_MODE +#include <stdio.h> + +// DPI access function via library +extern long long get_sim_time(void); + +typedef struct { + alt_u32 v; + alt_u32 p; + alt_u32 d; + alt_u32 ps; +} dqs_pos_t; + +/* +The parameters that were previously here are now supplied by generation, until the new data manager is working. +*/ + +struct { + FILE *outfp; + int bfm_skip_guaranteed_write; + int trk_sample_count; + int trk_long_idle_updates; + int lfifo_margin; + const char *stage; + + alt_u32 vfifo_idx; + + dqs_pos_t gwrite_pos[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + + dqs_pos_t dqs_enable_left_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH]; + dqs_pos_t dqs_enable_right_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH]; + dqs_pos_t dqs_enable_mid[RW_MGR_MEM_IF_READ_DQS_WIDTH]; + + dqs_pos_t dqs_wlevel_left_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + dqs_pos_t dqs_wlevel_right_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + dqs_pos_t dqs_wlevel_mid[RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + + alt_32 dq_read_left_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH][RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 dq_read_right_edge[RW_MGR_MEM_IF_READ_DQS_WIDTH][RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 dq_write_left_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_MEM_DQ_PER_WRITE_DQS]; + alt_32 dq_write_right_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_MEM_DQ_PER_WRITE_DQS]; + alt_32 dm_left_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_NUM_DM_PER_WRITE_GROUP]; + alt_32 dm_right_edge[RW_MGR_MEM_IF_WRITE_DQS_WIDTH][RW_MGR_NUM_DM_PER_WRITE_GROUP]; +} bfm_gbl; + + +#endif + +#if ENABLE_TCL_DEBUG +debug_data_t my_debug_data; +#endif + +#define NEWVERSION_RDDESKEW 1 +#define NEWVERSION_WRDESKEW 1 +#define NEWVERSION_GW 1 +#define NEWVERSION_WL 1 +#define NEWVERSION_DQSEN 1 + +// Just to make the debugging code more uniform +#ifndef RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM +#define RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM 0 +#endif + +#if HALF_RATE +#define HALF_RATE_MODE 1 +#else +#define HALF_RATE_MODE 0 +#endif + +#if QUARTER_RATE +#define QUARTER_RATE_MODE 1 +#else +#define QUARTER_RATE_MODE 0 +#endif +#define DELTA_D 1 + +// case:56390 +// VFIFO_CONTROL_WIDTH_PER_DQS is the number of VFIFOs actually instantiated per DQS. This is always one except: +// AV QDRII where it is 2 for x18 and x18w2, and 4 for x36 and x36w2 +// RLDRAMII x36 and x36w2 where it is 2. +// In 12.0sp1 we set this to 4 for all of the special cases above to keep it simple. +// In 12.0sp2 or 12.1 this should get moved to generation and unified with the same constant used in the phy mgr + +#define VFIFO_CONTROL_WIDTH_PER_DQS 1 + +#if ARRIAV + +#if QDRII + #if RW_MGR_MEM_DQ_PER_READ_DQS > 9 + #undef VFIFO_CONTROL_WIDTH_PER_DQS + #define VFIFO_CONTROL_WIDTH_PER_DQS 4 + #endif +#endif // protocol check + +#if RLDRAMII + #if RW_MGR_MEM_DQ_PER_READ_DQS > 9 + #undef VFIFO_CONTROL_WIDTH_PER_DQS + #define VFIFO_CONTROL_WIDTH_PER_DQS 2 + #endif +#endif // protocol check + +#endif // family check + +// In order to reduce ROM size, most of the selectable calibration steps are +// decided at compile time based on the user's calibration mode selection, +// as captured by the STATIC_CALIB_STEPS selection below. +// +// However, to support simulation-time selection of fast simulation mode, where +// we skip everything except the bare minimum, we need a few of the steps to +// be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the +// check, which is based on the rtl-supplied value, or we dynamically compute the +// value to use based on the dynamically-chosen calibration mode + +#if QDRII +#define BTFLD_FMT "%llx" +#else +#define BTFLD_FMT "%lx" +#endif + +#if BFM_MODE // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +// TODO: should make this configurable; could even have it read from config file or env at startup +#define DLEVEL 2 +// space around comma is required for varargs macro to remove comma if args is empty +#define DPRINT(level, fmt, args...) if (DLEVEL >= (level)) printf("[%lld] SEQ.C: " fmt "\n" , get_sim_time(), ## args) +#define IPRINT(fmt, args...) printf("[%lld] SEQ.C: " fmt "\n" , get_sim_time(), ## args) +#define BFM_GBL_SET(field,value) bfm_gbl.field = value +#define BFM_GBL_GET(field) bfm_gbl.field +#define BFM_STAGE(label) BFM_GBL_SET(stage,label) +#define BFM_INC_VFIFO bfm_gbl.vfifo_idx = (bfm_gbl.vfifo_idx + 1) % VFIFO_SIZE +#define COV(label) getpid() /* no-op marker for coverage */ + +#elif ENABLE_PRINTF_LOG // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#define DLEVEL 2 + +void wait_printf_queue() +{ + alt_u32 next_entry; + + while (debug_printf_output->count == PRINTF_READ_BUFFER_FIFO_WORDS || debug_printf_output->slave_lock != 0) + {} + + debug_printf_output->master_lock = 1; + next_entry = (debug_printf_output->head + debug_printf_output->count) % PRINTF_READ_BUFFER_FIFO_WORDS; + strcpy((char*)(&(debug_printf_output->read_buffer[next_entry])), (char*)(debug_printf_output->active_word)); + debug_printf_output->count++; + debug_printf_output->master_lock = 0; +} +#define DPRINT(level, fmt, args...) \ + if (DLEVEL >= (level)) { \ + snprintf((char*)(debug_printf_output->active_word), PRINTF_READ_BUFFER_SIZE*4, "DEBUG:" fmt, ## args); \ + wait_printf_queue(); \ + } +#define IPRINT(fmt, args...) \ + snprintf((char*)(debug_printf_output->active_word), PRINTF_READ_BUFFER_SIZE*4, "INFO:" fmt, ## args); \ + wait_printf_queue(); + +#define BFM_GBL_SET(field,value) bfm_gbl.field = value +#define BFM_GBL_GET(field) bfm_gbl.field +#define BFM_STAGE(label) BFM_GBL_SET(stage,label) +#define BFM_INC_VFIFO bfm_gbl.vfifo_idx = (bfm_gbl.vfifo_idx + 1) % VFIFO_SIZE +#define COV(label) + +#elif HPS_HW // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +// For HPS running on actual hardware + +#define DLEVEL 0 +#ifdef HPS_HW_SERIAL_SUPPORT +// space around comma is required for varargs macro to remove comma if args is empty +#define DPRINT(level, fmt, args...) if (DLEVEL >= (level)) printf("SEQ.C: " fmt "\n" , ## args) +#define IPRINT(fmt, args...) printf("SEQ.C: " fmt "\n" , ## args) +#if RUNTIME_CAL_REPORT +#define RPRINT(fmt, args...) printf("SEQ.C: " fmt "\n" , ## args) +#endif +#else +#define DPRINT(level, fmt, args...) +#define IPRINT(fmt, args...) +#endif +#define BFM_GBL_SET(field,value) +#define BFM_GBL_GET(field) ((long unsigned int)0) +#define BFM_STAGE(stage) +#define BFM_INC_VFIFO +#define COV(label) + +#else // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~----------------------------------- + +// Default mode +#define DPRINT(level, fmt, args...) +#define IPRINT(fmt, args...) +#define BFM_GBL_SET(field,value) +#define BFM_GBL_GET(field) 0 +#define BFM_STAGE(stage) +#define BFM_INC_VFIFO +#define COV(label) + +#endif // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~---------------------------------- + +#if BFM_MODE +#define TRACE_FUNC(fmt, args...) DPRINT(1, "%s[%ld]: " fmt, __func__, __LINE__ , ## args) +#else +#define TRACE_FUNC(fmt, args...) DPRINT(1, "%s[%d]: " fmt, __func__, __LINE__ , ## args) +#endif + +#if BFM_MODE +// In BFM mode, we do full calibration as for real-rtl +#define DYNAMIC_CALIB_STEPS STATIC_CALIB_STEPS +#else +#define DYNAMIC_CALIB_STEPS (dyn_calib_steps) +#endif + +#if STATIC_SIM_FILESET +#define STATIC_IN_RTL_SIM CALIB_IN_RTL_SIM +#else +#define STATIC_IN_RTL_SIM 0 +#endif + +#if STATIC_SKIP_MEM_INIT +#define STATIC_SKIP_DELAY_LOOPS CALIB_SKIP_DELAY_LOOPS +#else +#define STATIC_SKIP_DELAY_LOOPS 0 +#endif + +#if STATIC_FULL_CALIBRATION +#define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | STATIC_SKIP_DELAY_LOOPS) +#elif STATIC_QUICK_CALIBRATION +#define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | CALIB_SKIP_WRITES | CALIB_SKIP_DELAY_SWEEPS | CALIB_SKIP_ALL_BITS_CHK | STATIC_SKIP_DELAY_LOOPS) +#elif STATIC_SKIP_CALIBRATION +#define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | CALIB_SKIP_WRITES | CALIB_SKIP_WLEVEL | CALIB_SKIP_LFIFO | CALIB_SKIP_VFIFO | CALIB_SKIP_DELAY_SWEEPS | CALIB_SKIP_ALL_BITS_CHK | STATIC_SKIP_DELAY_LOOPS) +#else +#undef STATIC_CALIB_STEPS +// This should force an error +#endif + +// calibration steps requested by the rtl +alt_u16 dyn_calib_steps = 0; + +// To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option +// instead of static, we use boolean logic to select between +// non-skip and skip values +// +// The mask is set to include all bits when not-skipping, but is +// zero when skipping + +alt_u16 skip_delay_mask = 0; // mask off bits when skipping/not-skipping + +#define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \ + ((non_skip_value) & skip_delay_mask) + + +// TODO: The skip group strategy is completely missing + +gbl_t *gbl = 0; +param_t *param = 0; + +alt_u32 curr_shadow_reg = 0; + +#if ENABLE_DELAY_CHAIN_WRITE +alt_u32 vfifo_settings[RW_MGR_MEM_IF_READ_DQS_WIDTH]; +#endif // ENABLE_DELAY_CHAIN_WRITE + +#if ENABLE_NON_DESTRUCTIVE_CALIB +// Technically, the use of these variables could be separated from ENABLE_NON_DESTRUCTIVE_CALIB +// but currently they are part of a single feature which is not fully validated, so we're keeping +// them together + +// These variables can be modified by external rtl modules, and hence are "volatile" +volatile alt_u32 no_init = 0; +volatile alt_u32 abort_cal = 0; +#endif + +alt_u32 rw_mgr_mem_calibrate_write_test (alt_u32 rank_bgn, alt_u32 write_group, alt_u32 use_dm, alt_u32 all_correct, t_btfld *bit_chk, alt_u32 all_ranks); + +#if ENABLE_BRINGUP_DEBUGGING + +#define DI_BUFFER_DEBUG_SIZE 64 + +alt_u8 di_buf_gbl[DI_BUFFER_DEBUG_SIZE*4] = {0}; + +void load_di_buf_gbl(void) +{ + int i; + int j; + + for (i = 0; i < DI_BUFFER_DEBUG_SIZE; i++) { + alt_u32 val = IORD_32DIRECT(RW_MGR_DI_BASE + i*4, 0); + for (j = 0; j < 4; j++) { + alt_u8 byte = (val >> (8*j)) & 0xff; + di_buf_gbl[i*4 + j] = byte; + } + } +} + +#endif /* ENABLE_BRINGUP_DEBUGGING */ + + +#if ENABLE_DQSEN_SWEEP +void init_di_buffer(void) +{ + alt_u32 i; + + debug_data->di_report.flags = 0; + debug_data->di_report.cur_samples = 0; + + for (i = 0; i < NUM_DI_SAMPLE; i++) + { + debug_data->di_report.di_buffer[i].bit_chk = 0; + debug_data->di_report.di_buffer[i].delay = 0; + debug_data->di_report.di_buffer[i].d = 0; + debug_data->di_report.di_buffer[i].v = 0; + debug_data->di_report.di_buffer[i].p = 0; + debug_data->di_report.di_buffer[i].di_buffer_0a = 0; + debug_data->di_report.di_buffer[i].di_buffer_0b = 0; + debug_data->di_report.di_buffer[i].di_buffer_1a = 0; + debug_data->di_report.di_buffer[i].di_buffer_1b = 0; + debug_data->di_report.di_buffer[i].di_buffer_2a = 0; + debug_data->di_report.di_buffer[i].di_buffer_2b = 0; + debug_data->di_report.di_buffer[i].di_buffer_3a = 0; + debug_data->di_report.di_buffer[i].di_buffer_3b = 0; + debug_data->di_report.di_buffer[i].di_buffer_4a = 0; + debug_data->di_report.di_buffer[i].di_buffer_4b = 0; + } +} + +inline void flag_di_buffer_ready() +{ + debug_data->di_report.flags |= DI_REPORT_FLAGS_READY; +} + +inline void flag_di_buffer_done() +{ + debug_data->di_report.flags |= DI_REPORT_FLAGS_READY; + debug_data->di_report.flags |= DI_REPORT_FLAGS_DONE; +} + +void wait_di_buffer(void) +{ + if (debug_data->di_report.cur_samples == NUM_DI_SAMPLE) + { + flag_di_buffer_ready(); + while (debug_data->di_report.cur_samples != 0) + { + } + debug_data->di_report.flags = 0; + } +} + +void sample_di_data(alt_u32 bit_chk, alt_u32 delay, alt_u32 d, alt_u32 v, alt_u32 p) +{ + alt_u32 k; + alt_u32 di_status_word; + alt_u32 di_word_avail; + alt_u32 di_write_to_read_ratio; + alt_u32 di_write_to_read_ratio_2_exp; + + wait_di_buffer(); + + k = debug_data->di_report.cur_samples; + + debug_data->di_report.di_buffer[k].bit_chk = bit_chk; + debug_data->di_report.di_buffer[k].delay = delay; + debug_data->di_report.di_buffer[k].d = d; + debug_data->di_report.di_buffer[k].v = v; + debug_data->di_report.di_buffer[k].p = p; + + di_status_word = IORD_32DIRECT(BASE_RW_MGR + 8, 0); + di_word_avail = di_status_word & 0x0000FFFF; + di_write_to_read_ratio = (di_status_word & 0x00FF0000) >> 16; + di_write_to_read_ratio_2_exp = (di_status_word & 0xFF000000) >> 24; + + debug_data->di_report.di_buffer[k].di_buffer_0a = IORD_32DIRECT(BASE_RW_MGR + 16 + 0*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_0b = IORD_32DIRECT(BASE_RW_MGR + 16 + 1*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_1a = IORD_32DIRECT(BASE_RW_MGR + 16 + 2*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_1b = IORD_32DIRECT(BASE_RW_MGR + 16 + 3*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_2a = IORD_32DIRECT(BASE_RW_MGR + 16 + 4*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_2b = IORD_32DIRECT(BASE_RW_MGR + 16 + 5*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_3a = IORD_32DIRECT(BASE_RW_MGR + 16 + 6*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_3b = IORD_32DIRECT(BASE_RW_MGR + 16 + 7*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_4a = IORD_32DIRECT(BASE_RW_MGR + 16 + 8*4, 0); + debug_data->di_report.di_buffer[k].di_buffer_4b = IORD_32DIRECT(BASE_RW_MGR + 16 + 9*4, 0); + + debug_data->di_report.cur_samples = debug_data->di_report.cur_samples + 1; +} +#endif + +// This (TEST_SIZE) is used to test handling of large roms, to make +// sure we are sizing things correctly +// Note, the initialized data takes up twice the space in rom, since +// there needs to be a copy with the initial value and a copy that is +// written too, since on soft-reset, it needs to have the initial values +// without reloading the memory from external sources + +// #define TEST_SIZE (6*1024) + +#ifdef TEST_SIZE + +#define PRE_POST_TEST_SIZE 3 + +unsigned int pre_test_size_mem[PRE_POST_TEST_SIZE] = { 1, 2, 3}; + +unsigned int test_size_mem[TEST_SIZE/sizeof(unsigned int)] = { 100, 200, 300 }; + +unsigned int post_test_size_mem[PRE_POST_TEST_SIZE] = {10, 20, 30}; + +void write_test_mem(void) +{ + int i; + + for (i = 0; i < PRE_POST_TEST_SIZE; i++) { + pre_test_size_mem[i] = (i+1)*10; + post_test_size_mem[i] = (i+1); + } + + for (i = 0; i < sizeof(test_size_mem)/sizeof(unsigned int); i++) { + test_size_mem[i] = i; + } + +} + +int check_test_mem(int start) +{ + int i; + + for (i = 0; i < PRE_POST_TEST_SIZE; i++) { + if (start) { + if (pre_test_size_mem[i] != (i+1)) { + return 0; + } + if (post_test_size_mem[i] != (i+1)*10) { + return 0; + } + } else { + if (pre_test_size_mem[i] != (i+1)*10) { + return 0; + } + if (post_test_size_mem[i] != (i+1)) { + return 0; + } + } + } + + for (i = 0; i < sizeof(test_size_mem)/sizeof(unsigned int); i++) { + if (start) { + if (i < 3) { + if (test_size_mem[i] != (i+1)*100) { + return 0; + } + } else { + if (test_size_mem[i] != 0) { + return 0; + } + } + } else { + if (test_size_mem[i] != i) { + return 0; + } + } + } + + return 1; +} + +#endif // TEST_SIZE + +static void set_failing_group_stage(alt_u32 group, alt_u32 stage, alt_u32 substage) +{ + ALTERA_ASSERT(group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + // Only set the global stage if there was not been any other failing group + if (gbl->error_stage == CAL_STAGE_NIL) + { + gbl->error_substage = substage; + gbl->error_stage = stage; + gbl->error_group = group; + TCLRPT_SET(debug_summary_report->error_sub_stage, substage); + TCLRPT_SET(debug_summary_report->error_stage, stage); + TCLRPT_SET(debug_summary_report->error_group, group); + + } + + // Always set the group specific errors + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][group].error_stage, stage); + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][group].error_sub_stage, substage); + +} + +static inline void reg_file_set_group(alt_u32 set_group) +{ + // Read the current group and stage + alt_u32 cur_stage_group = IORD_32DIRECT (REG_FILE_CUR_STAGE, 0); + + // Clear the group + cur_stage_group &= 0x0000FFFF; + + // Set the group + cur_stage_group |= (set_group << 16); + + // Write the data back + IOWR_32DIRECT (REG_FILE_CUR_STAGE, 0, cur_stage_group); +} + +static inline void reg_file_set_stage(alt_u32 set_stage) +{ + // Read the current group and stage + alt_u32 cur_stage_group = IORD_32DIRECT (REG_FILE_CUR_STAGE, 0); + + // Clear the stage and substage + cur_stage_group &= 0xFFFF0000; + + // Set the stage + cur_stage_group |= (set_stage & 0x000000FF); + + // Write the data back + IOWR_32DIRECT (REG_FILE_CUR_STAGE, 0, cur_stage_group); +} + +static inline void reg_file_set_sub_stage(alt_u32 set_sub_stage) +{ + // Read the current group and stage + alt_u32 cur_stage_group = IORD_32DIRECT (REG_FILE_CUR_STAGE, 0); + + // Clear the substage + cur_stage_group &= 0xFFFF00FF; + + // Set the sub stage + cur_stage_group |= ((set_sub_stage << 8) & 0x0000FF00); + + // Write the data back + IOWR_32DIRECT (REG_FILE_CUR_STAGE, 0, cur_stage_group); +} + +static inline alt_u32 is_write_group_enabled_for_dm(alt_u32 write_group) +{ +#if DM_PINS_ENABLED + #if RLDRAMII + alt_32 decrement_counter = write_group + 1; + + while (decrement_counter > 0) + { + decrement_counter -= RW_MGR_MEM_IF_WRITE_DQS_WIDTH/RW_MGR_MEM_DATA_MASK_WIDTH; + } + + if (decrement_counter == 0) + { + return 1; + } + else + { + return 0; + } + #else + return 1; + #endif +#else + return 0; +#endif +} + +static inline void select_curr_shadow_reg_using_rank(alt_u32 rank) +{ +#if USE_SHADOW_REGS + //USER Map the rank to its shadow reg and set the global variable + curr_shadow_reg = (rank >> (NUM_RANKS_PER_SHADOW_REG - 1)); +#endif +} + +void initialize(void) +{ + TRACE_FUNC(); + + //USER calibration has control over path to memory + +#if HARD_PHY + // In Hard PHY this is a 2-bit control: + // 0: AFI Mux Select + // 1: DDIO Mux Select + IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 0x3); +#else + IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 1); +#endif + + //USER memory clock is not stable we begin initialization + + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 0); + + //USER calibration status all set to zero + + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, 0); + IOWR_32DIRECT (PHY_MGR_CAL_DEBUG_INFO, 0, 0); + + if (((DYNAMIC_CALIB_STEPS) & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) { + param->read_correct_mask_vg = ((t_btfld)1 << (RW_MGR_MEM_DQ_PER_READ_DQS / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1; + param->write_correct_mask_vg = ((t_btfld)1 << (RW_MGR_MEM_DQ_PER_READ_DQS / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1; + param->read_correct_mask = ((t_btfld)1 << RW_MGR_MEM_DQ_PER_READ_DQS) - 1; + param->write_correct_mask = ((t_btfld)1 << RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1; + param->dm_correct_mask = ((t_btfld)1 << (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH)) - 1; + } +} + + +#if MRS_MIRROR_PING_PONG_ATSO +// This code is specific to the ATSO setup. There are two ways to set +// the cs/odt mask: +// 1. the normal way (set_rank_and_odt_mask) +// This method will be used in general. The behavior will be to unmask +// BOTH CS (i.e. broadcast to both sides as if calibrating one large interface). +// 2. this function +// This method will be used for MRS settings only. This allows us to do settings +// on a per-side basis. This is needed because Slot 1 Rank 1 needs a mirrored MRS. +// This function is specific to our setup ONLY. +void set_rank_and_odt_mask_for_ping_pong_atso(alt_u32 side, alt_u32 odt_mode) +{ + alt_u32 odt_mask_0 = 0; + alt_u32 odt_mask_1 = 0; + alt_u32 cs_and_odt_mask; + + if(odt_mode == RW_MGR_ODT_MODE_READ_WRITE) + { + //USER 1 Rank + //USER Read: ODT = 0 + //USER Write: ODT = 1 + odt_mask_0 = 0x0; + odt_mask_1 = 0x1; + } + else + { + odt_mask_0 = 0x0; + odt_mask_1 = 0x0; + } + + cs_and_odt_mask = + (0xFF & ~(1 << side)) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); + + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, cs_and_odt_mask); +} +#endif + +#if DDR3 +void set_rank_and_odt_mask(alt_u32 rank, alt_u32 odt_mode) +{ + alt_u32 odt_mask_0 = 0; + alt_u32 odt_mask_1 = 0; + alt_u32 cs_and_odt_mask; + + if(odt_mode == RW_MGR_ODT_MODE_READ_WRITE) + { +#if USE_SHADOW_REGS + alt_u32 rank_one_hot = (0xFF & (1 << rank)); + select_curr_shadow_reg_using_rank(rank); + + //USER Assert afi_rrank and afi_wrank. These signals ultimately drive + //USER the read/write rank select signals which select the shadow register. + IOWR_32DIRECT (RW_MGR_SET_ACTIVE_RANK, 0, rank_one_hot); +#endif + + if ( LRDIMM ) { + // USER LRDIMMs have two cases to consider: single-slot and dual-slot. + // USER In single-slot, assert ODT for write only. + // USER In dual-slot, assert ODT for both slots for write, + // USER and on the opposite slot only for reads. + // USER + // USER Further complicating this is that both DIMMs have either 1 or 2 ODT + // USER inputs, which do the same thing (only one is actually required). + if ((RW_MGR_MEM_CHIP_SELECT_WIDTH/RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM) == 1) { + // USER Single-slot case + if (RW_MGR_MEM_ODT_WIDTH == 1) { + // USER Read = 0, Write = 1 + odt_mask_0 = 0x0; + odt_mask_1 = 0x1; + } else if (RW_MGR_MEM_ODT_WIDTH == 2) { + // USER Read = 00, Write = 11 + odt_mask_0 = 0x0; + odt_mask_1 = 0x3; + } + } else if ((RW_MGR_MEM_CHIP_SELECT_WIDTH/RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM) == 2) { + // USER Dual-slot case + if (RW_MGR_MEM_ODT_WIDTH == 2) { + // USER Read: asserted for opposite slot, Write: asserted for both + odt_mask_0 = (rank < 2) ? 0x2 : 0x1; + odt_mask_1 = 0x3; + } else if (RW_MGR_MEM_ODT_WIDTH == 4) { + // USER Read: asserted for opposite slot, Write: asserted for both + odt_mask_0 = (rank < 2) ? 0xC : 0x3; + odt_mask_1 = 0xF; + } + } + } else if(RW_MGR_MEM_NUMBER_OF_RANKS == 1) { + //USER 1 Rank + //USER Read: ODT = 0 + //USER Write: ODT = 1 + odt_mask_0 = 0x0; + odt_mask_1 = 0x1; + } else if(RW_MGR_MEM_NUMBER_OF_RANKS == 2) { + //USER 2 Ranks + if(RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1 || + (RDIMM && RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 2 + && RW_MGR_MEM_CHIP_SELECT_WIDTH == 4)) { + //USER - Dual-Slot , Single-Rank (1 chip-select per DIMM) + //USER OR + //USER - RDIMM, 4 total CS (2 CS per DIMM) means 2 DIMM + //USER Since MEM_NUMBER_OF_RANKS is 2 they are both single rank + //USER with 2 CS each (special for RDIMM) + //USER Read: Turn on ODT on the opposite rank + //USER Write: Turn on ODT on all ranks + odt_mask_0 = 0x3 & ~(1 << rank); + odt_mask_1 = 0x3; + } else { + //USER - Single-Slot , Dual-rank DIMMs (2 chip-selects per DIMM) + //USER Read: Turn on ODT off on all ranks + //USER Write: Turn on ODT on active rank + odt_mask_0 = 0x0; + odt_mask_1 = 0x3 & (1 << rank); + } + } else { + //USER 4 Ranks + //USER Read: + //USER ----------+-----------------------+ + //USER | | + //USER | ODT | + //USER Read From +-----------------------+ + //USER Rank | 3 | 2 | 1 | 0 | + //USER ----------+-----+-----+-----+-----+ + //USER 0 | 0 | 1 | 0 | 0 | + //USER 1 | 1 | 0 | 0 | 0 | + //USER 2 | 0 | 0 | 0 | 1 | + //USER 3 | 0 | 0 | 1 | 0 | + //USER ----------+-----+-----+-----+-----+ + //USER + //USER Write: + //USER ----------+-----------------------+ + //USER | | + //USER | ODT | + //USER Write To +-----------------------+ + //USER Rank | 3 | 2 | 1 | 0 | + //USER ----------+-----+-----+-----+-----+ + //USER 0 | 0 | 1 | 0 | 1 | + //USER 1 | 1 | 0 | 1 | 0 | + //USER 2 | 0 | 1 | 0 | 1 | + //USER 3 | 1 | 0 | 1 | 0 | + //USER ----------+-----+-----+-----+-----+ + switch(rank) + { + case 0: + odt_mask_0 = 0x4; + odt_mask_1 = 0x5; + break; + case 1: + odt_mask_0 = 0x8; + odt_mask_1 = 0xA; + break; + case 2: + odt_mask_0 = 0x1; + odt_mask_1 = 0x5; + break; + case 3: + odt_mask_0 = 0x2; + odt_mask_1 = 0xA; + break; + } + } + } + else + { + odt_mask_0 = 0x0; + odt_mask_1 = 0x0; + } + +#if ADVANCED_ODT_CONTROL + // odt_mask_0 = read + // odt_mask_1 = write + odt_mask_0 = (CFG_READ_ODT_CHIP >> (RW_MGR_MEM_ODT_WIDTH * rank)); + odt_mask_1 = (CFG_WRITE_ODT_CHIP >> (RW_MGR_MEM_ODT_WIDTH * rank)); + odt_mask_0 &= ((1 << RW_MGR_MEM_ODT_WIDTH) - 1); + odt_mask_1 &= ((1 << RW_MGR_MEM_ODT_WIDTH) - 1); +#endif + +#if MRS_MIRROR_PING_PONG_ATSO + // See set_cs_and_odt_mask_for_ping_pong_atso + cs_and_odt_mask = + (0xFC) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); +#else + if(RDIMM && RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 2 + && RW_MGR_MEM_CHIP_SELECT_WIDTH == 4 && RW_MGR_MEM_NUMBER_OF_RANKS == 2) { + //USER See RDIMM special case above + cs_and_odt_mask = + (0xFF & ~(1 << (2*rank))) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); + } else if (LRDIMM) { +#if LRDIMM + // USER LRDIMM special cases - When RM=2, CS[2] is remapped to A[16] so skip it, + // USER and when RM=4, CS[3:2] are remapped to A[17:16] so skip them both. + alt_u32 lrdimm_rank = 0; + alt_u32 lrdimm_rank_mask = 0; + + //USER When rank multiplication is active, the remapped CS pins must be forced low + //USER instead of high for proper targetted RTT_NOM programming. + if (LRDIMM_RANK_MULTIPLICATION_FACTOR == 2) { + // USER Mask = CS[5:0] = 011011 + lrdimm_rank_mask = (0x3 | (0x3 << 3)); + } else if (LRDIMM_RANK_MULTIPLICATION_FACTOR == 4) { + // USER Mask = CS[7:0] = 00110011 + lrdimm_rank_mask = (0x3 | (0x3 << 4)); + } + + // USER Handle LRDIMM cases where Rank multiplication may be active + if (((RW_MGR_MEM_CHIP_SELECT_WIDTH/RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM) == 1)) { + // USER Single-DIMM case + lrdimm_rank = ~(1 << rank); + } else if ((RW_MGR_MEM_CHIP_SELECT_WIDTH/RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM) == 2) { + if (rank < (RW_MGR_MEM_NUMBER_OF_RANKS >> 1)) { + // USER Dual-DIMM case, accessing first slot + lrdimm_rank = ~(1 << rank); + } else { + // USER Dual-DIMM case, accessing second slot + lrdimm_rank = ~(1 << (rank + RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM - (RW_MGR_MEM_NUMBER_OF_RANKS>>1))); + } + } + cs_and_odt_mask = + (lrdimm_rank_mask & lrdimm_rank) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); +#endif // LRDIMM + } else { + cs_and_odt_mask = + (0xFF & ~(1 << rank)) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); + } +#endif + + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, cs_and_odt_mask); +} +#else +#if DDR2 +void set_rank_and_odt_mask(alt_u32 rank, alt_u32 odt_mode) +{ + alt_u32 odt_mask_0 = 0; + alt_u32 odt_mask_1 = 0; + alt_u32 cs_and_odt_mask; + + if(odt_mode == RW_MGR_ODT_MODE_READ_WRITE) + { + if(RW_MGR_MEM_NUMBER_OF_RANKS == 1) { + //USER 1 Rank + //USER Read: ODT = 0 + //USER Write: ODT = 1 + odt_mask_0 = 0x0; + odt_mask_1 = 0x1; + } else if(RW_MGR_MEM_NUMBER_OF_RANKS == 2) { + //USER 2 Ranks + if(RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1 || + (RDIMM && RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 2 + && RW_MGR_MEM_CHIP_SELECT_WIDTH == 4)) { + //USER - Dual-Slot , Single-Rank (1 chip-select per DIMM) + //USER OR + //USER - RDIMM, 4 total CS (2 CS per DIMM) means 2 DIMM + //USER Since MEM_NUMBER_OF_RANKS is 2 they are both single rank + //USER with 2 CS each (special for RDIMM) + //USER Read/Write: Turn on ODT on the opposite rank + odt_mask_0 = 0x3 & ~(1 << rank); + odt_mask_1 = 0x3 & ~(1 << rank); + } else { + //USER - Single-Slot , Dual-rank DIMMs (2 chip-selects per DIMM) + //USER Read: Turn on ODT off on all ranks + //USER Write: Turn on ODT on active rank + odt_mask_0 = 0x0; + odt_mask_1 = 0x3 & (1 << rank); + } + } else { + //USER 4 Ranks + //USER Read/Write: + //USER -----------+-----------------------+ + //USER | | + //USER | ODT | + //USER Read/Write | | + //USER From +-----------------------+ + //USER Rank | 3 | 2 | 1 | 0 | + //USER -----------+-----+-----+-----+-----+ + //USER 0 | 0 | 1 | 0 | 0 | + //USER 1 | 1 | 0 | 0 | 0 | + //USER 2 | 0 | 0 | 0 | 1 | + //USER 3 | 0 | 0 | 1 | 0 | + //USER -----------+-----+-----+-----+-----+ + switch(rank) + { + case 0: + odt_mask_0 = 0x4; + odt_mask_1 = 0x4; + break; + case 1: + odt_mask_0 = 0x8; + odt_mask_1 = 0x8; + break; + case 2: + odt_mask_0 = 0x1; + odt_mask_1 = 0x1; + break; + case 3: + odt_mask_0 = 0x2; + odt_mask_1 = 0x2; + break; + } + } + } + else + { + odt_mask_0 = 0x0; + odt_mask_1 = 0x0; + } + + if(RDIMM && RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 2 + && RW_MGR_MEM_CHIP_SELECT_WIDTH == 4 && RW_MGR_MEM_NUMBER_OF_RANKS == 2) { + //USER See RDIMM/LRDIMM special case above + cs_and_odt_mask = + (0xFF & ~(1 << (2*rank))) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); + } else { + cs_and_odt_mask = + (0xFF & ~(1 << rank)) | + ((0xFF & odt_mask_0) << 8) | + ((0xFF & odt_mask_1) << 16); + } + + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, cs_and_odt_mask); +} +#else // QDRII and RLDRAMx +void set_rank_and_odt_mask(alt_u32 rank, alt_u32 odt_mode) +{ + alt_u32 cs_and_odt_mask = + (0xFF & ~(1 << rank)); + + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, cs_and_odt_mask); +} +#endif +#endif + +//USER Given a rank, select the set of shadow registers that is responsible for the +//USER delays of such rank, so that subsequent SCC updates will go to those shadow +//USER registers. +void select_shadow_regs_for_update (alt_u32 rank, alt_u32 group, alt_u32 update_scan_chains) +{ +#if USE_SHADOW_REGS + alt_u32 rank_one_hot = (0xFF & (1 << rank)); + + //USER Assert afi_rrank and afi_wrank. These signals ultimately drive + //USER the read/write rank select signals which select the shadow register. + IOWR_32DIRECT (RW_MGR_SET_ACTIVE_RANK, 0, rank_one_hot); + + //USER Cause the SCC manager to switch its register file, which is used as + //USER local cache of the various dtap/ptap settings. There's one register file + //USER per shadow register set. + IOWR_32DIRECT (SCC_MGR_ACTIVE_RANK, 0, rank_one_hot); + + if (update_scan_chains) { + alt_u32 i; + + //USER On the read side, a memory read is required because the read rank + //USER select signal (as well as the postamble delay chain settings) is clocked + //USER into the periphery by the postamble signal. Simply asserting afi_rrank + //USER is not enough. If update_scc_regfile is not set, we assume there'll be a + //USER subsequent read that'll handle this. + for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; ++i) { + + //USER The dummy read can go to any non-skipped rank. + //USER Skipped ranks are uninitialized and their banks are un-activated. + //USER Accessing skipped ranks can lead to bad behavior. + if (! param->skip_ranks[i]) { + + set_rank_and_odt_mask(i, RW_MGR_ODT_MODE_READ_WRITE); + + // must re-assert afi_wrank/afi_rrank prior to issuing read + // because set_rank_and_odt_mask may have changed the signals. + IOWR_32DIRECT (RW_MGR_SET_ACTIVE_RANK, 0, rank_one_hot); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x10); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_READ_B2B_WAIT1); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x10); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_READ_B2B_WAIT2); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_READ_B2B); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_READ_B2B); + + IOWR_32DIRECT (RW_MGR_RUN_ALL_GROUPS, 0, __RW_MGR_READ_B2B); + + //USER The dummy read above may cause the DQS enable signal to be stuck high. + //USER The following corrects this. + IOWR_32DIRECT (RW_MGR_RUN_ALL_GROUPS, 0, __RW_MGR_CLEAR_DQS_ENABLE); + + set_rank_and_odt_mask(i, RW_MGR_ODT_MODE_OFF); + + break; + } + } + + //USER Reset the fifos to get pointers to known state + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + + //USER On the write side the afi_wrank signal eventually propagates to the I/O + //USER through the write datapath. We need to make sure we wait long enough for + //USER this to happen. The operations above should be enough, hence no extra delay + //USER inserted here. + + //USER Make sure the data in the I/O scan chains are in-sync with the register + //USER file inside the SCC manager. If we don't do this, a subsequent SCC_UPDATE + //USER may cause stale data for the other shadow register to be loaded. This must + //USER be done for every scan chain of the current group. Note that in shadow + //USER register mode, the SCC_UPDATE signal is per-group. + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, group); + + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, group); + IOWR_32DIRECT (SCC_MGR_DQS_IO_ENA, 0, 0); + + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + IOWR_32DIRECT (SCC_MGR_DQ_ENA, 0, i); + } + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { + IOWR_32DIRECT (SCC_MGR_DM_ENA, 0, i); + } + } + + //USER Map the rank to its shadow reg + select_curr_shadow_reg_using_rank(rank); +#endif +} + +#if HHP_HPS +void scc_mgr_initialize(void) +{ + // Clear register file for HPS + // 16 (2^4) is the size of the full register file in the scc mgr: + // RFILE_DEPTH = log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS + MEM_IF_READ_DQS_WIDTH - 1) + 1; + alt_u32 i; + for (i = 0; i < 16; i++) { + DPRINT(1, "Clearing SCC RFILE index %lu", i); + IOWR_32DIRECT(SCC_MGR_HHP_RFILE, i << 2, 0); + } +} +#endif + +inline void scc_mgr_set_dqs_bus_in_delay(alt_u32 read_group, alt_u32 delay) +{ + ALTERA_ASSERT(read_group < RW_MGR_MEM_IF_READ_DQS_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQS_IN_DELAY(read_group, delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][read_group].dqs_bus_in_delay, delay); + +} + +static inline void scc_mgr_set_dqs_io_in_delay(alt_u32 write_group, alt_u32 delay) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQS_IO_IN_DELAY(delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].dqs_io_in_delay, delay); + +} + +static inline void scc_mgr_set_dqs_en_phase(alt_u32 read_group, alt_u32 phase) +{ + ALTERA_ASSERT(read_group < RW_MGR_MEM_IF_READ_DQS_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQS_EN_PHASE(read_group, phase); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][read_group].dqs_en_phase, phase); + +} + +void scc_mgr_set_dqs_en_phase_all_ranks (alt_u32 read_group, alt_u32 phase) +{ + alt_u32 r; + alt_u32 update_scan_chains; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + //USER although the h/w doesn't support different phases per shadow register, + //USER for simplicity our scc manager modeling keeps different phase settings per + //USER shadow reg, and it's important for us to keep them in sync to match h/w. + //USER for efficiency, the scan chain update should occur only once to sr0. + update_scan_chains = (r == 0) ? 1 : 0; + + select_shadow_regs_for_update(r, read_group, update_scan_chains); + scc_mgr_set_dqs_en_phase(read_group, phase); + + if (update_scan_chains) { + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + } +} + + +static inline void scc_mgr_set_dqdqs_output_phase(alt_u32 write_group, alt_u32 phase) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + #if CALIBRATE_BIT_SLIPS + alt_u32 num_fr_slips = 0; + while (phase > IO_DQDQS_OUT_PHASE_MAX) { + phase -= IO_DLL_CHAIN_LENGTH; + num_fr_slips++; + } + IOWR_32DIRECT (PHY_MGR_FR_SHIFT, write_group*4, num_fr_slips); +#endif + + // Load the setting in the SCC manager + WRITE_SCC_DQDQS_OUT_PHASE(write_group, phase); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].dqdqs_out_phase, phase); + +} + +void scc_mgr_set_dqdqs_output_phase_all_ranks (alt_u32 write_group, alt_u32 phase) +{ + alt_u32 r; + alt_u32 update_scan_chains; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + //USER although the h/w doesn't support different phases per shadow register, + //USER for simplicity our scc manager modeling keeps different phase settings per + //USER shadow reg, and it's important for us to keep them in sync to match h/w. + //USER for efficiency, the scan chain update should occur only once to sr0. + update_scan_chains = (r == 0) ? 1 : 0; + + select_shadow_regs_for_update(r, write_group, update_scan_chains); + scc_mgr_set_dqdqs_output_phase(write_group, phase); + + if (update_scan_chains) { + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, write_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + } +} + + +static inline void scc_mgr_set_dqs_en_delay(alt_u32 read_group, alt_u32 delay) +{ + ALTERA_ASSERT(read_group < RW_MGR_MEM_IF_READ_DQS_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQS_EN_DELAY(read_group, delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][read_group].dqs_en_delay, delay); + +} + +void scc_mgr_set_dqs_en_delay_all_ranks (alt_u32 read_group, alt_u32 delay) +{ + alt_u32 r; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + + select_shadow_regs_for_update(r, read_group, 0); + + scc_mgr_set_dqs_en_delay(read_group, delay); + + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, read_group); + +#if !USE_SHADOW_REGS + // In shadow register mode, the T11 settings are stored in registers + // in the core, which are updated by the DQS_ENA signals. Not issuing + // the SCC_MGR_UPD command allows us to save lots of rank switching + // overhead, by calling select_shadow_regs_for_update with update_scan_chains + // set to 0. + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); +#endif + } +} + +static void scc_mgr_set_oct_out1_delay(alt_u32 write_group, alt_u32 delay) +{ + alt_u32 read_group; + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + // Load the setting in the SCC manager + // Although OCT affects only write data, the OCT delay is controlled by the DQS logic block + // which is instantiated once per read group. For protocols where a write group consists + // of multiple read groups, the setting must be set multiple times. + for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + ++read_group) { + + WRITE_SCC_OCT_OUT1_DELAY(read_group, delay); + } + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].oct_out_delay1, delay); + +} + +static void scc_mgr_set_oct_out2_delay(alt_u32 write_group, alt_u32 delay) +{ + alt_u32 read_group; + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + // Load the setting in the SCC manager + // Although OCT affects only write data, the OCT delay is controlled by the DQS logic block + // which is instantiated once per read group. For protocols where a write group consists + // of multiple read groups, the setting must be set multiple times. + for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + ++read_group) { + + WRITE_SCC_OCT_OUT2_DELAY(read_group, delay); + } + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].oct_out_delay2, delay); + +} + +static inline void scc_mgr_set_dqs_bypass(alt_u32 write_group, alt_u32 bypass) +{ + // Load the setting in the SCC manager + WRITE_SCC_DQS_BYPASS(write_group, bypass); +} + +inline void scc_mgr_set_dq_out1_delay(alt_u32 write_group, alt_u32 dq_in_group, alt_u32 delay) +{ +#if ENABLE_TCL_DEBUG || ENABLE_ASSERT + alt_u32 dq = write_group*RW_MGR_MEM_DQ_PER_WRITE_DQS + dq_in_group; +#endif + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + ALTERA_ASSERT(dq < RW_MGR_MEM_DATA_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQ_OUT1_DELAY(dq_in_group, delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dq_settings[curr_shadow_reg][dq].dq_out_delay1, delay); + +} + +inline void scc_mgr_set_dq_out2_delay(alt_u32 write_group, alt_u32 dq_in_group, alt_u32 delay) +{ +#if ENABLE_TCL_DEBUG || ENABLE_ASSERT + alt_u32 dq = write_group*RW_MGR_MEM_DQ_PER_WRITE_DQS + dq_in_group; +#endif + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + ALTERA_ASSERT(dq < RW_MGR_MEM_DATA_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQ_OUT2_DELAY(dq_in_group, delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dq_settings[curr_shadow_reg][dq].dq_out_delay2, delay); + +} + +inline void scc_mgr_set_dq_in_delay(alt_u32 write_group, alt_u32 dq_in_group, alt_u32 delay) +{ +#if ENABLE_TCL_DEBUG || ENABLE_ASSERT + alt_u32 dq = write_group*RW_MGR_MEM_DQ_PER_WRITE_DQS + dq_in_group; +#endif + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + ALTERA_ASSERT(dq < RW_MGR_MEM_DATA_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQ_IN_DELAY(dq_in_group, delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dq_settings[curr_shadow_reg][dq].dq_in_delay, delay); + +} + +static inline void scc_mgr_set_dq_bypass(alt_u32 write_group, alt_u32 dq_in_group, alt_u32 bypass) +{ + // Load the setting in the SCC manager + WRITE_SCC_DQ_BYPASS(dq_in_group, bypass); +} + +static inline void scc_mgr_set_rfifo_mode(alt_u32 write_group, alt_u32 dq_in_group, alt_u32 mode) +{ + // Load the setting in the SCC manager + WRITE_SCC_RFIFO_MODE(dq_in_group, mode); +} + +static inline void scc_mgr_set_hhp_extras(void) +{ + // Load the fixed setting in the SCC manager + // bits: 0:0 = 1'b1 - dqs bypass + // bits: 1:1 = 1'b1 - dq bypass + // bits: 4:2 = 3'b001 - rfifo_mode + // bits: 6:5 = 2'b01 - rfifo clock_select + // bits: 7:7 = 1'b0 - separate gating from ungating setting + // bits: 8:8 = 1'b0 - separate OE from Output delay setting + alt_u32 value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0); + WRITE_SCC_HHP_EXTRAS(value); +} + +static inline void scc_mgr_set_hhp_dqse_map(void) +{ + // Load the fixed setting in the SCC manager + WRITE_SCC_HHP_DQSE_MAP(0); +} + +static inline void scc_mgr_set_dqs_out1_delay(alt_u32 write_group, alt_u32 delay) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQS_IO_OUT1_DELAY(delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].dqs_out_delay1, delay); + +} + +static inline void scc_mgr_set_dqs_out2_delay(alt_u32 write_group, alt_u32 delay) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + // Load the setting in the SCC manager + WRITE_SCC_DQS_IO_OUT2_DELAY(delay); + + // Make the setting in the TCL report + TCLRPT_SET(debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].dqs_out_delay2, delay); + +} + +inline void scc_mgr_set_dm_out1_delay(alt_u32 write_group, alt_u32 dm, alt_u32 delay) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + ALTERA_ASSERT(dm < RW_MGR_NUM_DM_PER_WRITE_GROUP); + + // Load the setting in the SCC manager + WRITE_SCC_DM_IO_OUT1_DELAY(dm, delay); + + // Make the setting in the TCL report + + if (RW_MGR_NUM_TRUE_DM_PER_WRITE_GROUP > 0) + { + TCLRPT_SET(debug_cal_report->cal_dm_settings[curr_shadow_reg][write_group][dm].dm_out_delay1, delay); + } +} + +inline void scc_mgr_set_dm_out2_delay(alt_u32 write_group, alt_u32 dm, alt_u32 delay) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + ALTERA_ASSERT(dm < RW_MGR_NUM_DM_PER_WRITE_GROUP); + + // Load the setting in the SCC manager + WRITE_SCC_DM_IO_OUT2_DELAY(dm, delay); + + // Make the setting in the TCL report + + if (RW_MGR_NUM_TRUE_DM_PER_WRITE_GROUP > 0) + { + TCLRPT_SET(debug_cal_report->cal_dm_settings[curr_shadow_reg][write_group][dm].dm_out_delay2, delay); + } +} + +static inline void scc_mgr_set_dm_in_delay(alt_u32 write_group, alt_u32 dm, alt_u32 delay) +{ + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + ALTERA_ASSERT(dm < RW_MGR_NUM_DM_PER_WRITE_GROUP); + + // Load the setting in the SCC manager + WRITE_SCC_DM_IO_IN_DELAY(dm, delay); + + // Make the setting in the TCL report + + if (RW_MGR_NUM_TRUE_DM_PER_WRITE_GROUP > 0) + { + TCLRPT_SET(debug_cal_report->cal_dm_settings[curr_shadow_reg][write_group][dm].dm_in_delay, delay); + } +} + +static inline void scc_mgr_set_dm_bypass(alt_u32 write_group, alt_u32 dm, alt_u32 bypass) +{ + // Load the setting in the SCC manager + WRITE_SCC_DM_BYPASS(dm, bypass); +} + +//USER Zero all DQS config +// TODO: maybe rename to scc_mgr_zero_dqs_config (or something) +void scc_mgr_zero_all (void) +{ + alt_u32 i, r; + + //USER Zero all DQS config settings, across all groups and all shadow registers + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + + // Strictly speaking this should be called once per group to make + // sure each group's delay chain is refreshed from the SCC register file, + // but since we're resetting all delay chains anyway, we can save some + // runtime by calling select_shadow_regs_for_update just once to switch + // rank. + select_shadow_regs_for_update(r, 0, 1); + + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + // The phases actually don't exist on a per-rank basis, but there's + // no harm updating them several times, so let's keep the code simple. + scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE); + scc_mgr_set_dqs_en_phase(i, 0); + scc_mgr_set_dqs_en_delay(i, 0); + } + + for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + scc_mgr_set_dqdqs_output_phase(i, 0); +#if ARRIAV || CYCLONEV + // av/cv don't have out2 + scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE); +#else + scc_mgr_set_oct_out1_delay(i, 0); + scc_mgr_set_oct_out2_delay(i, IO_DQS_OUT_RESERVE); +#endif + } + + //USER multicast to all DQS group enables + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, 0xff); + +#if USE_SHADOW_REGS + //USER in shadow-register mode, SCC_UPDATE is done on a per-group basis + //USER unless we explicitly ask for a multicast via the group counter + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, 0xFF); +#endif + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } +} + +void scc_set_bypass_mode(alt_u32 write_group, alt_u32 mode) +{ + // mode = 0 : Do NOT bypass - Half Rate Mode + // mode = 1 : Bypass - Full Rate Mode + +#if !HHP_HPS + alt_u32 i; +#endif + +#if HHP_HPS + // only need to set once for all groups, pins, dq, dqs, dm + if (write_group == 0) { + DPRINT(1, "Setting HHP Extras"); + scc_mgr_set_hhp_extras(); + DPRINT(1, "Done Setting HHP Extras"); + } +#endif + +#if !HHP_HPS + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) + { + scc_mgr_set_dq_bypass(write_group, i, mode); + scc_mgr_set_rfifo_mode(write_group, i, mode); + } +#endif + + //USER multicast to all DQ enables + IOWR_32DIRECT (SCC_MGR_DQ_ENA, 0, 0xff); + +#if !HHP_HPS + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) + { + scc_mgr_set_dm_bypass(write_group, i, mode); + } +#endif + + IOWR_32DIRECT (SCC_MGR_DM_ENA, 0, 0xff); + +#if !HHP_HPS + scc_mgr_set_dqs_bypass(write_group, mode); +#endif + + //USER update current DQS IO enable + IOWR_32DIRECT (SCC_MGR_DQS_IO_ENA, 0, 0); + + //USER update the DQS logic + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, write_group); + + //USER hit update + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); +} + +// Moving up to avoid warnings +void scc_mgr_load_dqs_for_write_group (alt_u32 write_group) +{ + alt_u32 read_group; + + // Although OCT affects only write data, the OCT delay is controlled by the DQS logic block + // which is instantiated once per read group. For protocols where a write group consists + // of multiple read groups, the setting must be scanned multiple times. + for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + ++read_group) { + + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, read_group); + } +} + +void scc_mgr_zero_group (alt_u32 write_group, alt_u32 test_begin, alt_32 out_only) +{ + alt_u32 i, r; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + + select_shadow_regs_for_update(r, write_group, 1); + + //USER Zero all DQ config settings + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) + { + scc_mgr_set_dq_out1_delay(write_group, i, 0); + scc_mgr_set_dq_out2_delay(write_group, i, IO_DQ_OUT_RESERVE); + if (!out_only) { + scc_mgr_set_dq_in_delay(write_group, i, 0); + } + } + + //USER multicast to all DQ enables + IOWR_32DIRECT (SCC_MGR_DQ_ENA, 0, 0xff); + + //USER Zero all DM config settings + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) + { + if (!out_only) { + // Do we really need this? + scc_mgr_set_dm_in_delay(write_group, i, 0); + } + scc_mgr_set_dm_out1_delay(write_group, i, 0); + scc_mgr_set_dm_out2_delay(write_group, i, IO_DM_OUT_RESERVE); + } + + //USER multicast to all DM enables + IOWR_32DIRECT (SCC_MGR_DM_ENA, 0, 0xff); + + //USER zero all DQS io settings + if (!out_only) { + scc_mgr_set_dqs_io_in_delay(write_group, 0); + } +#if ARRIAV || CYCLONEV + // av/cv don't have out2 + scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE); + scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE); + scc_mgr_load_dqs_for_write_group (write_group); +#else + scc_mgr_set_dqs_out1_delay(write_group, 0); + scc_mgr_set_dqs_out2_delay(write_group, IO_DQS_OUT_RESERVE); + scc_mgr_set_oct_out1_delay(write_group, 0); + scc_mgr_set_oct_out2_delay(write_group, IO_DQS_OUT_RESERVE); + scc_mgr_load_dqs_for_write_group (write_group); +#endif + + //USER multicast to all DQS IO enables (only 1) + IOWR_32DIRECT (SCC_MGR_DQS_IO_ENA, 0, 0); + +#if USE_SHADOW_REGS + //USER in shadow-register mode, SCC_UPDATE is done on a per-group basis + //USER unless we explicitly ask for a multicast via the group counter + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, 0xFF); +#endif + //USER hit update to zero everything + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } +} + +//USER load up dqs config settings + +void scc_mgr_load_dqs (alt_u32 dqs) +{ + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, dqs); +} + + +//USER load up dqs io config settings + +void scc_mgr_load_dqs_io (void) +{ + IOWR_32DIRECT (SCC_MGR_DQS_IO_ENA, 0, 0); +} + +//USER load up dq config settings + +void scc_mgr_load_dq (alt_u32 dq_in_group) +{ + IOWR_32DIRECT (SCC_MGR_DQ_ENA, 0, dq_in_group); +} + +//USER load up dm config settings + +void scc_mgr_load_dm (alt_u32 dm) +{ + IOWR_32DIRECT (SCC_MGR_DM_ENA, 0, dm); +} + +//USER apply and load a particular input delay for the DQ pins in a group +//USER group_bgn is the index of the first dq pin (in the write group) + +void scc_mgr_apply_group_dq_in_delay (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay) +{ + alt_u32 i, p; + + for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { + scc_mgr_set_dq_in_delay(write_group, p, delay); + scc_mgr_load_dq (p); + } +} + +//USER apply and load a particular output delay for the DQ pins in a group + +void scc_mgr_apply_group_dq_out1_delay (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay1) +{ + alt_u32 i, p; + + for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + scc_mgr_set_dq_out1_delay(write_group, i, delay1); + scc_mgr_load_dq (i); + } +} + +void scc_mgr_apply_group_dq_out2_delay (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay2) +{ + alt_u32 i, p; + + for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + scc_mgr_set_dq_out2_delay(write_group, i, delay2); + scc_mgr_load_dq (i); + } +} + +//USER apply and load a particular output delay for the DM pins in a group + +void scc_mgr_apply_group_dm_out1_delay (alt_u32 write_group, alt_u32 delay1) +{ + alt_u32 i; + + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { + scc_mgr_set_dm_out1_delay(write_group, i, delay1); + scc_mgr_load_dm (i); + } +} + + +//USER apply and load delay on both DQS and OCT out1 +void scc_mgr_apply_group_dqs_io_and_oct_out1 (alt_u32 write_group, alt_u32 delay) +{ + scc_mgr_set_dqs_out1_delay(write_group, delay); + scc_mgr_load_dqs_io (); + + scc_mgr_set_oct_out1_delay(write_group, delay); + scc_mgr_load_dqs_for_write_group (write_group); +} + +//USER apply and load delay on both DQS and OCT out2 +void scc_mgr_apply_group_dqs_io_and_oct_out2 (alt_u32 write_group, alt_u32 delay) +{ + scc_mgr_set_dqs_out2_delay(write_group, delay); + scc_mgr_load_dqs_io (); + + scc_mgr_set_oct_out2_delay(write_group, delay); + scc_mgr_load_dqs_for_write_group (write_group); +} + +//USER set delay on both DQS and OCT out1 by incrementally changing +//USER the settings one dtap at a time towards the target value, to avoid +//USER breaking the lock of the DLL/PLL on the memory device. +void scc_mgr_set_group_dqs_io_and_oct_out1_gradual (alt_u32 write_group, alt_u32 delay) +{ + alt_u32 d = READ_SCC_DQS_IO_OUT1_DELAY(); + + while (d > delay) { + --d; + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + } + while (d < delay) { + ++d; + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + } +} + +//USER set delay on both DQS and OCT out2 by incrementally changing +//USER the settings one dtap at a time towards the target value, to avoid +//USER breaking the lock of the DLL/PLL on the memory device. +void scc_mgr_set_group_dqs_io_and_oct_out2_gradual (alt_u32 write_group, alt_u32 delay) +{ + alt_u32 d = READ_SCC_DQS_IO_OUT2_DELAY(); + + while (d > delay) { + --d; + scc_mgr_apply_group_dqs_io_and_oct_out2 (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + } + while (d < delay) { + ++d; + scc_mgr_apply_group_dqs_io_and_oct_out2 (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + } +} + +//USER apply a delay to the entire output side: DQ, DM, DQS, OCT + +void scc_mgr_apply_group_all_out_delay (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay) +{ + //USER dq shift + + scc_mgr_apply_group_dq_out1_delay (write_group, group_bgn, delay); + + //USER dm shift + + scc_mgr_apply_group_dm_out1_delay (write_group, delay); + + //USER dqs and oct shift + + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, delay); +} + +//USER apply a delay to the entire output side (DQ, DM, DQS, OCT) and to all ranks +void scc_mgr_apply_group_all_out_delay_all_ranks (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay) +{ + alt_u32 r; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + + select_shadow_regs_for_update(r, write_group, 1); + + scc_mgr_apply_group_all_out_delay (write_group, group_bgn, delay); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } +} + +//USER apply a delay to the entire output side: DQ, DM, DQS, OCT + +void scc_mgr_apply_group_all_out_delay_add (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay) +{ + alt_u32 i, p, new_delay; + + //USER dq shift + + for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + + new_delay = READ_SCC_DQ_OUT2_DELAY(i); + new_delay += delay; + + if (new_delay > IO_IO_OUT2_DELAY_MAX) { + DPRINT(1, "%s(%lu, %lu, %lu) DQ[%lu,%lu]: %lu > %lu => %lu", + __func__, write_group, group_bgn, delay, i, p, + new_delay, (long unsigned int)IO_IO_OUT2_DELAY_MAX, (long unsigned int)IO_IO_OUT2_DELAY_MAX); + new_delay = IO_IO_OUT2_DELAY_MAX; + } + + scc_mgr_set_dq_out2_delay(write_group, i, new_delay); + scc_mgr_load_dq (i); + } + + //USER dm shift + + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { + new_delay = READ_SCC_DM_IO_OUT2_DELAY(i); + new_delay += delay; + + if (new_delay > IO_IO_OUT2_DELAY_MAX) { + DPRINT(1, "%s(%lu, %lu, %lu) DM[%lu]: %lu > %lu => %lu", + __func__, write_group, group_bgn, delay, i, + new_delay, (long unsigned int)IO_IO_OUT2_DELAY_MAX, (long unsigned int)IO_IO_OUT2_DELAY_MAX); + new_delay = IO_IO_OUT2_DELAY_MAX; + } + + scc_mgr_set_dm_out2_delay(write_group, i, new_delay); + scc_mgr_load_dm (i); + } + + //USER dqs shift + + new_delay = READ_SCC_DQS_IO_OUT2_DELAY(); + new_delay += delay; + + if (new_delay > IO_IO_OUT2_DELAY_MAX) { + DPRINT(1, "%s(%lu, %lu, %lu) DQS: %lu > %d => %d; adding %lu to OUT1", + __func__, write_group, group_bgn, delay, + new_delay, IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX, + new_delay - IO_IO_OUT2_DELAY_MAX); + scc_mgr_set_dqs_out1_delay(write_group, new_delay - IO_IO_OUT2_DELAY_MAX); + new_delay = IO_IO_OUT2_DELAY_MAX; + } + + scc_mgr_set_dqs_out2_delay(write_group, new_delay); + scc_mgr_load_dqs_io (); + + //USER oct shift + + new_delay = READ_SCC_OCT_OUT2_DELAY(write_group); + new_delay += delay; + + if (new_delay > IO_IO_OUT2_DELAY_MAX) { + DPRINT(1, "%s(%lu, %lu, %lu) DQS: %lu > %d => %d; adding %lu to OUT1", + __func__, write_group, group_bgn, delay, + new_delay, IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX, + new_delay - IO_IO_OUT2_DELAY_MAX); + scc_mgr_set_oct_out1_delay(write_group, new_delay - IO_IO_OUT2_DELAY_MAX); + new_delay = IO_IO_OUT2_DELAY_MAX; + } + + scc_mgr_set_oct_out2_delay(write_group, new_delay); + scc_mgr_load_dqs_for_write_group (write_group); +} + +//USER apply a delay to the entire output side (DQ, DM, DQS, OCT) and to all ranks +void scc_mgr_apply_group_all_out_delay_add_all_ranks (alt_u32 write_group, alt_u32 group_bgn, alt_u32 delay) +{ + alt_u32 r; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + + select_shadow_regs_for_update(r, write_group, 1); + + scc_mgr_apply_group_all_out_delay_add (write_group, group_bgn, delay); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } +} + +static inline void scc_mgr_spread_out2_delay_all_ranks (alt_u32 write_group, alt_u32 test_bgn) +{ +#if STRATIXV || ARRIAVGZ + alt_u32 found; + alt_u32 i; + alt_u32 p; + alt_u32 d; + alt_u32 r; + + const alt_u32 delay_step = IO_IO_OUT2_DELAY_MAX/(RW_MGR_MEM_DQ_PER_WRITE_DQS-1); /* we start at zero, so have one less dq to devide among */ + + TRACE_FUNC("(%lu,%lu)", write_group, test_bgn); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + select_shadow_regs_for_update(r, write_group, 1); + for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++, d += delay_step) { + DPRINT(1, "rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay: g=%lu r=%lu, i=%lu p=%lu d=%lu", + write_group, r, i, p, d); + scc_mgr_set_dq_out2_delay(write_group, i, d); + scc_mgr_load_dq (i); + } + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } +#endif +} + +#if DDR3 +// optimization used to recover some slots in ddr3 inst_rom +// could be applied to other protocols if we wanted to +void set_jump_as_return(void) +{ + // to save space, we replace return with jump to special shared RETURN instruction + // so we set the counter to large value so that we always jump + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0xFF); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_RETURN); + +} +#endif + +// should always use constants as argument to ensure all computations are performed at compile time +static inline void delay_for_n_mem_clocks(const alt_u32 clocks) +{ + alt_u32 afi_clocks; + alt_u8 inner; + alt_u8 outer; + alt_u16 c_loop; + + TRACE_FUNC("clocks=%lu ... start", clocks); + + afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO; /* scale (rounding up) to get afi clocks */ + + // Note, we don't bother accounting for being off a little bit because of a few extra instructions in outer loops + // Note, the loops have a test at the end, and do the test before the decrement, and so always perform the loop + // 1 time more than the counter value + if (afi_clocks == 0) { + inner = outer = c_loop = 0; + } else if (afi_clocks <= 0x100) { + inner = afi_clocks-1; + outer = 0; + c_loop = 0; + } else if (afi_clocks <= 0x10000) { + inner = 0xff; + outer = (afi_clocks-1) >> 8; + c_loop = 0; + } else { + inner = 0xff; + outer = 0xff; + c_loop = (afi_clocks-1) >> 16; + } + + // rom instructions are structured as follows: + // + // IDLE_LOOP2: jnz cntr0, TARGET_A + // IDLE_LOOP1: jnz cntr1, TARGET_B + // return + // + // so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and TARGET_B is + // set to IDLE_LOOP2 as well + // + // if we have no outer loop, though, then we can use IDLE_LOOP1 only, and set + // TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely + // + // a little confusing, but it helps save precious space in the inst_rom and sequencer rom + // and keeps the delays more accurate and reduces overhead + if (afi_clocks <= 0x100) { + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner)); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_IDLE_LOOP1); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_IDLE_LOOP1); + + } else { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer)); + + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_IDLE_LOOP2); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_IDLE_LOOP2); + + // hack to get around compiler not being smart enough + if (afi_clocks <= 0x10000) { + // only need to run once + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_IDLE_LOOP2); + } else { + do { + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_IDLE_LOOP2); + } while (c_loop-- != 0); + } + } + + TRACE_FUNC("clocks=%lu ... end", clocks); +} + +// should always use constants as argument to ensure all computations are performed at compile time +static inline void delay_for_n_ns(const alt_u32 nanoseconds) +{ + TRACE_FUNC("nanoseconds=%lu ... end", nanoseconds); + delay_for_n_mem_clocks((1000*nanoseconds) / (1000000/AFI_CLK_FREQ) * AFI_RATE_RATIO); +} + +#if RLDRAM3 +// Special routine to recover memory device from illegal state after +// ck/dk relationship is potentially violated. +static inline void recover_mem_device_after_ck_dqs_violation(void) +{ + //USER Issue MRS0 command. For some reason this is required once we + //USER violate tCKDK. Without this all subsequent write tests will fail + //USER even with known good delays. + + //USER Load MR0 + if ( RW_MGR_MEM_NUMBER_OF_RANKS == 1 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 2 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFC); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 4 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFC); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + //USER Wait MRSC + delay_for_n_mem_clocks(12); + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xF3); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_QUAD_RANK); + } + else { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + } + + //USER Wait MRSC + delay_for_n_mem_clocks(12); +} +#else +// Special routine to recover memory device from illegal state after +// ck/dqs relationship is violated. +static inline void recover_mem_device_after_ck_dqs_violation(void) +{ + // Current protocol doesn't require any special recovery +} +#endif + +#if (LRDIMM && DDR3) +// Routine to program specific LRDIMM control words. +static void rw_mgr_lrdimm_rc_program(alt_u32 fscw, alt_u32 rc_addr, alt_u32 rc_val) +{ + alt_u32 i; + const alt_u32 AC_BASE_CONTENT = __RW_MGR_CONTENT_ac_rdimm; + //USER These values should be dynamically loaded instead of hard-coded + const alt_u32 AC_ADDRESS_POSITION = 0x0; + const alt_u32 AC_BANK_ADDRESS_POSITION = 0xD; + alt_u32 ac_content; + alt_u32 lrdimm_cs_msk = RW_MGR_RANK_NONE; + + TRACE_FUNC(); + + //USER Turn on only CS0 and CS1 for each DIMM. + for (i = 0; i < RW_MGR_MEM_CHIP_SELECT_WIDTH; i+= RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM) + { + lrdimm_cs_msk &= (~(3 << i)); + } + + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, lrdimm_cs_msk); + + // Program the fscw first (RC7), followed by the actual value + for (i = 0; i < 2; i++) + { + alt_u32 addr; + alt_u32 val; + + addr = (i == 0) ? 7 : rc_addr; + val = (i == 0) ? fscw : rc_val; + + ac_content = + AC_BASE_CONTENT | + //USER Word address + ((addr & 0x7) << AC_ADDRESS_POSITION) | + (((addr >> 3) & 0x1) << (AC_BANK_ADDRESS_POSITION + 2)) | + //USER Configuration Word + (((val >> 2) & 0x3) << (AC_BANK_ADDRESS_POSITION)) | + ((val & 0x3) << (AC_ADDRESS_POSITION + 3)); + + //USER Override the AC row with the RDIMM command + IOWR_32DIRECT(BASE_RW_MGR, 0x1C00 + (__RW_MGR_ac_rdimm << 2), ac_content); + + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_RDIMM_CMD); + } + + // USER The following registers require a delay of tSTAB (6us) for proper functionality. + // USER F0RC2, F0RC10, F0RC11, F1RC8, F1RC11-F1RC15 + // USER Note that it is only necessary to wait tSTAB after all of these + // USER control words have been written, not after each one. Only F0RC0-F0RC15 + // USER are guaranteed to be written (and in order), but F1* are not so + // USER wait after each. + if ( ((fscw == 0) && ((rc_addr==2) || (rc_addr==10) || (rc_addr==11))) + || ((fscw == 1) && (rc_addr >= 8))) + { + delay_for_n_ns(6000); + } +} +#endif +#if (RDIMM || LRDIMM) && DDR3 +void rw_mgr_rdimm_initialize(void) +{ + alt_u32 i; + alt_u32 conf_word; +#if RDIMM + const alt_u32 AC_BASE_CONTENT = __RW_MGR_CONTENT_ac_rdimm; + //USER These values should be dynamically loaded instead of hard-coded + const alt_u32 AC_ADDRESS_POSITION = 0x0; + const alt_u32 AC_BANK_ADDRESS_POSITION = 0xD; + alt_u32 ac_content; +#endif + + TRACE_FUNC(); + + //USER RDIMM registers are programmed by writing 16 configuration words + //USER 1. An RDIMM command is a NOP with all CS asserted + //USER 2. The 4-bit address of the configuration words is + //USER * { mem_ba[2] , mem_a[2] , mem_a[1] , mem_a[0] } + //USER 3. The 4-bit configuration word is + //USER * { mem_ba[1] , mem_ba[0] , mem_a[4] , mem_a[3] } + +#if RDIMM + //USER Turn on all ranks + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, RW_MGR_RANK_ALL); +#endif + + for(i = 0; i < 16; i++) + { + + + if(i < 8) + { +#if ENABLE_TCL_DEBUG && USE_USER_RDIMM_VALUE + conf_word = (my_debug_data.command_parameters[0] >> (i * 4)) & 0xF; +#else + conf_word = (RDIMM_CONFIG_WORD_LOW >> (i * 4)) & 0xF; +#endif + } + else + { +#if ENABLE_TCL_DEBUG && USE_USER_RDIMM_VALUE + conf_word = (my_debug_data.command_parameters[1] >> ((i - 8) * 4)) & 0xF; +#else + conf_word = (RDIMM_CONFIG_WORD_HIGH >> ((i - 8) * 4)) & 0xF; +#endif + } + +#if RDIMM + ac_content = + AC_BASE_CONTENT | + //USER Word address + ((i & 0x7) << AC_ADDRESS_POSITION) | + (((i >> 3) & 0x1) << (AC_BANK_ADDRESS_POSITION + 2)) | + //USER Configuration Word + (((conf_word >> 2) & 0x3) << (AC_BANK_ADDRESS_POSITION)) | + ((conf_word & 0x3) << (AC_ADDRESS_POSITION + 3)); + + //USER Override the AC row with the RDIMM command + IOWR_32DIRECT(BASE_RW_MGR, 0x1C00 + (__RW_MGR_ac_rdimm << 2), ac_content); + + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_RDIMM_CMD); + //USER When sending the RC2 or RC10 word, tSTAB time must elapse before the next command + //USER is sent out. tSTAB is currently hard-coded to 6us. + if((i == 2) || (i == 10)) + { + //USER tSTAB = 6 us + delay_for_n_ns(6000); + } + +#endif +#if LRDIMM + // USER Program configuration word with FSCW set to zero. + rw_mgr_lrdimm_rc_program(0, i, conf_word); +#endif + } +} +#else +void rw_mgr_rdimm_initialize(void) { } +#endif + +#if DDR3 + +#if (ADVANCED_ODT_CONTROL || LRDIMM) +alt_u32 ddr3_mirror_mrs_cmd(alt_u32 bit_vector) { + // This function performs address mirroring of an AC ROM command, which + // requires swapping the following DDR3 bits: + // A[3] <=> A[4] + // A[5] <=> A[6] + // A[7] <=> A[8] + // BA[0] <=>BA[1] + // We assume AC_ROM_ENTRY = {BA[2:0], A[15:0]}. + alt_u32 unchanged_bits; + alt_u32 mask_a; + alt_u32 mask_b; + alt_u32 retval; + + unchanged_bits = (~(DDR3_AC_MIRR_MASK | (DDR3_AC_MIRR_MASK << 1))) & bit_vector; + mask_a = DDR3_AC_MIRR_MASK & bit_vector; + mask_b = (DDR3_AC_MIRR_MASK << 1) & bit_vector; + + retval = unchanged_bits | (mask_a << 1) | (mask_b >> 1); + + return retval; +} + +void rtt_change_MRS1_MRS2_NOM_WR (alt_u32 prev_ac_mr , alt_u32 odt_ac_mr, alt_u32 mirr_on, alt_u32 mr_cmd ) { + // This function updates the ODT-specific Mode Register bits (MRS1 or MRS2) in the AC ROM. + // Parameters: prev_ac_mr - Original, *un-mirrored* AC ROM Entry + // odt_ac_mr - ODT bits to update (un-mirrored) + // mirr_on - boolean flag indicating if the regular or mirrored entry is updated + // mr_cmd - Mode register command (only MR1 and MR2 are supported for DDR3) + alt_u32 new_ac_mr; + alt_u32 ac_rom_entry = 0; + alt_u32 ac_rom_mask; + + switch (mr_cmd) { + case 1: { + // USER MRS1 = RTT_NOM, RTT_DRV + ac_rom_mask = DDR3_MR1_ODT_MASK; + ac_rom_entry = mirr_on ? (0x1C00 | (__RW_MGR_ac_mrs1_mirr << 2)) + : (0x1C00 | (__RW_MGR_ac_mrs1 << 2)); + } break; + case 2: { + // USER MRS2 = RTT_WR + ac_rom_mask = DDR3_MR2_ODT_MASK; + ac_rom_entry = mirr_on ? (0x1C00 | (__RW_MGR_ac_mrs2_mirr << 2)) + : (0x1C00 | (__RW_MGR_ac_mrs2 << 2)); + } break; + } + + // USER calculate new AC values and update ROM + new_ac_mr = odt_ac_mr; + new_ac_mr |= (prev_ac_mr & ac_rom_mask); + if (mirr_on) { + new_ac_mr = ddr3_mirror_mrs_cmd(new_ac_mr); + } + IOWR_32DIRECT(BASE_RW_MGR, ac_rom_entry, new_ac_mr); +} +#endif //(ADVANCED_ODT_CONTROL || LRDIMM) + +void rw_mgr_mem_initialize (void) +{ + alt_u32 r; + +#if LRDIMM + alt_u32 rtt_nom; + alt_u32 rtt_drv; + alt_u32 rtt_wr; +#endif // LRDIMM + + TRACE_FUNC(); + + //USER The reset / cke part of initialization is broadcasted to all ranks + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, RW_MGR_RANK_ALL); + + // Here's how you load register for a loop + //USER Counters are located @ 0x800 + //USER Jump address are located @ 0xC00 + //USER For both, registers 0 to 3 are selected using bits 3 and 2, like in + //USER 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C + // I know this ain't pretty, but Avalon bus throws away the 2 least significant bits + + //USER start with memory RESET activated + + //USER tINIT is typically 200us (but can be adjusted in the GUI) + //USER The total number of cycles required for this nested counter structure to + //USER complete is defined by: + //USER num_cycles = (CTR2 + 1) * [(CTR1 + 1) * (2 * (CTR0 + 1) + 1) + 1] + 1 + + //USER Load counters + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL)); + + //USER Load jump address + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_INIT_RESET_0_CKE_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_INIT_RESET_0_CKE_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_INIT_RESET_0_CKE_0); + + //USER Execute count instruction + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_INIT_RESET_0_CKE_0); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + + //USER transition the RESET to high + //USER Wait for 500us + //USER num_cycles = (CTR2 + 1) * [(CTR1 + 1) * (2 * (CTR0 + 1) + 1) + 1] + 1 + //USER Load counters + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL)); + + //USER Load jump address + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_INIT_RESET_1_CKE_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_INIT_RESET_1_CKE_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_INIT_RESET_1_CKE_0); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_INIT_RESET_1_CKE_0); + + //USER bring up clock enable + + //USER tXRP < 250 ck cycles + delay_for_n_mem_clocks(250); + +#ifdef RDIMM + // USER initialize RDIMM buffer so MRS and RZQ Calibrate commands will be + // USER propagated to discrete memory devices + rw_mgr_rdimm_initialize(); +#endif + +#if LRDIMM + // USER initialize LRDIMM MB so MRS and RZQ Calibrate commands will be + // USER propagated to all sub-ranks. Per LRDIMM spec, all LRDIMM ranks must have + // USER RTT_WR set, but only physical ranks 0 and 1 should have RTT_NOM set. + // USER Therefore RTT_NOM=0 is broadcast to all ranks, and the non-zero value is + // USER programmed directly into Ranks 0 and 1 using physical MRS targetting. + rw_mgr_rdimm_initialize(); + + rtt_nom = LRDIMM_SPD_MR_RTT_NOM(LRDIMM_SPD_MR); + rtt_drv = LRDIMM_SPD_MR_RTT_DRV(LRDIMM_SPD_MR); + rtt_wr = LRDIMM_SPD_MR_RTT_WR(LRDIMM_SPD_MR); + + // USER Configure LRDIMM to broadcast LRDIMM MRS commands to all ranks + rw_mgr_lrdimm_rc_program(0, 14, (((RDIMM_CONFIG_WORD_HIGH >> 24) & 0xF) & (~0x4))); + + // USER Update contents of AC ROM with new RTT WR, DRV values only (NOM = Off) + rtt_change_MRS1_MRS2_NOM_WR(__RW_MGR_CONTENT_ac_mrs1, rtt_drv, 0, 1); + rtt_change_MRS1_MRS2_NOM_WR(__RW_MGR_CONTENT_ac_mrs1, rtt_drv, 1, 1); + rtt_change_MRS1_MRS2_NOM_WR(__RW_MGR_CONTENT_ac_mrs2, rtt_wr, 0, 2); + rtt_change_MRS1_MRS2_NOM_WR(__RW_MGR_CONTENT_ac_mrs2, rtt_wr, 1, 2); +#endif +#if RDIMM + // USER initialize RDIMM buffer so MRS and RZQ Calibrate commands will be + // USER propagated to discrete memory devices + rw_mgr_rdimm_initialize(); +#endif + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + +#if ADVANCED_ODT_CONTROL + alt_u32 rtt_nom = 0; + alt_u32 rtt_wr = 0; + alt_u32 rtt_drv = 0; + + switch (r) { + case 0: { + rtt_nom = MR1_RTT_RANK0; + rtt_wr = MR2_RTT_WR_RANK0; + rtt_drv = MR1_RTT_DRV_RANK0; + } break; + case 1: { + rtt_nom = MR1_RTT_RANK1; + rtt_wr = MR2_RTT_WR_RANK1; + rtt_drv = MR1_RTT_DRV_RANK1; + } break; + case 2: { + rtt_nom = MR1_RTT_RANK2; + rtt_wr = MR2_RTT_WR_RANK2; + rtt_drv = MR1_RTT_DRV_RANK2; + } break; + case 3: { + rtt_nom = MR1_RTT_RANK3; + rtt_wr = MR2_RTT_WR_RANK3; + rtt_drv = MR1_RTT_DRV_RANK3; + } break; + } + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs1, (rtt_nom|rtt_drv), + ((RW_MGR_MEM_ADDRESS_MIRRORING>>r)&0x1), 1); + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs2, rtt_wr, + ((RW_MGR_MEM_ADDRESS_MIRRORING>>r)&0x1), 2); +#endif //ADVANCED_ODT_CONTROL + + //USER set rank +#if MRS_MIRROR_PING_PONG_ATSO + // Special case + // SIDE 0 + set_rank_and_odt_mask_for_ping_pong_atso(0, RW_MGR_ODT_MODE_OFF); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_DLL_RESET); + + // SIDE 1 + set_rank_and_odt_mask_for_ping_pong_atso(1, RW_MGR_ODT_MODE_OFF); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_DLL_RESET_MIRR); + + // Unmask all CS + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); +#else + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER Use Mirror-ed commands for odd ranks if address mirrorring is on + if((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_DLL_RESET_MIRR); + } else { + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_DLL_RESET); + } +#endif + + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_ZQCL); + + //USER tZQinit = tDLLK = 512 ck cycles + delay_for_n_mem_clocks(512); + } +#if LRDIMM + // USER Configure LRDIMM to target physical ranks decoded by RM bits only (ranks 0,1 only) + // USER Set bit F0RC14.DBA0 to '1' so MRS commands target physical ranks only + rw_mgr_lrdimm_rc_program(0, 14, (((RDIMM_CONFIG_WORD_HIGH >> 24) & 0xF) | 0x4)); + // USER update AC ROM MR1 entry to include RTT_NOM + rtt_change_MRS1_MRS2_NOM_WR(__RW_MGR_CONTENT_ac_mrs1, (rtt_drv|rtt_nom), 0, 1); + rtt_change_MRS1_MRS2_NOM_WR(__RW_MGR_CONTENT_ac_mrs1, (rtt_drv|rtt_nom), 1, 1); + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER Use Mirror-ed commands for odd ranks if address mirrorring is on + if((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_MIRR); + delay_for_n_mem_clocks(4); + } else { + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + delay_for_n_mem_clocks(4); + } + } + + // USER Initiate LRDIMM MB->Physical Rank training here + // USER -> Set minimum skew mode for levelling - F3RC6 = 0001 + rw_mgr_lrdimm_rc_program(3, 6, 0x1); + // USER -> Set error status output in register F2RC3 for debugging purposes + rw_mgr_lrdimm_rc_program(2, 3, 0x8); + +#ifdef LRDIMM_EXT_CONFIG_ARRAY + // USER Configure LRDIMM ODT/Drive parameters using SPD information + { + static const alt_u8 lrdimm_cfg_array[][3] = LRDIMM_EXT_CONFIG_ARRAY; + alt_u32 cfg_reg_ctr; + + for (cfg_reg_ctr = 0; cfg_reg_ctr < (sizeof(lrdimm_cfg_array)/sizeof(lrdimm_cfg_array[0])); cfg_reg_ctr++) + { + alt_u32 lrdimm_fp = (alt_u32)lrdimm_cfg_array[cfg_reg_ctr][0]; + alt_u32 lrdimm_rc = (alt_u32)lrdimm_cfg_array[cfg_reg_ctr][1]; + alt_u32 lrdimm_val = (alt_u32)lrdimm_cfg_array[cfg_reg_ctr][2]; + + rw_mgr_lrdimm_rc_program(lrdimm_fp, lrdimm_rc, lrdimm_val); + } + } +#endif // LRDIMM_EXT_CONFIG_ARRAY + + // USER -> Initiate MB->DIMM training on the LRDIMM + rw_mgr_lrdimm_rc_program(0, 12, 0x2); +#if (!STATIC_SKIP_DELAY_LOOPS) + // USER Wait for max(tcal) * number of physical ranks. Tcal is approx. 10ms. + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS * RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM; r++) + { + delay_for_n_ns(80000000UL); + } +#endif // !STATIC_SKIP_DELAY_LOOPS + // USER Place MB back in normal operating mode + rw_mgr_lrdimm_rc_program(0, 12, 0x0); +#endif // LRDIMM +} + + +#if (ENABLE_NON_DESTRUCTIVE_CALIB || ENABLE_NON_DES_CAL) +void rw_mgr_mem_initialize_no_init (void) +{ + alt_u32 r; + alt_u32 mem_refresh_all_ranks(alt_u32 no_validate); + TRACE_FUNC(); + rw_mgr_rdimm_initialize(); + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, RW_MGR_RANK_ALL); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_RETURN); + delay_for_n_mem_clocks(512); + mem_refresh_all_ranks(1); + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + continue; + } + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + set_jump_as_return(); + if((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_DLL_RESET_MIRR); + } else { + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_DLL_RESET); + } + +// Reprogramming these is not really required but.... + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + + + + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_ZQCL); + delay_for_n_mem_clocks(512); + } + + IOWR_32DIRECT (RW_MGR_ENABLE_REFRESH, 0, 1); // Enable refresh engine + +} +#endif +#endif // DDR3 + +#if DDR2 +void rw_mgr_mem_initialize (void) +{ + alt_u32 r; + + TRACE_FUNC(); + + //USER *** NOTE *** + //USER The following STAGE (n) notation refers to the corresponding stage in the Micron datasheet + + // Here's how you load register for a loop + //USER Counters are located @ 0x800 + //USER Jump address are located @ 0xC00 + //USER For both, registers 0 to 3 are selected using bits 3 and 2, like in + //USER 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C + // I know this ain't pretty, but Avalon bus throws away the 2 least significant bits + + //USER *** STAGE (1, 2, 3) *** + + //USER start with CKE low + + //USER tINIT is typically 200us (but can be adjusted in the GUI) + //USER The total number of cycles required for this nested counter structure to + //USER complete is defined by: + //USER num_cycles = (CTR0 + 1) * [(CTR1 + 1) * (2 * (CTR2 + 1) + 1) + 1] + 1 + + //TODO: Need to manage multi-rank + + //USER Load counters + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL)); + + //USER Load jump address + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_INIT_CKE_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_INIT_CKE_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_INIT_CKE_0); + + //USER Execute count instruction + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_INIT_CKE_0); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + + //USER Bring up CKE + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_NOP); + + //USER *** STAGE (4) + + //USER Wait for 400ns + delay_for_n_ns(400); + + //USER Multi-rank section begins here + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER * **** * + //USER * NOTE * + //USER * **** * + //USER The following commands must be spaced by tMRD or tRPA which are in the order + //USER of 2 to 4 full rate cycles. This is peanuts in the NIOS domain, so for now + //USER we can avoid redundant wait loops + + // Possible FIXME BEN: for HHP, we need to add delay loops to be sure + // although, the sequencer write interface by itself likely has enough delay + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER *** STAGE (5) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR2); + + //USER *** STAGE (6) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR3); + + //USER *** STAGE (7) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR); + + //USER *** STAGE (8) + //USER DLL reset + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR_DLL_RESET); + + //USER *** STAGE (9) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER *** STAGE (10) + + //USER Issue 2 refresh commands spaced by tREF + + //USER First REFRESH + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_REFRESH); + + //USER tREF = 200ns + delay_for_n_ns(200); + + //USER Second REFRESH + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_REFRESH); + + //USER Second idle loop + delay_for_n_ns(200); + + //USER *** STAGE (11) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR_CALIB); + + //USER *** STAGE (12) + //USER OCD defaults + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR_OCD_ENABLE); + + //USER *** STAGE (13) + //USER OCD exit + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR); + + //USER *** STAGE (14) + + //USER The memory is now initialized. Before being able to use it, we must still + //USER wait for the DLL to lock, 200 clock cycles after it was reset @ STAGE (8). + //USER Since we cannot keep track of time in any other way, let's start counting from now + delay_for_n_mem_clocks(200); + } +} +#endif // DDR2 + +#if LPDDR2 +void rw_mgr_mem_initialize (void) +{ + alt_u32 r; + + //USER *** NOTE *** + //USER The following STAGE (n) notation refers to the corresponding stage in the Micron datasheet + + // Here's how you load register for a loop + //USER Counters are located @ 0x800 + //USER Jump address are located @ 0xC00 + //USER For both, registers 0 to 3 are selected using bits 3 and 2, like in + //USER 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C + // I know this ain't pretty, but Avalon bus throws away the 2 least significant bits + + //USER *** STAGE (1, 2, 3) *** + + //USER start with CKE low + + //USER tINIT1 = 100ns + + //USER 100ns @ 300MHz (3.333 ns) ~ 30 cycles + //USER If a is the number of iteration in a loop + //USER it takes the following number of cycles to complete the operation: + //USER number_of_cycles = (2 + n) * a + //USER where n is the number of instruction in the inner loop + //USER One possible solution is n = 0 , a = 15 => a = 0x10 + + //USER Load counter + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(0x10)); + + //USER Load jump address + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_INIT_CKE_0); + + //USER Execute count instruction + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_INIT_CKE_0); + + //USER tINIT3 = 200us + delay_for_n_ns(200000); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + + //USER Multi-rank section begins here + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER MRW RESET + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR63_RESET); + } + + //USER tINIT5 = 10us + delay_for_n_ns(10000); + + //USER Multi-rank section begins here + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER MRW ZQC + // Note: We cannot calibrate other ranks when the current rank is calibrating for tZQINIT + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR10_ZQC); + + //USER tZQINIT = 1us + delay_for_n_ns(1000); + + //USER * **** * + //USER * NOTE * + //USER * **** * + //USER The following commands must be spaced by tMRW which is in the order + //USER of 3 to 5 full rate cycles. This is peanuts in the NIOS domain, so for now + //USER we can avoid redundant wait loops + + //USER MRW MR1 + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR1_CALIB); + + //USER MRW MR2 + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR2); + + //USER MRW MR3 + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR3); + } +} +#endif // LPDDR2 + +#if LPDDR1 +void rw_mgr_mem_initialize (void) +{ + alt_u32 r; + + TRACE_FUNC(); + + //USER *** NOTE *** + //USER The following STAGE (n) notation refers to the corresponding stage in the Micron datasheet + + // Here's how you load register for a loop + //USER Counters are located @ 0x800 + //USER Jump address are located @ 0xC00 + //USER For both, registers 0 to 3 are selected using bits 3 and 2, like in + //USER 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C + // I know this ain't pretty, but Avalon bus throws away the 2 least significant bits + + //USER *** STAGE (1, 2, 3) *** + + //USER start with CKE high + + //USER tINIT = 200us + + //USER 200us @ 300MHz (3.33 ns) ~ 60000 clock cycles + //USER If a and b are the number of iteration in 2 nested loops + //USER it takes the following number of cycles to complete the operation: + //USER number_of_cycles = ((2 + n) * b + 2) * a + //USER where n is the number of instruction in the inner loop + //USER One possible solution is n = 0 , a = 256 , b = 118 => a = FF, b = 76 + + //TODO: Need to manage multi-rank + + //USER Load counters + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(0xFF)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(0x76)); + + //USER Load jump address + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_INIT_CKE_1); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_INIT_CKE_1_inloop); + + //USER Execute count instruction and bring up CKE + //USER IOWR_32DIRECT (BASE_RW_MGR, 0, __RW_MGR_COUNT_REG_0); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_INIT_CKE_1); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + + //USER Multi-rank section begins here + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER * **** * + //USER * NOTE * + //USER * **** * + //USER The following commands must be spaced by tMRD or tRPA which are in the order + //USER of 2 to 4 full rate cycles. This is peanuts in the NIOS domain, so for now + //USER we can avoid redundant wait loops + + // Possible FIXME BEN: for HHP, we need to add delay loops to be sure + // although, the sequencer write interface by itself likely has enough delay + + //USER *** STAGE (9) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER *** STAGE (10) + + //USER Issue 2 refresh commands spaced by tREF + + //USER First REFRESH + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_REFRESH); + + //USER tREF = 200ns + delay_for_n_ns(200); + + //USER Second REFRESH + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_REFRESH); + + //USER Second idle loop + delay_for_n_ns(200); + + //USER *** STAGE (11) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR_CALIB); + + //USER *** STAGE (13) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR); + + } +} +#endif // LPDDR1 + +#if QDRII +void rw_mgr_mem_initialize (void) +{ + TRACE_FUNC(); + + //USER Turn off QDRII DLL to reset it + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_IDLE); + + //USER Turn on QDRII DLL and wait 25us for it to lock + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_NOP); + delay_for_n_ns(25000); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); +} +#endif + +#if QDRII +void rw_mgr_mem_dll_lock_wait (void) +{ + //USER The DLL in QDR requires 25us to lock + delay_for_n_ns(25000); +} +#else +void rw_mgr_mem_dll_lock_wait (void) { } +#endif + +#if RLDRAMII +void rw_mgr_mem_initialize (void) +{ + TRACE_FUNC(); + + //USER start with memory RESET activated + + //USER tINIT = 200us + delay_for_n_ns(200000); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + + //USER Dummy MRS, followed by valid MRS commands to reset the DLL on memory device + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS_INIT); + + //USER 8192 memory cycles for DLL to lock. + // 8192 cycles are required by Renesas LLDRAM-II, though we don't officially support it + delay_for_n_mem_clocks(8192); + + //USER Refresh all banks + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_REF_X8); + + //USER 1024 memory cycles + delay_for_n_mem_clocks(1024); +} +#endif + +#if RLDRAM3 +void rw_mgr_mem_initialize (void) +{ + TRACE_FUNC(); + alt_u32 r; + + // Here's how you load register for a loop + //USER Counters are located @ 0x800 + //USER Jump address are located @ 0xC00 + //USER For both, registers 0 to 3 are selected using bits 3 and 2, like in + //USER 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C + // I know this ain't pretty, but Avalon bus throws away the 2 least significant bits + + //USER start with memory RESET activated + + //USER tINIT = 200us + + //USER 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles + //USER If a and b are the number of iteration in 2 nested loops + //USER it takes the following number of cycles to complete the operation: + //USER number_of_cycles = ((2 + n) * a + 2) * b + //USER where n is the number of instruction in the inner loop + //USER One possible solution is n = 0 , a = 256 , b = 106 => a = FF, b = 6A + + //USER Load counters + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(0xFF)); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, SKIP_DELAY_LOOP_VALUE_OR_ZERO(0x6A)); + + //USER Load jump address + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_INIT_RESET_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_INIT_RESET_0_inloop); + + //USER Execute count instruction + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_INIT_RESET_0); + + //USER indicate that memory is stable + IOWR_32DIRECT (PHY_MGR_RESET_MEM_STBL, 0, 1); + + //USER transition the RESET to high + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_NOP); + + //USER Wait for 10000 cycles + delay_for_n_mem_clocks(10000); + + //USER Load MR0 + if ( RW_MGR_MEM_NUMBER_OF_RANKS == 1 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 2 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFC); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 4 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFC); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + //USER Wait MRSC + delay_for_n_mem_clocks(12); + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xF3); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_QUAD_RANK); + } + else { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0); + } + + + //USER Wait MRSC + delay_for_n_mem_clocks(12); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + continue; + } + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + //USER Load MR1 (reset DLL reset and kick off long ZQ calibration) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_CALIB); + + //USER Wait 512 cycles for DLL to reset and for ZQ calibration to complete + delay_for_n_mem_clocks(512); + } + + //USER Load MR2 (set write protocol to Single Bank) + if ( RW_MGR_MEM_NUMBER_OF_RANKS == 1 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 2 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFC); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 4 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xF0); + } + else { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + } + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2_CALIB); + + //USER Wait MRSC and a bit more + delay_for_n_mem_clocks(64); +} +#endif + +//USER At the end of calibration we have to program the user settings in, and +//USER hand off the memory to the user. + +#if DDR3 +void rw_mgr_mem_handoff (void) +{ + alt_u32 r; +#if LRDIMM + alt_u32 rtt_nom; + alt_u32 rtt_drv; + alt_u32 rtt_wr; +#endif // LRDIMM + + TRACE_FUNC(); + +#if LRDIMM + rtt_nom = LRDIMM_SPD_MR_RTT_NOM(LRDIMM_SPD_MR); + rtt_drv = LRDIMM_SPD_MR_RTT_DRV(LRDIMM_SPD_MR); + rtt_wr = LRDIMM_SPD_MR_RTT_WR(LRDIMM_SPD_MR); + + // USER Configure LRDIMM to broadcast LRDIMM MRS commands to all ranks + // USER Set bit F0RC14.DBA0 to '0' so MRS commands target all physical ranks in a logical rank + rw_mgr_lrdimm_rc_program(0, 14, (((RDIMM_CONFIG_WORD_HIGH >> 24) & 0xF) & (~0x4))); + + // USER Update contents of AC ROM with new RTT WR, DRV values + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs1, rtt_drv, 0, 1); + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs1, rtt_drv, 1, 1); + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs2, rtt_wr, 0, 2); + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs2, rtt_wr, 1, 2); +#endif // LRDIMM + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + +#if MRS_MIRROR_PING_PONG_ATSO + // Side 0 + set_rank_and_odt_mask_for_ping_pong_atso(0, RW_MGR_ODT_MODE_OFF); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_USER); + + // Side 1 + set_rank_and_odt_mask_for_ping_pong_atso(1, RW_MGR_ODT_MODE_OFF); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_USER_MIRR); + + // Unmask all CS + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); +#else + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER precharge all banks ... + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER load up MR settings specified by user + + //USER Use Mirror-ed commands for odd ranks if address mirrorring is on + if((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_MIRR); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_USER_MIRR); + } else { + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS3); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS0_USER); + } +#endif + //USER need to wait tMOD (12CK or 15ns) time before issuing other commands, + //USER but we will have plenty of NIOS cycles before actual handoff so its okay. + } + + +#if LRDIMM + delay_for_n_mem_clocks(12); + // USER Set up targetted MRS commands + rw_mgr_lrdimm_rc_program(0, 14, (((RDIMM_CONFIG_WORD_HIGH >> 24) & 0xF) | 0x4)); + // USER update AC ROM MR1 entry to include RTT_NOM for physical ranks 0,1 only + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs1, (rtt_drv|rtt_nom), 0, 1); + rtt_change_MRS1_MRS2_NOM_WR (__RW_MGR_CONTENT_ac_mrs1, (rtt_drv|rtt_nom), 1, 1); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER Use Mirror-ed commands for odd ranks if address mirrorring is on + if((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) { + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1_MIRR); + delay_for_n_mem_clocks(4); + } else { + delay_for_n_mem_clocks(4); + set_jump_as_return(); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + delay_for_n_mem_clocks(4); + } + } +#endif // LRDIMM +} +#endif // DDR3 + +#if DDR2 +void rw_mgr_mem_handoff (void) +{ + alt_u32 r; + + TRACE_FUNC(); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER precharge all banks ... + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER load up MR settings specified by user + + // FIXME BEN: for HHP, we need to add delay loops to be sure + // We can check this with BFM perhaps + // Likely enough delay in RW_MGR though + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR2); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR3); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR_USER); + + //USER need to wait tMOD (12CK or 15ns) time before issuing other commands, + //USER but we will have plenty of NIOS cycles before actual handoff so its okay. + } +} +#endif //USER DDR2 + +#if LPDDR2 +void rw_mgr_mem_handoff (void) +{ + alt_u32 r; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER precharge all banks... + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER load up MR settings specified by user + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR1_USER); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR2); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR3); + } +} +#endif //USER LPDDR2 + +#if LPDDR1 +void rw_mgr_mem_handoff (void) +{ + alt_u32 r; + + TRACE_FUNC(); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER precharge all banks ... + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + //USER load up MR settings specified by user + + // FIXME BEN: for HHP, we need to add delay loops to be sure + // We can check this with BFM perhaps + // Likely enough delay in RW_MGR though + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_EMR); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MR_USER); + + //USER need to wait tMOD (12CK or 15ns) time before issuing other commands, + //USER but we will have plenty of NIOS cycles before actual handoff so its okay. + } +} +#endif //USER LPDDR1 + +#if RLDRAMII +void rw_mgr_mem_handoff (void) +{ + TRACE_FUNC(); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS); +} +#endif + +#if RLDRAM3 +void rw_mgr_mem_handoff (void) +{ + TRACE_FUNC(); + alt_u32 r; + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + continue; + } + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER Load user requested MR1 + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS1); + } + + if ( RW_MGR_MEM_NUMBER_OF_RANKS == 1 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 2 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFC); + } else if ( RW_MGR_MEM_NUMBER_OF_RANKS == 4 ) { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xF0); + } + else { + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, 0xFE); + } + //USER Load user requested MR2 + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_MRS2); + + //USER Wait MRSC and a bit more + delay_for_n_mem_clocks(64); +} +#endif + +#if QDRII +void rw_mgr_mem_handoff (void) +{ + TRACE_FUNC(); +} +#endif + +#if DDRX +//USER performs a guaranteed read on the patterns we are going to use during a read test to ensure memory works +alt_u32 rw_mgr_mem_calibrate_read_test_patterns (alt_u32 rank_bgn, alt_u32 group, alt_u32 num_tries, t_btfld *bit_chk, alt_u32 all_ranks) +{ + alt_u32 r, vg; + t_btfld correct_mask_vg; + t_btfld tmp_bit_chk; + alt_u32 rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : (rank_bgn + NUM_RANKS_PER_SHADOW_REG); + + *bit_chk = param->read_correct_mask; + correct_mask_vg = param->read_correct_mask_vg; + + for (r = rank_bgn; r < rank_end; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); + + //USER Load up a constant bursts of read commands + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_GUARANTEED_READ); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_GUARANTEED_READ_CONT); + + tmp_bit_chk = 0; + for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) + { + //USER reset the fifos to get pointers to known state + + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + + tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, ((group*RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS+vg) << 2), __RW_MGR_GUARANTEED_READ); + tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(IORD_32DIRECT(BASE_RW_MGR, 0))); + + if (vg == 0) { + break; + } + } + *bit_chk &= tmp_bit_chk; + } + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, (group << 2), __RW_MGR_CLEAR_DQS_ENABLE); + + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); + DPRINT(2, "test_load_patterns(%lu,ALL) => (%lu == %lu) => %lu", group, *bit_chk, param->read_correct_mask, (long unsigned int)(*bit_chk == param->read_correct_mask)); + return (*bit_chk == param->read_correct_mask); +} + +alt_u32 rw_mgr_mem_calibrate_read_test_patterns_all_ranks (alt_u32 group, alt_u32 num_tries, t_btfld *bit_chk) +{ + if (rw_mgr_mem_calibrate_read_test_patterns (0, group, num_tries, bit_chk, 1)) + { + return 1; + } + else + { + // case:139851 - if guaranteed read fails, we can retry using different dqs enable phases. + // It is possible that with the initial phase, dqs enable is asserted/deasserted too close + // to an dqs edge, truncating the read burst. + alt_u32 p; + for (p = 0; p <= IO_DQS_EN_PHASE_MAX; p++) { + scc_mgr_set_dqs_en_phase_all_ranks (group, p); + if (rw_mgr_mem_calibrate_read_test_patterns (0, group, num_tries, bit_chk, 1)) + { + return 1; + } + } + return 0; + } +} +#endif + +//USER load up the patterns we are going to use during a read test +#if DDRX +void rw_mgr_mem_calibrate_read_load_patterns (alt_u32 rank_bgn, alt_u32 all_ranks) +{ + alt_u32 r; + alt_u32 rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : (rank_bgn + NUM_RANKS_PER_SHADOW_REG); + + TRACE_FUNC(); + + for (r = rank_bgn; r < rank_end; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); + + //USER Load up a constant bursts + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_GUARANTEED_WRITE_WAIT0); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_GUARANTEED_WRITE_WAIT1); + +#if QUARTER_RATE + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x01); +#endif +#if HALF_RATE + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x02); +#endif +#if FULL_RATE + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x04); +#endif + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_GUARANTEED_WRITE_WAIT2); + +#if QUARTER_RATE + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x01); +#endif +#if HALF_RATE + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x02); +#endif +#if FULL_RATE + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x04); +#endif + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_GUARANTEED_WRITE_WAIT3); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_GUARANTEED_WRITE); + } + + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); +} +#endif + +#if QDRII +void rw_mgr_mem_calibrate_read_load_patterns (alt_u32 rank_bgn, alt_u32 all_ranks) +{ + TRACE_FUNC(); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_GUARANTEED_WRITE_WAIT0); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_GUARANTEED_WRITE_WAIT1); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_GUARANTEED_WRITE); +} +#endif + +#if RLDRAMX +void rw_mgr_mem_calibrate_read_load_patterns (alt_u32 rank_bgn, alt_u32 all_ranks) +{ + TRACE_FUNC(); + alt_u32 r; + alt_u32 rank_end = RW_MGR_MEM_NUMBER_OF_RANKS;//all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : (rank_bgn + NUM_RANKS_PER_SHADOW_REG); +#if QUARTER_RATE + alt_u32 write_data_cycles = 0x10; +#else + alt_u32 write_data_cycles = 0x20; +#endif + + for (r = rank_bgn; r < rank_end; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, write_data_cycles); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_GUARANTEED_WRITE_WAIT0); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, write_data_cycles); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_GUARANTEED_WRITE_WAIT1); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, write_data_cycles); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_GUARANTEED_WRITE_WAIT2); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, write_data_cycles); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_GUARANTEED_WRITE_WAIT3); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_GUARANTEED_WRITE); + } + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); +} +#endif + +static inline void rw_mgr_mem_calibrate_read_load_patterns_all_ranks (void) +{ + rw_mgr_mem_calibrate_read_load_patterns (0, 1); +} + + +// pe checkout pattern for harden managers +//void pe_checkout_pattern (void) +//{ +// // test RW manager +// +// // do some reads to check load buffer +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_READ_B2B_WAIT1); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_READ_B2B_WAIT2); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_READ_B2B); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_READ_B2B); +// +// // clear error word +// IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); +// +// IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_READ_B2B); +// +// alt_u32 readdata; +// +// // read error word +// readdata = IORD_32DIRECT(BASE_RW_MGR, 0); +// +// // read DI buffer +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 0*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 1*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 2*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 3*4, 0); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_READ_B2B_WAIT1); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_READ_B2B_WAIT2); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_READ_B2B); +// +// IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x0); +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_READ_B2B); +// +// // clear error word +// IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); +// +// // do read +// IOWR_32DIRECT (RW_MGR_LOOPBACK_MODE, 0, __RW_MGR_READ_B2B); +// +// // read error word +// readdata = IORD_32DIRECT(BASE_RW_MGR, 0); +// +// // error word should be 0x00 +// +// // read DI buffer +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 0*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 1*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 2*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 3*4, 0); +// +// // clear error word +// IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); +// +// // do dm read +// IOWR_32DIRECT (RW_MGR_LOOPBACK_MODE, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1); +// +// // read error word +// readdata = IORD_32DIRECT(BASE_RW_MGR, 0); +// +// // error word should be ff +// +// // read DI buffer +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 0*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 1*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 2*4, 0); +// readdata = IORD_32DIRECT(RW_MGR_DI_BASE + 3*4, 0); +// +// // exit loopback mode +// IOWR_32DIRECT (BASE_RW_MGR, 0, __RW_MGR_IDLE_LOOP2); +// +// // start of phy manager access +// +// readdata = IORD_32DIRECT (PHY_MGR_MAX_RLAT_WIDTH, 0); +// readdata = IORD_32DIRECT (PHY_MGR_MAX_AFI_WLAT_WIDTH, 0); +// readdata = IORD_32DIRECT (PHY_MGR_MAX_AFI_RLAT_WIDTH, 0); +// readdata = IORD_32DIRECT (PHY_MGR_CALIB_SKIP_STEPS, 0); +// readdata = IORD_32DIRECT (PHY_MGR_CALIB_VFIFO_OFFSET, 0); +// readdata = IORD_32DIRECT (PHY_MGR_CALIB_LFIFO_OFFSET, 0); +// +// // start of data manager test +// +// readdata = IORD_32DIRECT (DATA_MGR_DRAM_CFG , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_WL , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_ADD , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_RL , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_RFC , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_REFI , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_WR , 0); +// readdata = IORD_32DIRECT (DATA_MGR_MEM_T_MRD , 0); +// readdata = IORD_32DIRECT (DATA_MGR_COL_WIDTH , 0); +// readdata = IORD_32DIRECT (DATA_MGR_ROW_WIDTH , 0); +// readdata = IORD_32DIRECT (DATA_MGR_BANK_WIDTH , 0); +// readdata = IORD_32DIRECT (DATA_MGR_CS_WIDTH , 0); +// readdata = IORD_32DIRECT (DATA_MGR_ITF_WIDTH , 0); +// readdata = IORD_32DIRECT (DATA_MGR_DVC_WIDTH , 0); +// +//} + +//USER try a read and see if it returns correct data back. has dummy reads inserted into the mix +//USER used to align dqs enable. has more thorough checks than the regular read test. + +alt_u32 rw_mgr_mem_calibrate_read_test (alt_u32 rank_bgn, alt_u32 group, alt_u32 num_tries, alt_u32 all_correct, t_btfld *bit_chk, alt_u32 all_groups, alt_u32 all_ranks) +{ + alt_u32 r, vg; + t_btfld correct_mask_vg; + t_btfld tmp_bit_chk; + alt_u32 rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : (rank_bgn + NUM_RANKS_PER_SHADOW_REG); + +#if LRDIMM + // USER Disable MB Write-levelling mode and enter normal operation + rw_mgr_lrdimm_rc_program(0,12,0x0); +#endif + + *bit_chk = param->read_correct_mask; + correct_mask_vg = param->read_correct_mask_vg; + + alt_u32 quick_read_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION) || BFM_MODE; + + for (r = rank_bgn; r < rank_end; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x10); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_READ_B2B_WAIT1); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x10); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_READ_B2B_WAIT2); + + if(quick_read_mode) { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x1); /* need at least two (1+1) reads to capture failures */ + } else if (all_groups) { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x06); + } else { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x32); + } + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_READ_B2B); + if(all_groups) { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, RW_MGR_MEM_IF_READ_DQS_WIDTH * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1); + } else { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x0); + } + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_READ_B2B); + + tmp_bit_chk = 0; + for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) + { + //USER reset the fifos to get pointers to known state + + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + + tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS); + + IOWR_32DIRECT (all_groups ? RW_MGR_RUN_ALL_GROUPS : RW_MGR_RUN_SINGLE_GROUP, ((group*RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS+vg) << 2), __RW_MGR_READ_B2B); + tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(IORD_32DIRECT(BASE_RW_MGR, 0))); + + if (vg == 0) { + break; + } + } + *bit_chk &= tmp_bit_chk; + } + +#if ENABLE_BRINGUP_DEBUGGING + load_di_buf_gbl(); +#endif + + #if DDRX + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, (group << 2), __RW_MGR_CLEAR_DQS_ENABLE); + #endif + + if (all_correct) + { + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); + DPRINT(2, "read_test(%lu,ALL,%lu) => (%lu == %lu) => %lu", group, all_groups, *bit_chk, param->read_correct_mask, (long unsigned int)(*bit_chk == param->read_correct_mask)); + return (*bit_chk == param->read_correct_mask); + } + else + { + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); + DPRINT(2, "read_test(%lu,ONE,%lu) => (%lu != %lu) => %lu", group, all_groups, *bit_chk, (long unsigned int)0, (long unsigned int)(*bit_chk != 0x00)); + return (*bit_chk != 0x00); + } +} + +static inline alt_u32 rw_mgr_mem_calibrate_read_test_all_ranks (alt_u32 group, alt_u32 num_tries, alt_u32 all_correct, t_btfld *bit_chk, alt_u32 all_groups) +{ + return rw_mgr_mem_calibrate_read_test (0, group, num_tries, all_correct, bit_chk, all_groups, 1); +} + +#if ENABLE_DELAY_CHAIN_WRITE +void rw_mgr_incr_vfifo_auto(alt_u32 grp) { + alt_u32 v; + v = vfifo_settings[grp]%VFIFO_SIZE; + rw_mgr_incr_vfifo(grp, &v); + vfifo_settings[grp] = v; +} + +void rw_mgr_decr_vfifo_auto(alt_u32 grp) { + alt_u32 v; + v = vfifo_settings[grp]%VFIFO_SIZE; + rw_mgr_decr_vfifo(grp, &v); + vfifo_settings[grp] = v; +} +#endif // ENABLE_DELAY_CHAIN_WRITE + +void rw_mgr_incr_vfifo(alt_u32 grp, alt_u32 *v) { + //USER fiddle with FIFO + if(HARD_PHY) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_HARD_PHY, 0, grp); + } else if (QUARTER_RATE_MODE && !HARD_VFIFO) { + if ((*v & 3) == 3) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_QR, 0, grp); + } else if ((*v & 2) == 2) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR_HR, 0, grp); + } else if ((*v & 1) == 1) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_HR, 0, grp); + } else { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR, 0, grp); + } + } else if (HARD_VFIFO) { + // Arria V & Cyclone V have a hard full-rate VFIFO that only has a single incr signal + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR, 0, grp); + } + else { + if (!HALF_RATE_MODE || (*v & 1) == 1) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_HR, 0, grp); + } else { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR, 0, grp); + } + } + + (*v)++; +#if USE_DQS_TRACKING && !HHP_HPS + IOWR_32DIRECT (TRK_V_POINTER, (grp << 2), *v); +#endif + BFM_INC_VFIFO; +} + +//Used in quick cal to properly loop through the duplicated VFIFOs in AV QDRII/RLDRAM +static inline void rw_mgr_incr_vfifo_all(alt_u32 grp, alt_u32 *v) { +#if VFIFO_CONTROL_WIDTH_PER_DQS == 1 + rw_mgr_incr_vfifo(grp, v); +#else + alt_u32 i; + for(i = 0; i < VFIFO_CONTROL_WIDTH_PER_DQS; i++) { + rw_mgr_incr_vfifo(grp*VFIFO_CONTROL_WIDTH_PER_DQS+i, v); + if(i != 0) { + (*v)--; + } + } +#endif +} + +void rw_mgr_decr_vfifo(alt_u32 grp, alt_u32 *v) { + + alt_u32 i; + + for (i = 0; i < VFIFO_SIZE-1; i++) { + rw_mgr_incr_vfifo(grp, v); + } +} + +//USER find a good dqs enable to use + +#if QDRII || RLDRAMX +alt_u32 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase (alt_u32 grp) +{ + alt_u32 v; + alt_u32 found; + alt_u32 dtaps_per_ptap, tmp_delay; + t_btfld bit_chk; + + TRACE_FUNC("%lu", grp); + + reg_file_set_sub_stage(CAL_SUBSTAGE_DQS_EN_PHASE); + + found = 0; + + //USER first push vfifo until we get a passing read + for (v = 0; v < VFIFO_SIZE && found == 0;) { + DPRINT(2, "find_dqs_en_phase: vfifo %lu", BFM_GBL_GET(vfifo_idx)); + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found = 1; + } + + if (!found) { + //USER fiddle with FIFO +#if (VFIFO_CONTROL_WIDTH_PER_DQS != 1) + alt_u32 i; + for (i = 0; i < VFIFO_CONTROL_WIDTH_PER_DQS; i++) { + rw_mgr_incr_vfifo(grp*VFIFO_CONTROL_WIDTH_PER_DQS+i, &v); + v--; // undo increment of v in rw_mgr_incr_vfifo + } + v++; // add back single increment +#else + rw_mgr_incr_vfifo(grp, &v); +#endif + } + } + +#if (VFIFO_CONTROL_WIDTH_PER_DQS != 1) + if (found) { + // we found a vfifo setting that works for at least one vfifo "group" + // Some groups may need next vfifo setting, so check each one to + // see if we get new bits passing by increment the vfifo + alt_u32 i; + t_btfld best_bit_chk_inv; + alt_u8 found_on_first_check = (v == 1); + + best_bit_chk_inv = ~bit_chk; + + for (i = 0; i < VFIFO_CONTROL_WIDTH_PER_DQS; i++) { + rw_mgr_incr_vfifo(grp*VFIFO_CONTROL_WIDTH_PER_DQS+i, &v); + v--; // undo increment of v in rw_mgr_incr_vfifo, just in case it matters for next check + rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0); + if ((bit_chk & best_bit_chk_inv) != 0) { + // found some new bits + best_bit_chk_inv = ~bit_chk; + } else { + // no improvement, so put back + rw_mgr_decr_vfifo(grp*VFIFO_CONTROL_WIDTH_PER_DQS+i, &v); + v++; + if (found_on_first_check) { + // found on first vfifo check, so we also need to check earlier vfifo values + rw_mgr_decr_vfifo(grp*VFIFO_CONTROL_WIDTH_PER_DQS+i, &v); + v++; // undo decrement of v in rw_mgr_incr_vfifo, just in case it matters for next check + rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0); + if ((bit_chk & best_bit_chk_inv) != 0) { + // found some new bits + best_bit_chk_inv = ~bit_chk; + } else { + // no improvement, so put back + rw_mgr_incr_vfifo(grp*VFIFO_CONTROL_WIDTH_PER_DQS+i, &v); + v--; + } + } // found_on_first_check + } // check for new bits + } // loop over all vfifo control bits + } +#endif + + if (found) { + DPRINT(2, "find_dqs_en_phase: found vfifo=%lu", BFM_GBL_GET(vfifo_idx)); + // Not really dqs_enable left/right edge, but close enough for testing purposes + BFM_GBL_SET(dqs_enable_left_edge[grp].v,BFM_GBL_GET(vfifo_idx)); + BFM_GBL_SET(dqs_enable_right_edge[grp].v,BFM_GBL_GET(vfifo_idx)); + BFM_GBL_SET(dqs_enable_mid[grp].v,BFM_GBL_GET(vfifo_idx)); + } else { + DPRINT(2, "find_dqs_en_phase: no valid vfifo found"); + } + +#if ENABLE_TCL_DEBUG + // FIXME: Not a dynamically calculated value for dtaps_per_ptap + dtaps_per_ptap = 0; + tmp_delay = 0; + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP; + } + dtaps_per_ptap--; + ALTERA_ASSERT(dtaps_per_ptap <= IO_DQS_EN_DELAY_MAX); + + TCLRPT_SET(debug_summary_report->computed_dtap_per_ptap, dtaps_per_ptap); +#endif + + return found; +} +#endif + +#if DDRX +#if NEWVERSION_DQSEN + +// Navid's version + +alt_u32 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase (alt_u32 grp) +{ + alt_u32 i, d, v, p, sr; + alt_u32 max_working_cnt; + alt_u32 fail_cnt; + t_btfld bit_chk; + alt_u32 dtaps_per_ptap; + alt_u32 found_begin, found_end; + alt_u32 work_bgn, work_mid, work_end, tmp_delay; + alt_u32 test_status; + alt_u32 found_passing_read, found_failing_read, initial_failing_dtap; +#if RUNTIME_CAL_REPORT + alt_u32 start_v[NUM_SHADOW_REGS], start_p[NUM_SHADOW_REGS], start_d[NUM_SHADOW_REGS]; + alt_u32 end_v[NUM_SHADOW_REGS], end_p[NUM_SHADOW_REGS], end_d[NUM_SHADOW_REGS]; + for(sr = 0; sr < NUM_SHADOW_REGS; sr++) { + start_v[sr] = 0; + start_p[sr] = 0; + start_d[sr] = 0; + } +#endif + + TRACE_FUNC("%lu", grp); + BFM_STAGE("find_dqs_en_phase"); + ALTERA_ASSERT(grp < RW_MGR_MEM_IF_READ_DQS_WIDTH); + + reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); + + scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); + scc_mgr_set_dqs_en_phase_all_ranks(grp, 0); + + fail_cnt = 0; + + //USER ************************************************************** + //USER * Step 0 : Determine number of delay taps for each phase tap * + + dtaps_per_ptap = 0; + tmp_delay = 0; + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP; + } + dtaps_per_ptap--; + ALTERA_ASSERT(dtaps_per_ptap <= IO_DQS_EN_DELAY_MAX); + tmp_delay = 0; + TCLRPT_SET(debug_summary_report->computed_dtap_per_ptap, dtaps_per_ptap); + + // VFIFO sweep +#if ENABLE_DQSEN_SWEEP + init_di_buffer(); + work_bgn = 0; + for (d = 0; d <= dtaps_per_ptap; d++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { + work_bgn = tmp_delay; + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + for (i = 0; i < VFIFO_SIZE; i++) { + for (p = 0; p <= IO_DQS_EN_PHASE_MAX; p++, work_bgn += IO_DELAY_PER_OPA_TAP) { + DPRINT(2, "find_dqs_en_phase: begin: vfifo=%lu ptap=%lu dtap=%lu", BFM_GBL_GET(vfifo_idx), p, d); + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + test_status = rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0); + + //if (p ==0 && d == 0) + sample_di_data(bit_chk, work_bgn, d, i, p); + } + //Increment FIFO + rw_mgr_incr_vfifo(grp, &v); + } + + work_bgn++; + } + flag_di_buffer_done(); +#endif + + //USER ********************************************************* + //USER * Step 1 : First push vfifo until we get a failing read * + for (v = 0; v < VFIFO_SIZE; ) { + DPRINT(2, "find_dqs_en_phase: vfifo %lu", BFM_GBL_GET(vfifo_idx)); + test_status = rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0); + if (!test_status) { + fail_cnt++; + + if (fail_cnt == 2) { + break; + } + } + + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + + if (v >= VFIFO_SIZE) { + //USER no failing read found!! Something must have gone wrong + DPRINT(2, "find_dqs_en_phase: vfifo failed"); + return 0; + } + + max_working_cnt = 0; + + //USER ******************************************************** + //USER * step 2: find first working phase, increment in ptaps * + found_begin = 0; + work_bgn = 0; + for (d = 0; d <= dtaps_per_ptap; d++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { + work_bgn = tmp_delay; + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + for (i = 0; i < VFIFO_SIZE; i++) { + for (p = 0; p <= IO_DQS_EN_PHASE_MAX; p++, work_bgn += IO_DELAY_PER_OPA_TAP) { + DPRINT(2, "find_dqs_en_phase: begin: vfifo=%lu ptap=%lu dtap=%lu", BFM_GBL_GET(vfifo_idx), p, d); + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + test_status = rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0); + + if (test_status) { + max_working_cnt = 1; + found_begin = 1; + break; + } + } + + if (found_begin) { + break; + } + + if (p > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + } + + if (found_begin) { + break; + } + } + + if (i >= VFIFO_SIZE) { + //USER cannot find working solution + DPRINT(2, "find_dqs_en_phase: no vfifo/ptap/dtap"); + return 0; + } + + work_end = work_bgn; + + //USER If d is 0 then the working window covers a phase tap and we can follow the old procedure + //USER otherwise, we've found the beginning, and we need to increment the dtaps until we find the end + if (d == 0) { + //USER ******************************************************************** + //USER * step 3a: if we have room, back off by one and increment in dtaps * + COV(EN_PHASE_PTAP_OVERLAP); + + //USER Special case code for backing up a phase + if (p == 0) { + p = IO_DQS_EN_PHASE_MAX ; + rw_mgr_decr_vfifo(grp, &v); + } else { + p = p - 1; + } + tmp_delay = work_bgn - IO_DELAY_PER_OPA_TAP; + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + found_begin = 0; + for (d = 0; d <= IO_DQS_EN_DELAY_MAX && tmp_delay < work_bgn; d++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { + + DPRINT(2, "find_dqs_en_phase: begin-2: vfifo=%lu ptap=%lu dtap=%lu", BFM_GBL_GET(vfifo_idx), p, d); + + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found_begin = 1; + work_bgn = tmp_delay; + break; + } + } + +#if BFM_MODE + { + alt_32 p2, v2, d2; + + // print out where the actual beginning is + if (found_begin) { + v2 = BFM_GBL_GET(vfifo_idx); + p2 = p; + d2 = d; + } else if (p == IO_DQS_EN_PHASE_MAX) { + v2 = (BFM_GBL_GET(vfifo_idx) + 1) % VFIFO_SIZE; + p2 = 0; + d2 = 0; + } else { + v2 = BFM_GBL_GET(vfifo_idx); + p2 = p + 1; + d2 = 0; + } + + DPRINT(2, "find_dqs_en_phase: begin found: vfifo=%lu ptap=%lu dtap=%lu begin=%lu", + v2, p2, d2, work_bgn); + BFM_GBL_SET(dqs_enable_left_edge[grp].v,v2); + BFM_GBL_SET(dqs_enable_left_edge[grp].p,p2); + BFM_GBL_SET(dqs_enable_left_edge[grp].d,d2); + BFM_GBL_SET(dqs_enable_left_edge[grp].ps,work_bgn); + } +#endif + // Record the debug data + // Currently dqsen is the same for all ranks + for (sr = 0; sr < NUM_SHADOW_REGS; sr++) + { + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].work_begin, work_bgn); + if (found_begin) + { + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].phase_begin, p); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].delay_begin, d); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].vfifo_begin, v % VFIFO_SIZE); +#if RUNTIME_CAL_REPORT + start_v[sr] = v % VFIFO_SIZE; + start_p[sr] = p; + start_d[sr] = d; +#endif + } + else if (p == IO_DQS_EN_PHASE_MAX) + { + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].phase_begin, 0); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].delay_begin, 0); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].vfifo_begin, (v+1) % VFIFO_SIZE); +#if RUNTIME_CAL_REPORT + start_v[sr] = (v+1) % VFIFO_SIZE; + start_p[sr] = p; + start_d[sr] = d; +#endif + } + else + { + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].phase_begin, p+1); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].delay_begin, 0); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].vfifo_begin, v % VFIFO_SIZE); +#if RUNTIME_CAL_REPORT + start_v[sr] = v % VFIFO_SIZE; + start_p[sr] = p+1; + start_d[sr] = d; +#endif + } + } + + //USER We have found a working dtap before the ptap found above + if (found_begin == 1) { + max_working_cnt++; + } + + //USER Restore VFIFO to old state before we decremented it (if needed) + p = p + 1; + if (p > IO_DQS_EN_PHASE_MAX) { + p = 0; + rw_mgr_incr_vfifo(grp, &v); + } + + scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); + + //USER *********************************************************************************** + //USER * step 4a: go forward from working phase to non working phase, increment in ptaps * + p = p + 1; + work_end += IO_DELAY_PER_OPA_TAP; + if (p > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + p = 0; + rw_mgr_incr_vfifo(grp, &v); + } + + found_end = 0; + for (; i < VFIFO_SIZE + 1; i++) { + for (; p <= IO_DQS_EN_PHASE_MAX; p++, work_end += IO_DELAY_PER_OPA_TAP) { + DPRINT(2, "find_dqs_en_phase: end: vfifo=%lu ptap=%lu dtap=%lu", BFM_GBL_GET(vfifo_idx), p, (long unsigned int)0); + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found_end = 1; + break; + } else { + max_working_cnt++; + } + } + + if (found_end) { + break; + } + + if (p > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + p = 0; + } + } + + if (i >= VFIFO_SIZE + 1) { + //USER cannot see edge of failing read + DPRINT(2, "find_dqs_en_phase: end: failed"); + return 0; + } + + //USER ********************************************************* + //USER * step 5a: back off one from last, increment in dtaps * + + //USER Special case code for backing up a phase + if (p == 0) { + p = IO_DQS_EN_PHASE_MAX; + rw_mgr_decr_vfifo(grp, &v); + } else { + p = p - 1; + } + + work_end -= IO_DELAY_PER_OPA_TAP; + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + //USER * The actual increment of dtaps is done outside of the if/else loop to share code + d = 0; + + DPRINT(2, "find_dqs_en_phase: found end v/p: vfifo=%lu ptap=%lu", BFM_GBL_GET(vfifo_idx), p); + } else { + + //USER ******************************************************************** + //USER * step 3-5b: Find the right edge of the window using delay taps * + COV(EN_PHASE_PTAP_NO_OVERLAP); + + DPRINT(2, "find_dqs_en_phase: begin found: vfifo=%lu ptap=%lu dtap=%lu begin=%lu", BFM_GBL_GET(vfifo_idx), p, d, work_bgn); + BFM_GBL_SET(dqs_enable_left_edge[grp].v,BFM_GBL_GET(vfifo_idx)); + BFM_GBL_SET(dqs_enable_left_edge[grp].p,p); + BFM_GBL_SET(dqs_enable_left_edge[grp].d,d); + BFM_GBL_SET(dqs_enable_left_edge[grp].ps,work_bgn); + + work_end = work_bgn; + + //USER * The actual increment of dtaps is done outside of the if/else loop to share code + + //USER Only here to counterbalance a subtract later on which is not needed if this branch + //USER of the algorithm is taken + max_working_cnt++; + } + + //USER The dtap increment to find the failing edge is done here + for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) { + + DPRINT(2, "find_dqs_en_phase: end-2: dtap=%lu", d); + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + } + + //USER Go back to working dtap + if (d != 0) { + work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP; + } + + DPRINT(2, "find_dqs_en_phase: found end v/p/d: vfifo=%lu ptap=%lu dtap=%lu end=%lu", BFM_GBL_GET(vfifo_idx), p, d-1, work_end); + BFM_GBL_SET(dqs_enable_right_edge[grp].v,BFM_GBL_GET(vfifo_idx)); + BFM_GBL_SET(dqs_enable_right_edge[grp].p,p); + BFM_GBL_SET(dqs_enable_right_edge[grp].d,d-1); + BFM_GBL_SET(dqs_enable_right_edge[grp].ps,work_end); + + // Record the debug data + for (sr = 0; sr < NUM_SHADOW_REGS; sr++) + { + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].work_end, work_end); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].phase_end, p); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].delay_end, d-1); + TCLRPT_SET(debug_cal_report->cal_dqsen_margins[sr][grp].vfifo_end, v % VFIFO_SIZE); +#if RUNTIME_CAL_REPORT + end_v[sr] = v % VFIFO_SIZE; + end_p[sr] = p; + end_d[sr] = d-1; +#endif + } + + if (work_end >= work_bgn) { + //USER we have a working range + } else { + //USER nil range + DPRINT(2, "find_dqs_en_phase: end-2: failed"); + return 0; + } + + DPRINT(2, "find_dqs_en_phase: found range [%lu,%lu]", work_bgn, work_end); + +#if USE_DQS_TRACKING + // *************************************************************** + //USER * We need to calculate the number of dtaps that equal a ptap + //USER * To do that we'll back up a ptap and re-find the edge of the + //USER * window using dtaps + + DPRINT(2, "find_dqs_en_phase: calculate dtaps_per_ptap for tracking"); + + //USER Special case code for backing up a phase + if (p == 0) { + p = IO_DQS_EN_PHASE_MAX; + rw_mgr_decr_vfifo(grp, &v); + DPRINT(2, "find_dqs_en_phase: backed up cycle/phase: v=%lu p=%lu", BFM_GBL_GET(vfifo_idx), p); + } else { + p = p - 1; + DPRINT(2, "find_dqs_en_phase: backed up phase only: v=%lu p=%lu", BFM_GBL_GET(vfifo_idx), p); + } + + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + //USER Increase dtap until we first see a passing read (in case the window is smaller than a ptap), + //USER and then a failing read to mark the edge of the window again + + //USER Find a passing read + DPRINT(2, "find_dqs_en_phase: find passing read"); + found_passing_read = 0; + found_failing_read = 0; + initial_failing_dtap = d; + for (; d <= IO_DQS_EN_DELAY_MAX; d++) { + DPRINT(2, "find_dqs_en_phase: testing read d=%lu", d); + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found_passing_read = 1; + break; + } + } + + if (found_passing_read) { + //USER Find a failing read + DPRINT(2, "find_dqs_en_phase: find failing read"); + for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) { + DPRINT(2, "find_dqs_en_phase: testing read d=%lu", d); + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found_failing_read = 1; + break; + } + } + } else { + DPRINT(1, "find_dqs_en_phase: failed to calculate dtaps per ptap. Fall back on static value"); + } + + //USER The dynamically calculated dtaps_per_ptap is only valid if we found a passing/failing read + //USER If we didn't, it means d hit the max (IO_DQS_EN_DELAY_MAX). + //USER Otherwise, dtaps_per_ptap retains its statically calculated value. + if(found_passing_read && found_failing_read) { + dtaps_per_ptap = d - initial_failing_dtap; + } + + ALTERA_ASSERT(dtaps_per_ptap <= IO_DQS_EN_DELAY_MAX); +#if HHP_HPS + IOWR_32DIRECT (REG_FILE_DTAPS_PER_PTAP, 0, dtaps_per_ptap); +#else + IOWR_32DIRECT (TRK_DTAPS_PER_PTAP, 0, dtaps_per_ptap); +#endif + + DPRINT(2, "find_dqs_en_phase: dtaps_per_ptap=%lu - %lu = %lu", d, initial_failing_dtap, dtaps_per_ptap); +#endif + + //USER ******************************************** + //USER * step 6: Find the centre of the window * + + work_mid = (work_bgn + work_end) / 2; + tmp_delay = 0; + + DPRINT(2, "work_bgn=%ld work_end=%ld work_mid=%ld", work_bgn, work_end, work_mid); + //USER Get the middle delay to be less than a VFIFO delay + for (p = 0; p <= IO_DQS_EN_PHASE_MAX; p++, tmp_delay += IO_DELAY_PER_OPA_TAP); + DPRINT(2, "vfifo ptap delay %ld", tmp_delay); + while(work_mid > tmp_delay) work_mid -= tmp_delay; + DPRINT(2, "new work_mid %ld", work_mid); + tmp_delay = 0; + for (p = 0; p <= IO_DQS_EN_PHASE_MAX && tmp_delay < work_mid; p++, tmp_delay += IO_DELAY_PER_OPA_TAP); + tmp_delay -= IO_DELAY_PER_OPA_TAP; + DPRINT(2, "new p %ld, tmp_delay=%ld", p-1, tmp_delay); + for (d = 0; d <= IO_DQS_EN_DELAY_MAX && tmp_delay < work_mid; d++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP); + DPRINT(2, "new d %ld, tmp_delay=%ld", d, tmp_delay); + + // DQSEN same for all shadow reg + for(sr = 0; sr < NUM_SHADOW_REGS; sr++) { + TCLRPT_SET(debug_cal_report->cal_dqs_in_margins[sr][grp].dqsen_margin, max_working_cnt -1); + } + + scc_mgr_set_dqs_en_phase_all_ranks(grp, p-1); + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + //USER push vfifo until we can successfully calibrate. We can do this because + //USER the largest possible margin in 1 VFIFO cycle + + for (i = 0; i < VFIFO_SIZE; i++) { + DPRINT(2, "find_dqs_en_phase: center: vfifo=%lu", BFM_GBL_GET(vfifo_idx)); + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + + if (i >= VFIFO_SIZE) { + DPRINT(2, "find_dqs_en_phase: center: failed"); + return 0; + } +#if RUNTIME_CAL_REPORT + for(sr = 0; sr < NUM_SHADOW_REGS; sr++) { + RPRINT("DQS Enable ; Group %lu ; Rank %lu ; Start VFIFO %2li ; Phase %li ; Delay %2li", grp, sr, start_v[sr], start_p[sr], start_d[sr]); + RPRINT("DQS Enable ; Group %lu ; Rank %lu ; End VFIFO %2li ; Phase %li ; Delay %2li", grp, sr, end_v[sr], end_p[sr], end_d[sr]); + // Case 174276: Normalizing VFIFO center + RPRINT("DQS Enable ; Group %lu ; Rank %lu ; Center VFIFO %2li ; Phase %li ; Delay %2li", grp, sr, (v % VFIFO_SIZE), p-1, d); + } +#endif + DPRINT(2, "find_dqs_en_phase: center found: vfifo=%li ptap=%lu dtap=%lu", BFM_GBL_GET(vfifo_idx), p-1, d); + #if ENABLE_DELAY_CHAIN_WRITE + vfifo_settings[grp] = v; + #endif // ENABLE_DELAY_CHAIN_WRITE + BFM_GBL_SET(dqs_enable_mid[grp].v,BFM_GBL_GET(vfifo_idx)); + BFM_GBL_SET(dqs_enable_mid[grp].p,p-1); + BFM_GBL_SET(dqs_enable_mid[grp].d,d); + BFM_GBL_SET(dqs_enable_mid[grp].ps,work_mid); + return 1; +} + +#if 0 +// Ryan's algorithm + +alt_u32 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase (alt_u32 grp) +{ + alt_u32 i, d, v, p; + alt_u32 min_working_p, max_working_p, min_working_d, max_working_d, max_working_cnt; + alt_u32 fail_cnt; + t_btfld bit_chk; + alt_u32 dtaps_per_ptap; + alt_u32 found_begin, found_end; + alt_u32 tmp_delay; + + TRACE_FUNC("%lu", grp); + + reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); + + scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); + scc_mgr_set_dqs_en_phase_all_ranks(grp, 0); + + fail_cnt = 0; + + //USER ************************************************************** + //USER * Step 0 : Determine number of delay taps for each phase tap * + + dtaps_per_ptap = 0; + tmp_delay = 0; + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP; + } + dtaps_per_ptap--; + + //USER ********************************************************* + //USER * Step 1 : First push vfifo until we get a failing read * + for (v = 0; v < VFIFO_SIZE; ) { + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + fail_cnt++; + + if (fail_cnt == 2) { + break; + } + } + + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + + if (i >= VFIFO_SIZE) { + //USER no failing read found!! Something must have gone wrong + return 0; + } + + max_working_cnt = 0; + min_working_p = 0; + + //USER ******************************************************** + //USER * step 2: find first working phase, increment in ptaps * + found_begin = 0; + for (d = 0; d <= dtaps_per_ptap; d++) { + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + for (i = 0; i < VFIFO_SIZE; i++) { + for (p = 0; p <= IO_DQS_EN_PHASE_MAX; p++) { + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + max_working_cnt = 1; + found_begin = 1; + break; + } + } + + if (found_begin) { + break; + } + + if (p > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + } + + if (found_begin) { + break; + } + } + + if (i >= VFIFO_SIZE) { + //USER cannot find working solution + return 0; + } + + min_working_p = p; + + //USER If d is 0 then the working window covers a phase tap and we can follow the old procedure + //USER otherwise, we've found the beginning, and we need to increment the dtaps until we find the end + if (d == 0) { + //USER ******************************************************************** + //USER * step 3a: if we have room, back off by one and increment in dtaps * + min_working_d = 0; + + //USER Special case code for backing up a phase + if (p == 0) { + p = IO_DQS_EN_PHASE_MAX ; + rw_mgr_decr_vfifo(grp, &v); + } else { + p = p - 1; + } + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + found_begin = 0; + for (d = 0; d <= dtaps_per_ptap; d++) { + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found_begin = 1; + min_working_d = d; + break; + } + } + + //USER We have found a working dtap before the ptap found above + if (found_begin == 1) { + min_working_p = p; + max_working_cnt++; + } + + //USER Restore VFIFO to old state before we decremented it + p = p + 1; + if (p > IO_DQS_EN_PHASE_MAX) { + p = 0; + rw_mgr_incr_vfifo(grp, &v); + } + + scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); + + + //USER *********************************************************************************** + //USER * step 4a: go forward from working phase to non working phase, increment in ptaps * + p = p + 1; + if (p > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + p = 0; + rw_mgr_incr_vfifo(grp, &v); + } + + found_end = 0; + for (; i < VFIFO_SIZE+1; i++) { + for (; p <= IO_DQS_EN_PHASE_MAX; p++) { + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found_end = 1; + break; + } else { + max_working_cnt++; + } + } + + if (found_end) { + break; + } + + if (p > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + p = 0; + } + } + + if (i >= VFIFO_SIZE+1) { + //USER cannot see edge of failing read + return 0; + } + + //USER ********************************************************* + //USER * step 5a: back off one from last, increment in dtaps * + max_working_d = 0; + + //USER Special case code for backing up a phase + if (p == 0) { + p = IO_DQS_EN_PHASE_MAX; + rw_mgr_decr_vfifo(grp, &v); + } else { + p = p - 1; + } + + max_working_p = p; + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + + for (d = 0; d <= IO_DQS_EN_DELAY_MAX; d++) { + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + } + + //USER Go back to working dtap + if (d != 0) { + max_working_d = d - 1; + } + + } else { + + //USER ******************************************************************** + //USER * step 3-5b: Find the right edge of the window using delay taps * + + max_working_p = min_working_p; + min_working_d = d; + + for (; d <= IO_DQS_EN_DELAY_MAX; d++) { + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + } + + //USER Go back to working dtap + if (d != 0) { + max_working_d = d - 1; + } + + //USER Only here to counterbalance a subtract later on which is not needed if this branch + //USER of the algorithm is taken + max_working_cnt++; + } + + //USER ******************************************** + //USER * step 6: Find the centre of the window * + + //USER If the number of working phases is even we will step back a phase and find the + //USER edge with a larger delay chain tap + if ((max_working_cnt & 1) == 0) { + p = min_working_p + (max_working_cnt-1)/2; + + //USER Special case code for backing up a phase + if (max_working_p == 0) { + max_working_p = IO_DQS_EN_PHASE_MAX; + rw_mgr_decr_vfifo(grp, &v); + } else { + max_working_p = max_working_p - 1; + } + + scc_mgr_set_dqs_en_phase_all_ranks(grp, max_working_p); + + //USER Code to determine at which dtap we should start searching again for a failure + //USER If we've moved back such that the max and min p are the same, we should start searching + //USER from where the window actually exists + if (max_working_p == min_working_p) { + d = min_working_d; + } else { + d = max_working_d; + } + + for (; d <= IO_DQS_EN_DELAY_MAX; d++) { + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + } + + //USER Go back to working dtap + if (d != 0) { + max_working_d = d - 1; + } + } else { + p = min_working_p + (max_working_cnt)/2; + } + + while (p > IO_DQS_EN_PHASE_MAX) { + p -= (IO_DQS_EN_PHASE_MAX + 1); + } + + d = (min_working_d + max_working_d)/2; + + scc_mgr_set_dqs_en_phase_all_ranks(grp, p); + scc_mgr_set_dqs_en_delay_all_ranks(grp, d); + + //USER push vfifo until we can successfully calibrate + + for (i = 0; i < VFIFO_SIZE; i++) { + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + + if (i >= VFIFO_SIZE) { + return 0; + } + + return 1; +} + +#endif + +#else +// Val's original version + +alt_u32 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase (alt_u32 grp) +{ + alt_u32 i, j, v, d; + alt_u32 min_working_d, max_working_cnt; + alt_u32 fail_cnt; + t_btfld bit_chk; + alt_u32 delay_per_ptap_mid; + + TRACE_FUNC("%lu", grp); + + reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); + + scc_mgr_set_dqs_en_delay_all_ranks(grp, 0); + scc_mgr_set_dqs_en_phase_all_ranks(grp, 0); + + fail_cnt = 0; + + //USER first push vfifo until we get a failing read + v = 0; + for (i = 0; i < VFIFO_SIZE; i++) { + if (!rw_mgr_mem_calibrate_read_test_all_ranks (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) { + fail_cnt++; + + if (fail_cnt == 2) { + break; + } + } + + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } + + if (v >= VFIFO_SIZE) { + //USER no failing read found!! Something must have gone wrong + + return 0; + } + + max_working_cnt = 0; + min_working_d = 0; + + for (i = 0; i < VFIFO_SIZE+1; i++) { + for (d = 0; d <= IO_DQS_EN_PHASE_MAX; d++) { + scc_mgr_set_dqs_en_phase_all_ranks(grp, d); + + rw_mgr_mem_calibrate_read_test_all_ranks (grp, NUM_READ_PB_TESTS, PASS_ONE_BIT, &bit_chk, 0); + if (bit_chk) { + //USER passing read + + if (max_working_cnt == 0) { + min_working_d = d; + } + + max_working_cnt++; + } else { + if (max_working_cnt > 0) { + //USER already have one working value + break; + } + } + } + + if (d > IO_DQS_EN_PHASE_MAX) { + //USER fiddle with FIFO + rw_mgr_incr_vfifo(grp, &v); + } else { + //USER found working solution! + + d = min_working_d + (max_working_cnt - 1) / 2; + + while (d > IO_DQS_EN_PHASE_MAX) { + d -= (IO_DQS_EN_PHASE_MAX + 1); + } + + break; + } + } + + if (i >= VFIFO_SIZE+1) { + //USER cannot find working solution or cannot see edge of failing read + + return 0; + } + + //USER in the case the number of working steps is even, use 50ps taps to further center the window + + if ((max_working_cnt & 1) == 0) { + delay_per_ptap_mid = IO_DELAY_PER_OPA_TAP / 2; + + //USER increment in 50ps taps until we reach the required amount + + for (i = 0, j = 0; i <= IO_DQS_EN_DELAY_MAX && j < delay_per_ptap_mid; i++, j += IO_DELAY_PER_DQS_EN_DCHAIN_TAP); + + scc_mgr_set_dqs_en_delay_all_ranks(grp, i - 1); + } + + scc_mgr_set_dqs_en_phase_all_ranks(grp, d); + + //USER push vfifo until we can successfully calibrate + + for (i = 0; i < VFIFO_SIZE; i++) { + if (rw_mgr_mem_calibrate_read_test_all_ranks (grp, NUM_READ_PB_TESTS, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } + + //USER fiddle with FIFO + rw_mgr_incr_vfifo (grp, &v); + } + + if (i >= VFIFO_SIZE) { + return 0; + } + + return 1; +} + +#endif +#endif + + +// Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different dq_in_delay values +static inline alt_u32 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay (alt_u32 write_group, alt_u32 read_group, alt_u32 test_bgn) +{ +#if STRATIXV || ARRIAV || CYCLONEV || ARRIAVGZ + alt_u32 found; + alt_u32 i; + alt_u32 p; + alt_u32 d; + alt_u32 r; + + const alt_u32 delay_step = IO_IO_IN_DELAY_MAX/(RW_MGR_MEM_DQ_PER_READ_DQS-1); /* we start at zero, so have one less dq to devide among */ + + TRACE_FUNC("(%lu,%lu,%lu)", write_group, read_group, test_bgn); + + // try different dq_in_delays since the dq path is shorter than dqs + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + select_shadow_regs_for_update(r, write_group, 1); + for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++, d += delay_step) { + DPRINT(1, "rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay: g=%lu/%lu r=%lu, i=%lu p=%lu d=%lu", + write_group, read_group, r, i, p, d); + scc_mgr_set_dq_in_delay(write_group, p, d); + scc_mgr_load_dq (p); + } + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + + found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group); + + DPRINT(1, "rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay: g=%lu/%lu found=%lu; Reseting delay chain to zero", + write_group, read_group, found); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + select_shadow_regs_for_update(r, write_group, 1); + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { + scc_mgr_set_dq_in_delay(write_group, p, 0); + scc_mgr_load_dq (p); + } + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + + return found; +#else + return rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group); +#endif +} + +//USER per-bit deskew DQ and center + +#if NEWVERSION_RDDESKEW + +alt_u32 rw_mgr_mem_calibrate_vfifo_center (alt_u32 rank_bgn, alt_u32 write_group, alt_u32 read_group, alt_u32 test_bgn, alt_u32 use_read_test, alt_u32 update_fom) +{ + alt_u32 i, p, d, min_index; + //USER Store these as signed since there are comparisons with signed numbers + t_btfld bit_chk; + t_btfld sticky_bit_chk; + alt_32 left_edge[RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 right_edge[RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 final_dq[RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_32 mid; + alt_32 orig_mid_min, mid_min; + alt_32 new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs, final_dqs_en; + alt_32 dq_margin, dqs_margin; + alt_u32 stop; + + TRACE_FUNC("%lu %lu", read_group, test_bgn); +#if BFM_MODE + if (use_read_test) { + BFM_STAGE("vfifo_center"); + } else { + BFM_STAGE("vfifo_center_after_writes"); + } +#endif + + ALTERA_ASSERT(read_group < RW_MGR_MEM_IF_READ_DQS_WIDTH); + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + start_dqs = READ_SCC_DQS_IN_DELAY(read_group); + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + start_dqs_en = READ_SCC_DQS_EN_DELAY(read_group); + } + + select_curr_shadow_reg_using_rank(rank_bgn); + + //USER per-bit deskew + + //USER set the left and right edge of each bit to an illegal value + //USER use (IO_IO_IN_DELAY_MAX + 1) as an illegal value + sticky_bit_chk = 0; + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + left_edge[i] = IO_IO_IN_DELAY_MAX + 1; + right_edge[i] = IO_IO_IN_DELAY_MAX + 1; + } + + //USER Search for the left edge of the window for each bit + for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) { + scc_mgr_apply_group_dq_in_delay (write_group, test_bgn, d); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + //USER Stop searching when the read test doesn't pass AND when we've seen a passing read on every bit + if (use_read_test) { + stop = !rw_mgr_mem_calibrate_read_test (rank_bgn, read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT, &bit_chk, 0, 0); + } else { + rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ONE_BIT, &bit_chk, 0); + bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS * (read_group - (write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH))); + stop = (bit_chk == 0); + } + sticky_bit_chk = sticky_bit_chk | bit_chk; + stop = stop && (sticky_bit_chk == param->read_correct_mask); + DPRINT(2, "vfifo_center(left): dtap=%lu => " BTFLD_FMT " == " BTFLD_FMT " && %lu", d, sticky_bit_chk, param->read_correct_mask, stop); + + if (stop == 1) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + if (bit_chk & 1) { + //USER Remember a passing test as the left_edge + left_edge[i] = d; + } else { + //USER If a left edge has not been seen yet, then a future passing test will mark this edge as the right edge + if (left_edge[i] == IO_IO_IN_DELAY_MAX + 1) { + right_edge[i] = -(d + 1); + } + } + DPRINT(2, "vfifo_center[l,d=%lu]: bit_chk_test=%d left_edge[%lu]: %ld right_edge[%lu]: %ld", + d, (int)(bit_chk & 1), i, left_edge[i], i, right_edge[i]); + bit_chk = bit_chk >> 1; + } + } + } + + //USER Reset DQ delay chains to 0 + scc_mgr_apply_group_dq_in_delay (write_group, test_bgn, 0); + sticky_bit_chk = 0; + for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) { + + DPRINT(2, "vfifo_center: left_edge[%lu]: %ld right_edge[%lu]: %ld", i, left_edge[i], i, right_edge[i]); + + //USER Check for cases where we haven't found the left edge, which makes our assignment of the the + //USER right edge invalid. Reset it to the illegal value. + if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) { + right_edge[i] = IO_IO_IN_DELAY_MAX + 1; + DPRINT(2, "vfifo_center: reset right_edge[%lu]: %ld", i, right_edge[i]); + } + + //USER Reset sticky bit (except for bits where we have seen both the left and right edge) + sticky_bit_chk = sticky_bit_chk << 1; + if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) && (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) { + sticky_bit_chk = sticky_bit_chk | 1; + } + + if (i == 0) + { + break; + } + } + + //USER Search for the right edge of the window for each bit + for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) { + scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs); + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + alt_u32 delay = d + start_dqs_en; + if (delay > IO_DQS_EN_DELAY_MAX) { + delay = IO_DQS_EN_DELAY_MAX; + } + scc_mgr_set_dqs_en_delay(read_group, delay); + } + scc_mgr_load_dqs (read_group); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + //USER Stop searching when the read test doesn't pass AND when we've seen a passing read on every bit + if (use_read_test) { + stop = !rw_mgr_mem_calibrate_read_test (rank_bgn, read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT, &bit_chk, 0, 0); + } else { + rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ONE_BIT, &bit_chk, 0); + bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS * (read_group - (write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH))); + stop = (bit_chk == 0); + } + sticky_bit_chk = sticky_bit_chk | bit_chk; + stop = stop && (sticky_bit_chk == param->read_correct_mask); + + DPRINT(2, "vfifo_center(right): dtap=%lu => " BTFLD_FMT " == " BTFLD_FMT " && %lu", d, sticky_bit_chk, param->read_correct_mask, stop); + + if (stop == 1) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + if (bit_chk & 1) { + //USER Remember a passing test as the right_edge + right_edge[i] = d; + } else { + if (d != 0) { + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + if (right_edge[i] == IO_IO_IN_DELAY_MAX + 1) { + left_edge[i] = -(d + 1); + } + } else { + //USER d = 0 failed, but it passed when testing the left edge, so it must be marginal, set it to -1 + if (right_edge[i] == IO_IO_IN_DELAY_MAX + 1 && left_edge[i] != IO_IO_IN_DELAY_MAX + 1) { + right_edge[i] = -1; + } + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + else if (right_edge[i] == IO_IO_IN_DELAY_MAX + 1) { + left_edge[i] = -(d + 1); + } + + } + } + + DPRINT(2, "vfifo_center[r,d=%lu]: bit_chk_test=%d left_edge[%lu]: %ld right_edge[%lu]: %ld", + d, (int)(bit_chk & 1), i, left_edge[i], i, right_edge[i]); + bit_chk = bit_chk >> 1; + } + } + } + + // Store all observed margins +#if ENABLE_TCL_DEBUG + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + alt_u32 dq = read_group*RW_MGR_MEM_DQ_PER_READ_DQS + i; + + ALTERA_ASSERT(dq < RW_MGR_MEM_DATA_WIDTH); + + TCLRPT_SET(debug_cal_report->cal_dq_in_margins[curr_shadow_reg][dq].left_edge, left_edge[i]); + TCLRPT_SET(debug_cal_report->cal_dq_in_margins[curr_shadow_reg][dq].right_edge, right_edge[i]); + } +#endif + + //USER Check that all bits have a window + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + DPRINT(2, "vfifo_center: left_edge[%lu]: %ld right_edge[%lu]: %ld", i, left_edge[i], i, right_edge[i]); + BFM_GBL_SET(dq_read_left_edge[read_group][i],left_edge[i]); + BFM_GBL_SET(dq_read_right_edge[read_group][i],right_edge[i]); + if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i] == IO_IO_IN_DELAY_MAX + 1)) { + + //USER Restore delay chain settings before letting the loop in + //USER rw_mgr_mem_calibrate_vfifo to retry different dqs/ck relationships + scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs); + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + scc_mgr_set_dqs_en_delay(read_group, start_dqs_en); + } + scc_mgr_load_dqs (read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + DPRINT(1, "vfifo_center: failed to find edge [%lu]: %ld %ld", i, left_edge[i], right_edge[i]); + if (use_read_test) { + set_failing_group_stage(read_group*RW_MGR_MEM_DQ_PER_READ_DQS + i, CAL_STAGE_VFIFO, CAL_SUBSTAGE_VFIFO_CENTER); + } else { + set_failing_group_stage(read_group*RW_MGR_MEM_DQ_PER_READ_DQS + i, CAL_STAGE_VFIFO_AFTER_WRITES, CAL_SUBSTAGE_VFIFO_CENTER); + } + return 0; + } + } + + //USER Find middle of window for each DQ bit + mid_min = left_edge[0] - right_edge[0]; + min_index = 0; + for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + mid = left_edge[i] - right_edge[i]; + if (mid < mid_min) { + mid_min = mid; + min_index = i; + } + } + + //USER -mid_min/2 represents the amount that we need to move DQS. If mid_min is odd and positive we'll need to add one to + //USER make sure the rounding in further calculations is correct (always bias to the right), so just add 1 for all positive values + if (mid_min > 0) { + mid_min++; + } + mid_min = mid_min / 2; + + DPRINT(1, "vfifo_center: mid_min=%ld (index=%lu)", mid_min, min_index); + + //USER Determine the amount we can change DQS (which is -mid_min) + orig_mid_min = mid_min; +#if ENABLE_DQS_IN_CENTERING + new_dqs = start_dqs - mid_min; + if (new_dqs > IO_DQS_IN_DELAY_MAX) { + new_dqs = IO_DQS_IN_DELAY_MAX; + } else if (new_dqs < 0) { + new_dqs = 0; + } + mid_min = start_dqs - new_dqs; + DPRINT(1, "vfifo_center: new mid_min=%ld new_dqs=%ld", mid_min, new_dqs); + + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX) { + mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX; + } else if (start_dqs_en - mid_min < 0) { + mid_min += start_dqs_en - mid_min; + } + } + new_dqs = start_dqs - mid_min; +#else + new_dqs = start_dqs; + mid_min = 0; +#endif + + DPRINT(1, "vfifo_center: start_dqs=%ld start_dqs_en=%ld new_dqs=%ld mid_min=%ld", + start_dqs, IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1, new_dqs, mid_min); + + //USER Initialize data for export structures + dqs_margin = IO_IO_IN_DELAY_MAX + 1; + dq_margin = IO_IO_IN_DELAY_MAX + 1; + + //USER add delay to bring centre of all DQ windows to the same "level" + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { + //USER Use values before divide by 2 to reduce round off error + shift_dq = (left_edge[i] - right_edge[i] - (left_edge[min_index] - right_edge[min_index]))/2 + (orig_mid_min - mid_min); + + DPRINT(2, "vfifo_center: before: shift_dq[%lu]=%ld", i, shift_dq); + + if (shift_dq + (alt_32)READ_SCC_DQ_IN_DELAY(p) > (alt_32)IO_IO_IN_DELAY_MAX) { + shift_dq = (alt_32)IO_IO_IN_DELAY_MAX - READ_SCC_DQ_IN_DELAY(i); + } else if (shift_dq + (alt_32)READ_SCC_DQ_IN_DELAY(p) < 0) { + shift_dq = -(alt_32)READ_SCC_DQ_IN_DELAY(p); + } + DPRINT(2, "vfifo_center: after: shift_dq[%lu]=%ld", i, shift_dq); + final_dq[i] = READ_SCC_DQ_IN_DELAY(p) + shift_dq; + scc_mgr_set_dq_in_delay(write_group, p, final_dq[i]); + scc_mgr_load_dq (p); + + DPRINT(2, "vfifo_center: margin[%lu]=[%ld,%ld]", i, + left_edge[i] - shift_dq + (-mid_min), + right_edge[i] + shift_dq - (-mid_min)); + //USER To determine values for export structures + if (left_edge[i] - shift_dq + (-mid_min) < dq_margin) { + dq_margin = left_edge[i] - shift_dq + (-mid_min); + } + if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin) { + dqs_margin = right_edge[i] + shift_dq - (-mid_min); + } + } + +#if ENABLE_DQS_IN_CENTERING + final_dqs = new_dqs; + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + final_dqs_en = start_dqs_en - mid_min; + } +#else + final_dqs = start_dqs; + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + final_dqs_en = start_dqs_en; + } +#endif + + //USER Move DQS-en + if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) { + scc_mgr_set_dqs_en_delay(read_group, final_dqs_en); + scc_mgr_load_dqs (read_group); + } + +#if QDRII || RLDRAMX + //USER Move DQS. Do it gradually to minimize the chance of causing a timing + //USER failure in core FPGA logic driven by an input-strobe-derived clock + d = READ_SCC_DQS_IN_DELAY(read_group); + while (d != final_dqs) { + if (d > final_dqs) { + --d; + } else { + ++d; + } + scc_mgr_set_dqs_bus_in_delay(read_group, d); + scc_mgr_load_dqs (read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } +#else + //USER Move DQS + scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs); + scc_mgr_load_dqs (read_group); +#endif + + if(update_fom) { + //USER Export values + gbl->fom_in += (dq_margin + dqs_margin)/(RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + TCLRPT_SET(debug_summary_report->fom_in, debug_summary_report->fom_in + (dq_margin + dqs_margin)/(RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH)); + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][write_group].fom_in, debug_cal_report->cal_status_per_group[curr_shadow_reg][write_group].fom_in + (dq_margin + dqs_margin)/(RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH)); + } + + TCLRPT_SET(debug_cal_report->cal_dqs_in_margins[curr_shadow_reg][read_group].dqs_margin, dqs_margin); + TCLRPT_SET(debug_cal_report->cal_dqs_in_margins[curr_shadow_reg][read_group].dq_margin, dq_margin); + + DPRINT(2, "vfifo_center: dq_margin=%ld dqs_margin=%ld", dq_margin, dqs_margin); + +#if RUNTIME_CAL_REPORT + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + if (use_read_test) { + RPRINT("Read Deskew ; DQ %2lu ; Rank %lu ; Left edge %3li ; Right edge %3li ; DQ delay %2li ; DQS delay %2li", read_group*RW_MGR_MEM_DQ_PER_READ_DQS + i, curr_shadow_reg, left_edge[i], right_edge[i], final_dq[i], final_dqs); + } else { + RPRINT("Read after Write ; DQ %2lu ; Rank %lu ; Left edge %3li ; Right edge %3li ; DQ delay %2li ; DQS delay %2li", read_group*RW_MGR_MEM_DQ_PER_READ_DQS + i, curr_shadow_reg, left_edge[i], right_edge[i], final_dq[i], final_dqs); + } + } +#endif + + //USER Do not remove this line as it makes sure all of our decisions have been applied + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + return (dq_margin >= 0) && (dqs_margin >= 0); +} + +#else + +alt_u32 rw_mgr_mem_calibrate_vfifo_center (alt_u32 rank_bgn, alt_u32 grp, alt_u32 test_bgn, alt_u32 use_read_test) +{ + alt_u32 i, p, d; + alt_u32 mid; + t_btfld bit_chk; + alt_u32 max_working_dq[RW_MGR_MEM_DQ_PER_READ_DQS]; + alt_u32 dq_margin, dqs_margin; + alt_u32 start_dqs; + + TRACE_FUNC("%lu %lu", grp, test_bgn); + + //USER per-bit deskew. + //USER start of the per-bit sweep with the minimum working delay setting for + //USER all bits. + + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + max_working_dq[i] = 0; + } + + for (d = 1; d <= IO_IO_IN_DELAY_MAX; d++) { + scc_mgr_apply_group_dq_in_delay (write_group, test_bgn, d); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (!rw_mgr_mem_calibrate_read_test (rank_bgn, grp, NUM_READ_PB_TESTS, PASS_ONE_BIT, &bit_chk, 0, 0)) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + if (bit_chk & 1) { + max_working_dq[i] = d; + } + bit_chk = bit_chk >> 1; + } + } + } + + //USER determine minimum working value for DQ + + dq_margin = IO_IO_IN_DELAY_MAX; + + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + if (max_working_dq[i] < dq_margin) { + dq_margin = max_working_dq[i]; + } + } + + //USER add delay to bring all DQ windows to the same "level" + + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { + if (max_working_dq[i] > dq_margin) { + scc_mgr_set_dq_in_delay(write_group, i, max_working_dq[i] - dq_margin); + } else { + scc_mgr_set_dq_in_delay(write_group, i, 0); + } + + scc_mgr_load_dq (p, p); + } + + //USER sweep DQS window, may potentially have more window due to per-bit-deskew that was done + //USER in the previous step. + + start_dqs = READ_SCC_DQS_IN_DELAY(grp); + + for (d = start_dqs + 1; d <= IO_DQS_IN_DELAY_MAX; d++) { + scc_mgr_set_dqs_bus_in_delay(grp, d); + scc_mgr_load_dqs (grp); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (!rw_mgr_mem_calibrate_read_test (rank_bgn, grp, NUM_READ_TESTS, PASS_ALL_BITS, &bit_chk, 0, 0)) { + break; + } + } + + scc_mgr_set_dqs_bus_in_delay(grp, start_dqs); + + //USER margin on the DQS pin + + dqs_margin = d - start_dqs - 1; + + //USER find mid point, +1 so that we don't go crazy pushing DQ + + mid = (dq_margin + dqs_margin + 1) / 2; + + gbl->fom_in += dq_margin + dqs_margin; +// TCLRPT_SET(debug_summary_report->fom_in, debug_summary_report->fom_in + (dq_margin + dqs_margin)); +// TCLRPT_SET(debug_cal_report->cal_status_per_group[grp].fom_in, (dq_margin + dqs_margin)); + + + + +#if ENABLE_DQS_IN_CENTERING + //USER center DQS ... if the headroom is setup properly we shouldn't need to + + if (dqs_margin > mid) { + scc_mgr_set_dqs_bus_in_delay(grp, READ_SCC_DQS_IN_DELAY(grp) + dqs_margin - mid); + + if (DDRX) { + alt_u32 delay = READ_SCC_DQS_EN_DELAY(grp) + dqs_margin - mid; + + if (delay > IO_DQS_EN_DELAY_MAX) { + delay = IO_DQS_EN_DELAY_MAX; + } + + scc_mgr_set_dqs_en_delay(grp, delay); + } + } +#endif + + scc_mgr_load_dqs (grp); + + //USER center DQ + + if (dq_margin > mid) { + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) { + scc_mgr_set_dq_in_delay(write_group, i, READ_SCC_DQ_IN_DELAY(i) + dq_margin - mid); + scc_mgr_load_dq (p, p); + } + + dqs_margin += dq_margin - mid; + dq_margin -= dq_margin - mid; + } + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + return (dq_margin + dqs_margin) > 0; +} + +#endif + +//USER calibrate the read valid prediction FIFO. +//USER +//USER - read valid prediction will consist of finding a good DQS enable phase, DQS enable delay, DQS input phase, and DQS input delay. +//USER - we also do a per-bit deskew on the DQ lines. + +#if DYNAMIC_CALIBRATION_MODE || STATIC_QUICK_CALIBRATION + +#if !ENABLE_SUPER_QUICK_CALIBRATION + +//USER VFIFO Calibration -- Quick Calibration +alt_u32 rw_mgr_mem_calibrate_vfifo (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 v, d, i; + alt_u32 found; + t_btfld bit_chk; + + TRACE_FUNC("%lu %lu", grp, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_VFIFO); + + //USER Load up the patterns used by read calibration + + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); + + //USER maximum phase values for the sweep + + + //USER update info for sims + + reg_file_set_group(g); + + found = 0; + v = 0; + for (i = 0; i < VFIFO_SIZE && found == 0; i++) { + for (d = 0; d <= IO_DQS_EN_PHASE_MAX && found == 0; d++) { + if (DDRX) + { + scc_mgr_set_dqs_en_phase_all_ranks(g, d); + } + + //USER calibrate the vfifo with the current dqs enable phase setting + + if (rw_mgr_mem_calibrate_read_test_all_ranks (g, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found = 1; + } + } + + if (found) { + break; + } else { + rw_mgr_incr_vfifo_all (g, &v); + } + } + + return found; +} + +#else + +//USER VFIFO Calibration -- Super Quick Calibration +alt_u32 rw_mgr_mem_calibrate_vfifo (alt_u32 grp, alt_u32 test_bgn2) +{ + alt_u32 g, v, d, i; + alt_u32 test_bgn; + alt_u32 found; + t_btfld bit_chk; + alt_u32 phase_increment; + alt_u32 final_v_setting = 0; + alt_u32 final_d_setting = 0; + + TRACE_FUNC("%lu %lu", grp, test_bgn2); + + #if ARRIAV || CYCLONEV + // Compensate for simulation model behaviour + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + scc_mgr_set_dqs_bus_in_delay(i, 10); + scc_mgr_load_dqs (i); + } + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + #endif + + //USER The first call to this function will calibrate all groups + if (grp !=0) { + return 1; + } + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_VFIFO); + + //USER Load up the patterns used by read calibration + + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); + + //USER maximum phase values for the sweep + + //USER Calibrate group 0 + g = 0; + test_bgn = 0; + + //USER update info for sims + + reg_file_set_group(g); + + found = 0; + + //USER In behavioral simulation only phases 0 and IO_DQS_EN_PHASE_MAX/2 are relevant + //USER All other values produces the same results as those 2, so there's really no + //USER point in sweeping them all + phase_increment = (IO_DQS_EN_PHASE_MAX + 1) / 2; + //USER Make sure phase_increment is > 0 to prevent infinite loop + if (phase_increment == 0) phase_increment++; + + v = 0; + for (i = 0; i < VFIFO_SIZE && found == 0; i++) { + for (d = 0; d <= IO_DQS_EN_PHASE_MAX && found == 0; d += phase_increment) { + + scc_mgr_set_dqs_en_phase_all_ranks(g, d); + + //USER calibrate the vfifo with the current dqs enable phase setting + + if (rw_mgr_mem_calibrate_read_test_all_ranks (g, 1, PASS_ONE_BIT, &bit_chk, 0)) { + found = 1; + final_v_setting = v; + final_d_setting = d; + } + } + + if (!found) { + rw_mgr_incr_vfifo_all (g, &v); + } else { + break; + } + } + + if (!found) return 0; + + //USER Now copy the calibration settings to all other groups + for (g = 1, test_bgn = RW_MGR_MEM_DQ_PER_READ_DQS; (g < RW_MGR_MEM_IF_READ_DQS_WIDTH) && found; g++, test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) { + //USER Set the VFIFO + v = 0; + for (i = 0; i < final_v_setting; i++) { + rw_mgr_incr_vfifo_all (g, &v); + } + + //USER Set the proper phase + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, g); + scc_mgr_set_dqs_en_phase_all_ranks(g, final_d_setting); + + //USER Verify that things worked as expected + if(!rw_mgr_mem_calibrate_read_test_all_ranks (g, 1, PASS_ONE_BIT, &bit_chk, 0)) { + //USER Fail + found = 0; + } + } + + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, 0); + return found; +} + +#endif +#endif + +#if DYNAMIC_CALIBRATION_MODE || STATIC_FULL_CALIBRATION + +#if NEWVERSION_GW + +//USER VFIFO Calibration -- Full Calibration +alt_u32 rw_mgr_mem_calibrate_vfifo (alt_u32 read_group, alt_u32 test_bgn) +{ + alt_u32 p, d, rank_bgn, sr; + alt_u32 dtaps_per_ptap; + alt_u32 tmp_delay; + t_btfld bit_chk; + alt_u32 grp_calibrated; + alt_u32 write_group, write_test_bgn; + alt_u32 failed_substage; + alt_u32 dqs_in_dtaps, orig_start_dqs; + + TRACE_FUNC("%lu %lu", read_group, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_VFIFO); + + if (DDRX) { + write_group = read_group; + write_test_bgn = test_bgn; + } else { + write_group = read_group / (RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + write_test_bgn = read_group * RW_MGR_MEM_DQ_PER_READ_DQS; + } + + // USER Determine number of delay taps for each phase tap + dtaps_per_ptap = 0; + tmp_delay = 0; + if (!QDRII) { + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP; + } + dtaps_per_ptap--; + tmp_delay = 0; + } + + //USER update info for sims + + reg_file_set_group(read_group); + + grp_calibrated = 0; + + reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ); + failed_substage = CAL_SUBSTAGE_GUARANTEED_READ; + + for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d+=2) { + + if (DDRX || RLDRAMX) { + // In RLDRAMX we may be messing the delay of pins in the same write group but outside of + // the current read group, but that's ok because we haven't calibrated the output side yet. + if (d > 0) { + scc_mgr_apply_group_all_out_delay_add_all_ranks (write_group, write_test_bgn, d); + } + } + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0; p++) { + //USER set a particular dqdqs phase + if (DDRX) { + scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p); + } + + //USER Previous iteration may have failed as a result of ck/dqs or ck/dk violation, + //USER in which case the device may require special recovery. + if (DDRX || RLDRAMX) { + if (d != 0 || p != 0) { + recover_mem_device_after_ck_dqs_violation(); + } + } + + DPRINT(1, "calibrate_vfifo: g=%lu p=%lu d=%lu", read_group, p, d); + BFM_GBL_SET(gwrite_pos[read_group].p, p); + BFM_GBL_SET(gwrite_pos[read_group].d, d); + + //USER Load up the patterns used by read calibration using current DQDQS phase + +#if BFM_MODE + // handled by pre-initializing memory if skipping + if (bfm_gbl.bfm_skip_guaranteed_write == 0) { + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); + } +#else + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); + +#if DDRX +#if !AP_MODE + if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)) { + if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks (read_group, 1, &bit_chk)) { + DPRINT(1, "Guaranteed read test failed: g=%lu p=%lu d=%lu", read_group, p, d); + break; + } + } +#endif +#endif +#endif + + // Loop over different DQS in delay chains for the purpose of DQS Enable calibration finding one bit working + orig_start_dqs = READ_SCC_DQS_IN_DELAY(read_group); + for (dqs_in_dtaps = orig_start_dqs; dqs_in_dtaps <= IO_DQS_IN_DELAY_MAX && grp_calibrated == 0; dqs_in_dtaps++) { + + for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { + + if (! param->skip_shadow_regs[sr]) { + + //USER Select shadow register set + select_shadow_regs_for_update(rank_bgn, read_group, 1); + + WRITE_SCC_DQS_IN_DELAY(read_group, dqs_in_dtaps); + scc_mgr_load_dqs (read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + } + +// case:56390 +#if 0 && ARRIAV && QDRII + // Note, much of this counts on the fact that we don't need to keep track + // of what vfifo offset we are at because incr_vfifo doesn't use it + // We also assume only a single group, and that the vfifo incrementers start at offset zero + +#define BIT(w,b) (((w) >> (b)) & 1) + { + alt_u32 prev; + alt_u32 vbase; + alt_u32 i; + + grp_calibrated = 0; + + // check every combination of vfifo relative settings + for (prev = vbase = 0; vbase < (1 << VFIFO_CONTROL_WIDTH_PER_DQS); prev=vbase, vbase++ ) { + // check each bit to see if we need to increment, decrement, or leave the corresponding vfifo alone + for (i = 0; i < VFIFO_CONTROL_WIDTH_PER_DQS; i++) { + if (BIT(vbase,i) > BIT(prev,i)) { + rw_mgr_incr_vfifo(read_group*VFIFO_CONTROL_WIDTH_PER_DQS + i,0); + } else if (BIT(vbase,i) < BIT(prev,i)) { + rw_mgr_decr_vfifo(read_group*VFIFO_CONTROL_WIDTH_PER_DQS + i,0); + } + } + if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay (write_group, read_group, test_bgn)) { + + // Before doing read deskew, set DQS in back to the reserve value + WRITE_SCC_DQS_IN_DELAY(read_group, orig_start_dqs); + scc_mgr_load_dqs (read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (! rw_mgr_mem_calibrate_vfifo_center (0, write_group, read_group, test_bgn, 1)) { + // remember last failed stage + failed_substage = CAL_SUBSTAGE_VFIFO_CENTER; + } else { + grp_calibrated = 1; + } + } else { + failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE; + } + if (grp_calibrated) { + break; + } + + break; // comment out for fix + + } + } +#else + grp_calibrated = 1; + if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay (write_group, read_group, test_bgn)) { + // USER Read per-bit deskew can be done on a per shadow register basis + for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { +#if RUNTIME_CAL_REPORT + //Report print can cause a delay at each instance of rw_mgr_mem_calibrate_vfifo_center, need to re-issue guaranteed write to ensure no refresh violation + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); +#endif + //USER Determine if this set of ranks should be skipped entirely + if (! param->skip_shadow_regs[sr]) { + + //USER Select shadow register set + select_shadow_regs_for_update(rank_bgn, read_group, 1); + + // Before doing read deskew, set DQS in back to the reserve value + WRITE_SCC_DQS_IN_DELAY(read_group, orig_start_dqs); + scc_mgr_load_dqs (read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + // If doing read after write calibration, do not update FOM now - do it then +#if READ_AFTER_WRITE_CALIBRATION + if (! rw_mgr_mem_calibrate_vfifo_center (rank_bgn, write_group, read_group, test_bgn, 1, 0)) { +#else + if (! rw_mgr_mem_calibrate_vfifo_center (rank_bgn, write_group, read_group, test_bgn, 1, 1)) { +#endif + grp_calibrated = 0; + failed_substage = CAL_SUBSTAGE_VFIFO_CENTER; + } + } + } + } else { + grp_calibrated = 0; + failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE; + } +#endif +#if BFM_MODE + if (bfm_gbl.bfm_skip_guaranteed_write > 0 && !grp_calibrated) { + // This should never happen with pre-initialized guaranteed write load pattern + // unless calibration was always going to fail + DPRINT(0, "calibrate_vfifo: skip guaranteed write calibration failed"); + break; + } else if (bfm_gbl.bfm_skip_guaranteed_write == -1) { + // if skip value is -1, then we expect to fail, but we want to use + // the regular guaranteed write next time + if (grp_calibrated) { + // We shouldn't be succeeding for this test, so this is an error + DPRINT(0, "calibrate_vfifo: ERROR: skip guaranteed write == -1, but calibration passed"); + grp_calibrated = 0; + break; + } else { + DPRINT(0, "calibrate_vfifo: skip guaranteed write == -1, expected failure, trying again with no skip"); + bfm_gbl.bfm_skip_guaranteed_write = 0; + } + } + +#endif + } + + } +#if BFM_MODE + if (bfm_gbl.bfm_skip_guaranteed_write && !grp_calibrated) break; +#endif + } + + if (grp_calibrated == 0) { + set_failing_group_stage(write_group, CAL_STAGE_VFIFO, failed_substage); + + return 0; + } + + //USER Reset the delay chains back to zero if they have moved > 1 (check for > 1 because loop will increase d even when pass in first case) + if (DDRX || RLDRAMII) { + if (d > 2) { + scc_mgr_zero_group(write_group, write_test_bgn, 1); + } + } + + + return 1; +} + +#else + +//USER VFIFO Calibration -- Full Calibration +alt_u32 rw_mgr_mem_calibrate_vfifo (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 p, rank_bgn, sr; + alt_u32 grp_calibrated; + alt_u32 failed_substage; + + TRACE_FUNC("%lu %lu", g, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_VFIFO); + + reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ); + + failed_substage = CAL_SUBSTAGE_GUARANTEED_READ; + + //USER update info for sims + + reg_file_set_group(g); + + grp_calibrated = 0; + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0; p++) { + //USER set a particular dqdqs phase + if (DDRX) { + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + } + + //USER Load up the patterns used by read calibration using current DQDQS phase + + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); +#if DDRX + if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)) { + if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks (read_group, 1, &bit_chk)) { + break; + } + } +#endif + + grp_calibrated = 1; + if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay (g, g, test_bgn)) { + // USER Read per-bit deskew can be done on a per shadow register basis + for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { + + //USER Determine if this set of ranks should be skipped entirely + if (! param->skip_shadow_regs[sr]) { + + //USER Select shadow register set + select_shadow_regs_for_update(rank_bgn, read_group, 1); + + if (! rw_mgr_mem_calibrate_vfifo_center (rank_bgn, g, test_bgn, 1)) { + grp_calibrated = 0; + failed_substage = CAL_SUBSTAGE_VFIFO_CENTER; + } + } + } + } else { + grp_calibrated = 0; + failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE; + } + } + + if (grp_calibrated == 0) { + set_failing_group_stage(g, CAL_STAGE_VFIFO, failed_substage); + return 0; + } + + + return 1; +} + +#endif + +#endif + +#if READ_AFTER_WRITE_CALIBRATION +//USER VFIFO Calibration -- Read Deskew Calibration after write deskew +alt_u32 rw_mgr_mem_calibrate_vfifo_end (alt_u32 read_group, alt_u32 test_bgn) +{ + alt_u32 rank_bgn, sr; + alt_u32 grp_calibrated; + alt_u32 write_group; + + TRACE_FUNC("%lu %lu", read_group, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES); + reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER); + + if (DDRX) { + write_group = read_group; + } else { + write_group = read_group / (RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + } + + //USER update info for sims + reg_file_set_group(read_group); + + grp_calibrated = 1; + // USER Read per-bit deskew can be done on a per shadow register basis + for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { + + //USER Determine if this set of ranks should be skipped entirely + if (! param->skip_shadow_regs[sr]) { + + //USER Select shadow register set + select_shadow_regs_for_update(rank_bgn, read_group, 1); + + // This is the last calibration round, update FOM here + if (! rw_mgr_mem_calibrate_vfifo_center (rank_bgn, write_group, read_group, test_bgn, 0, 1)) { + grp_calibrated = 0; + } + } + } + + + if (grp_calibrated == 0) { + set_failing_group_stage(write_group, CAL_STAGE_VFIFO_AFTER_WRITES, CAL_SUBSTAGE_VFIFO_CENTER); + return 0; + } + + return 1; +} +#endif + + +//USER Calibrate LFIFO to find smallest read latency + +alt_u32 rw_mgr_mem_calibrate_lfifo (void) +{ + alt_u32 found_one; + t_btfld bit_chk; + alt_u32 g; + + TRACE_FUNC(); + BFM_STAGE("lfifo"); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_LFIFO); + reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY); + + //USER Load up the patterns used by read calibration for all ranks + + rw_mgr_mem_calibrate_read_load_patterns_all_ranks (); + + found_one = 0; + + do { + IOWR_32DIRECT (PHY_MGR_PHY_RLAT, 0, gbl->curr_read_lat); + DPRINT(2, "lfifo: read_lat=%lu", gbl->curr_read_lat); + + if (!rw_mgr_mem_calibrate_read_test_all_ranks (0, NUM_READ_TESTS, PASS_ALL_BITS, &bit_chk, 1)) { + break; + } + + found_one = 1; + + //USER reduce read latency and see if things are working + //USER correctly + + gbl->curr_read_lat--; + } while (gbl->curr_read_lat > 0); + + //USER reset the fifos to get pointers to known state + + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + + if (found_one) { + //USER add a fudge factor to the read latency that was determined + gbl->curr_read_lat += 2; +#if BFM_MODE + gbl->curr_read_lat += BFM_GBL_GET(lfifo_margin); +#endif + IOWR_32DIRECT (PHY_MGR_PHY_RLAT, 0, gbl->curr_read_lat); +#if RUNTIME_CAL_REPORT + RPRINT("LFIFO Calibration ; PHY Read Latency %li", gbl->curr_read_lat); +#endif + + DPRINT(2, "lfifo: success: using read_lat=%lu", gbl->curr_read_lat); + + return 1; + } else { + set_failing_group_stage(0xff, CAL_STAGE_LFIFO, CAL_SUBSTAGE_READ_LATENCY); + + for (g = 0; g < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; g++) + { + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][g].error_stage, CAL_STAGE_LFIFO); + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][g].error_sub_stage, CAL_SUBSTAGE_READ_LATENCY); + } + + DPRINT(2, "lfifo: failed at initial read_lat=%lu", gbl->curr_read_lat); + + return 0; + } +} + +//USER issue write test command. +//USER two variants are provided. one that just tests a write pattern and another that +//USER tests datamask functionality. + +#if QDRII +void rw_mgr_mem_calibrate_write_test_issue (alt_u32 group, alt_u32 test_dm) +{ + alt_u32 quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) && ENABLE_SUPER_QUICK_CALIBRATION) || BFM_MODE; + + //USER CNTR 1 - This is used to ensure enough time elapses for read data to come back. + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x30); + + if (test_dm) { + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + if(quick_write_mode) { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x08); + } else { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x40); + } + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, (group) << 2, __RW_MGR_LFSR_WR_RD_DM_BANK_0); + } else { + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + if(quick_write_mode) { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x08); + } else { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x40); + } + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_LFSR_WR_RD_BANK_0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_LFSR_WR_RD_BANK_0_WAIT); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, (group) << 2, __RW_MGR_LFSR_WR_RD_BANK_0); + } +} +#else +void rw_mgr_mem_calibrate_write_test_issue (alt_u32 group, alt_u32 test_dm) +{ + alt_u32 mcc_instruction; + alt_u32 quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) && ENABLE_SUPER_QUICK_CALIBRATION) || BFM_MODE; + alt_u32 rw_wl_nop_cycles; + + //USER Set counter and jump addresses for the right + //USER number of NOP cycles. + //USER The number of supported NOP cycles can range from -1 to infinity + //USER Three different cases are handled: + //USER + //USER 1. For a number of NOP cycles greater than 0, the RW Mgr looping + //USER mechanism will be used to insert the right number of NOPs + //USER + //USER 2. For a number of NOP cycles equals to 0, the micro-instruction + //USER issuing the write command will jump straight to the micro-instruction + //USER that turns on DQS (for DDRx), or outputs write data (for RLD), skipping + //USER the NOP micro-instruction all together + //USER + //USER 3. A number of NOP cycles equal to -1 indicates that DQS must be turned + //USER on in the same micro-instruction that issues the write command. Then we need + //USER to directly jump to the micro-instruction that sends out the data + //USER + //USER NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters (2 and 3). One + //USER jump-counter (0) is used to perform multiple write-read operations. + //USER one counter left to issue this command in "multiple-group" mode. + +#if MULTIPLE_AFI_WLAT + rw_wl_nop_cycles = gbl->rw_wl_nop_cycles_per_group[group]; +#else + rw_wl_nop_cycles = gbl->rw_wl_nop_cycles; +#endif + + if(rw_wl_nop_cycles == -1) + { + #if DDRX + //USER CNTR 2 - We want to execute the special write operation that + //USER turns on DQS right away and then skip directly to the instruction that + //USER sends out the data. We set the counter to a large number so that the + //USER jump is always taken + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0xFF); + + //USER CNTR 3 - Not used + if(test_dm) + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP); + } + else + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_BANK_0_WL_1; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_LFSR_WR_RD_BANK_0_DATA); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_LFSR_WR_RD_BANK_0_NOP); + } + + #endif + } + else if(rw_wl_nop_cycles == 0) + { + #if DDRX + //USER CNTR 2 - We want to skip the NOP operation and go straight to + //USER the DQS enable instruction. We set the counter to a large number so that the + //USER jump is always taken + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0xFF); + + //USER CNTR 3 - Not used + if(test_dm) + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_DM_BANK_0; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS); + } + else + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_BANK_0; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_LFSR_WR_RD_BANK_0_DQS); + } + #endif + + #if RLDRAMX + //USER CNTR 2 - We want to skip the NOP operation and go straight to + //USER the write data instruction. We set the counter to a large number so that the + //USER jump is always taken + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0xFF); + + //USER CNTR 3 - Not used + if(test_dm) + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_DM_BANK_0; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA); + } + else + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_BANK_0; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_LFSR_WR_RD_BANK_0_DATA); + } + #endif + } + else + { + //USER CNTR 2 - In this case we want to execute the next instruction and NOT + //USER take the jump. So we set the counter to 0. The jump address doesn't count + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x0); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, 0x0); + + //USER CNTR 3 - Set the nop counter to the number of cycles we need to loop for, minus 1 + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, rw_wl_nop_cycles - 1); + if(test_dm) + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_DM_BANK_0; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP); + } + else + { + mcc_instruction = __RW_MGR_LFSR_WR_RD_BANK_0; + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_LFSR_WR_RD_BANK_0_NOP); + } + } + + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + + if(quick_write_mode) { + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x08); + } else { +#if ENABLE_NON_DES_CAL + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x08); // Break this up for refresh purposes +#else + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x40); +#endif + } + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, mcc_instruction); + + //USER CNTR 1 - This is used to ensure enough time elapses for read data to come back. + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x30); + + if(test_dm) + { + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT); + } else { + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_LFSR_WR_RD_BANK_0_WAIT); + } + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, (group << 2), mcc_instruction); + +#if ENABLE_NON_DES_CAL + alt_u32 i = 0; + for (i=0; i < 8; i++) + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, (group << 2), mcc_instruction); +#endif + + + + + + + +} +#endif + +//USER Test writes, can check for a single bit pass or multiple bit pass + +alt_u32 rw_mgr_mem_calibrate_write_test (alt_u32 rank_bgn, alt_u32 write_group, alt_u32 use_dm, alt_u32 all_correct, t_btfld *bit_chk, alt_u32 all_ranks) +{ + alt_u32 r; + t_btfld correct_mask_vg; + t_btfld tmp_bit_chk; + alt_u32 vg; + alt_u32 rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS : (rank_bgn + NUM_RANKS_PER_SHADOW_REG); + + *bit_chk = param->write_correct_mask; + correct_mask_vg = param->write_correct_mask_vg; + + for (r = rank_bgn; r < rank_end; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); + + tmp_bit_chk = 0; + for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) { + + //USER reset the fifos to get pointers to known state + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + + tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_WRITE_DQS / RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS); + rw_mgr_mem_calibrate_write_test_issue (write_group*RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg, use_dm); + + tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(IORD_32DIRECT(BASE_RW_MGR, 0))); + DPRINT(2, "write_test(%lu,%lu,%lu) :[%lu,%lu] " BTFLD_FMT " & ~%x => " BTFLD_FMT " => " BTFLD_FMT, + write_group, use_dm, all_correct, r, vg, + correct_mask_vg, IORD_32DIRECT(BASE_RW_MGR, 0), correct_mask_vg & ~IORD_32DIRECT(BASE_RW_MGR, 0), + tmp_bit_chk); + + if (vg == 0) { + break; + } + } + *bit_chk &= tmp_bit_chk; + } + + if (all_correct) + { + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); + DPRINT(2, "write_test(%lu,%lu,ALL) : " BTFLD_FMT " == " BTFLD_FMT " => %lu", write_group, use_dm, + *bit_chk, param->write_correct_mask, (long unsigned int)(*bit_chk == param->write_correct_mask)); + return (*bit_chk == param->write_correct_mask); + } + else + { + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); + DPRINT(2, "write_test(%lu,%lu,ONE) : " BTFLD_FMT " != " BTFLD_FMT " => %lu", write_group, use_dm, + *bit_chk, (long unsigned int)0, (long unsigned int)(*bit_chk != 0)); + return (*bit_chk != 0x00); + } +} + +static inline alt_u32 rw_mgr_mem_calibrate_write_test_all_ranks (alt_u32 write_group, alt_u32 use_dm, alt_u32 all_correct, t_btfld *bit_chk) +{ + return rw_mgr_mem_calibrate_write_test (0, write_group, use_dm, all_correct, bit_chk, 1); +} + + +//USER level the write operations + +#if DYNAMIC_CALIBRATION_MODE || STATIC_QUICK_CALIBRATION + +#if QDRII + +//USER Write Levelling -- Quick Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + TRACE_FUNC("%lu %lu", g, test_bgn); + + return 0; +} + +#endif + +#if RLDRAMX +#if !ENABLE_SUPER_QUICK_CALIBRATION + +//USER Write Levelling -- Quick Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 d; + t_btfld bit_chk; + + TRACE_FUNC("%lu %lu", g, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + reg_file_set_group(g); + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + if (d > IO_IO_OUT1_DELAY_MAX) { + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + return 1; +} + +#else + +//USER Write Levelling -- Super Quick Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 d; + t_btfld bit_chk; + + TRACE_FUNC("%lu %lu", g, test_bgn); + + //USER The first call to this function will calibrate all groups + if (g != 0) { + return 1; + } + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + reg_file_set_group(g); + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + if (d > IO_IO_OUT1_DELAY_MAX) { + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + reg_file_set_sub_stage(CAL_SUBSTAGE_WLEVEL_COPY); + + //USER Now copy the calibration settings to all other groups + for (g = 1, test_bgn = RW_MGR_MEM_DQ_PER_WRITE_DQS; g < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; g++, test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) { + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + //USER Verify that things worked as expected + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WLEVEL_COPY); + + return 0; + } + } + + return 1; +} + +#endif +#endif + +#if DDRX +#if !ENABLE_SUPER_QUICK_CALIBRATION + +//USER Write Levelling -- Quick Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 p; + t_btfld bit_chk; + + TRACE_FUNC("%lu %lu", g, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + + //USER maximum phases for the sweep + + //USER starting phases + + //USER update info for sims + + reg_file_set_group(g); + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX; p++) { + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + if (p > IO_DQDQS_OUT_PHASE_MAX) { + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + return 1; +} + +#else + +//USER Write Levelling -- Super Quick Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 p; + t_btfld bit_chk; + + TRACE_FUNC("%lu %lu", g, test_bgn); + + //USER The first call to this function will calibrate all groups + if (g != 0) { + return 1; + } + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + + //USER maximum phases for the sweep + + //USER starting phases + + //USER update info for sims + + reg_file_set_group(g); + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX; p++) { + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + if (p > IO_DQDQS_OUT_PHASE_MAX) { + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + reg_file_set_sub_stage(CAL_SUBSTAGE_WLEVEL_COPY); + + //USER Now copy the calibration settings to all other groups + for (g = 1, test_bgn = RW_MGR_MEM_DQ_PER_READ_DQS; (g < RW_MGR_MEM_IF_READ_DQS_WIDTH); g++, test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) { + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, g); + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + //USER Verify that things worked as expected + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WLEVEL_COPY); + + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, 0); + return 0; + } + } + + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, 0); + return 1; +} + +#endif +#endif + +#endif + +#if DYNAMIC_CALIBRATION_MODE || STATIC_FULL_CALIBRATION + +#if QDRII +//USER Write Levelling -- Full Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + TRACE_FUNC("%lu %lu", g, test_bgn); + + return 0; +} +#endif + +#if RLDRAMX +//USER Write Levelling -- Full Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 d; + t_btfld bit_chk; + alt_u32 work_bgn, work_end; + alt_u32 d_bgn, d_end; + alt_u32 found_begin; + + TRACE_FUNC("%lu %lu", g, test_bgn); + BFM_STAGE("wlevel"); + + ALTERA_ASSERT(g < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + + //USER maximum delays for the sweep + + //USER update info for sims + + reg_file_set_group(g); + + //USER starting and end range where writes work + + scc_mgr_spread_out2_delay_all_ranks (g,test_bgn); + + work_bgn = 0; + work_end = 0; + + //USER step 1: find first working dtap, increment in dtaps + found_begin = 0; + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++, work_bgn += IO_DELAY_PER_DCHAIN_TAP) { + DPRINT(2, "wlevel: begin: d=%lu", d); + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + found_begin = 1; + d_bgn = d; + break; + } else { + recover_mem_device_after_ck_dqs_violation(); + } + } + + if (!found_begin) { + //USER fail, cannot find first working delay + + DPRINT(2, "wlevel: failed to find first working delay", d); + + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + DPRINT(2, "wlevel: found begin d=%lu work_bgn=%lu", d_bgn, work_bgn); + BFM_GBL_SET(dqs_wlevel_left_edge[g].d,d_bgn); + BFM_GBL_SET(dqs_wlevel_left_edge[g].ps,work_bgn); + + reg_file_set_sub_stage(CAL_SUBSTAGE_LAST_WORKING_DELAY); + + //USER step 2 : find first non-working dtap, increment in dtaps + work_end = work_bgn; + d = d + 1; + for (; d <= IO_IO_OUT1_DELAY_MAX; d++, work_end += IO_DELAY_PER_DCHAIN_TAP) { + DPRINT(2, "wlevel: end: d=%lu", d); + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + recover_mem_device_after_ck_dqs_violation(); + break; + } + } + d_end = d - 1; + + if (d_end >= d_bgn) { + //USER we have a working range + } else { + //USER nil range + //Note: don't think this is possible + + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_LAST_WORKING_DELAY); + + return 0; + } + + DPRINT(2, "wlevel: found end: d=%lu work_end=%lu", d_end, work_end); + BFM_GBL_SET(dqs_wlevel_right_edge[g].d,d_end); + BFM_GBL_SET(dqs_wlevel_right_edge[g].ps,work_end); + + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[curr_shadow_reg][g].dqdqs_start, work_bgn); + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[curr_shadow_reg][g].dqdqs_end, work_end); + + + //USER center + + d = (d_end + d_bgn) / 2; + + DPRINT(2, "wlevel: found middle: d=%lu work_mid=%lu", d, (work_end + work_bgn)/2); + BFM_GBL_SET(dqs_wlevel_mid[g].d,d); + BFM_GBL_SET(dqs_wlevel_mid[g].ps,(work_end + work_bgn)/2); + + scc_mgr_zero_group (g, test_bgn, 1); + scc_mgr_apply_group_all_out_delay_add_all_ranks (g, test_bgn, d); + + return 1; +} +#endif + + +#if DDRX +#if NEWVERSION_WL + +//USER Write Levelling -- Full Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 p, d, sr; + +#if CALIBRATE_BIT_SLIPS +#if QUARTER_RATE_MODE + alt_32 num_additional_fr_cycles = 3; +#elif HALF_RATE_MODE + alt_32 num_additional_fr_cycles = 1; +#else + alt_32 num_additional_fr_cycles = 0; +#endif +#if MULTIPLE_AFI_WLAT + num_additional_fr_cycles++; +#endif +#else + alt_u32 num_additional_fr_cycles = 0; +#endif + + t_btfld bit_chk; + alt_u32 work_bgn, work_end, work_mid; + alt_u32 tmp_delay; + alt_u32 found_begin; + alt_u32 dtaps_per_ptap; + + TRACE_FUNC("%lu %lu", g, test_bgn); + BFM_STAGE("wlevel"); + + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + + //USER maximum phases for the sweep + +#if USE_DQS_TRACKING +#if HHP_HPS + dtaps_per_ptap = IORD_32DIRECT(REG_FILE_DTAPS_PER_PTAP, 0); +#else + dtaps_per_ptap = IORD_32DIRECT(TRK_DTAPS_PER_PTAP, 0); +#endif +#else + dtaps_per_ptap = 0; + tmp_delay = 0; + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DCHAIN_TAP; + } + dtaps_per_ptap--; + tmp_delay = 0; +#endif + + //USER starting phases + + //USER update info for sims + + reg_file_set_group(g); + + //USER starting and end range where writes work + + scc_mgr_spread_out2_delay_all_ranks (g,test_bgn); + + work_bgn = 0; + work_end = 0; + + //USER step 1: find first working phase, increment in ptaps, and then in dtaps if ptaps doesn't find a working phase + found_begin = 0; + tmp_delay = 0; + for (d = 0; d <= dtaps_per_ptap; d++, tmp_delay += IO_DELAY_PER_DCHAIN_TAP) { + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + work_bgn = tmp_delay; + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX + num_additional_fr_cycles*IO_DLL_CHAIN_LENGTH; p++, work_bgn += IO_DELAY_PER_OPA_TAP) { + DPRINT(2, "wlevel: begin-1: p=%lu d=%lu", p, d); + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + found_begin = 1; + break; + } + } + + if (found_begin) { + break; + } + } + + if (p > IO_DQDQS_OUT_PHASE_MAX + num_additional_fr_cycles*IO_DLL_CHAIN_LENGTH) { + //USER fail, cannot find first working phase + + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + DPRINT(2, "wlevel: first valid p=%lu d=%lu", p, d); + + reg_file_set_sub_stage(CAL_SUBSTAGE_LAST_WORKING_DELAY); + + //USER If d is 0 then the working window covers a phase tap and we can follow the old procedure + //USER otherwise, we've found the beginning, and we need to increment the dtaps until we find the end + if (d == 0) { + COV(WLEVEL_PHASE_PTAP_OVERLAP); + work_end = work_bgn + IO_DELAY_PER_OPA_TAP; + + //USER step 2: if we have room, back off by one and increment in dtaps + + if (p > 0) { +#ifdef BFM_MODE + int found = 0; +#endif + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p - 1); + + tmp_delay = work_bgn - IO_DELAY_PER_OPA_TAP; + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX && tmp_delay < work_bgn; d++, tmp_delay += IO_DELAY_PER_DCHAIN_TAP) { + DPRINT(2, "wlevel: begin-2: p=%lu d=%lu", (p-1), d); + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { +#ifdef BFM_MODE + found = 1; +#endif + work_bgn = tmp_delay; + break; + } + } + +#ifdef BFM_MODE + { + alt_u32 d2; + alt_u32 p2; + if (found) { + d2 = d; + p2 = p - 1; + } else { + d2 = 0; + p2 = p; + } + + DPRINT(2, "wlevel: found begin-A: p=%lu d=%lu ps=%lu", p2, d2, work_bgn); + + BFM_GBL_SET(dqs_wlevel_left_edge[g].p,p2); + BFM_GBL_SET(dqs_wlevel_left_edge[g].d,d2); + BFM_GBL_SET(dqs_wlevel_left_edge[g].ps,work_bgn); + } +#endif + + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, 0); + } else { + DPRINT(2, "wlevel: found begin-B: p=%lu d=%lu ps=%lu", p, d, work_bgn); + + BFM_GBL_SET(dqs_wlevel_left_edge[g].p,p); + BFM_GBL_SET(dqs_wlevel_left_edge[g].d,d); + BFM_GBL_SET(dqs_wlevel_left_edge[g].ps,work_bgn); + } + + //USER step 3: go forward from working phase to non working phase, increment in ptaps + + for (p = p + 1; p <= IO_DQDQS_OUT_PHASE_MAX + num_additional_fr_cycles*IO_DLL_CHAIN_LENGTH; p++, work_end += IO_DELAY_PER_OPA_TAP) { + DPRINT(2, "wlevel: end-0: p=%lu d=%lu", p, (long unsigned int)0); + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + //USER step 4: back off one from last, increment in dtaps + //USER The actual increment is done outside the if/else statement since it is shared with other code + + p = p - 1; + + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + work_end -= IO_DELAY_PER_OPA_TAP; + d = 0; + + } else { + //USER step 5: Window doesn't cover phase tap, just increment dtaps until failure + //USER The actual increment is done outside the if/else statement since it is shared with other code + COV(WLEVEL_PHASE_PTAP_NO_OVERLAP); + work_end = work_bgn; + DPRINT(2, "wlevel: found begin-C: p=%lu d=%lu ps=%lu", p, d, work_bgn); + BFM_GBL_SET(dqs_wlevel_left_edge[g].p,p); + BFM_GBL_SET(dqs_wlevel_left_edge[g].d,d); + BFM_GBL_SET(dqs_wlevel_left_edge[g].ps,work_bgn); + + } + + //USER The actual increment until failure + for (; d <= IO_IO_OUT1_DELAY_MAX; d++, work_end += IO_DELAY_PER_DCHAIN_TAP) { + DPRINT(2, "wlevel: end: p=%lu d=%lu", p, d); + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + scc_mgr_zero_group (g, test_bgn, 1); + + work_end -= IO_DELAY_PER_DCHAIN_TAP; + + if (work_end >= work_bgn) { + //USER we have a working range + } else { + //USER nil range + + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_LAST_WORKING_DELAY); + + return 0; + } + + DPRINT(2, "wlevel: found end: p=%lu d=%lu; range: [%lu,%lu]", p, d-1, work_bgn, work_end); + BFM_GBL_SET(dqs_wlevel_right_edge[g].p,p); + BFM_GBL_SET(dqs_wlevel_right_edge[g].d,d-1); + BFM_GBL_SET(dqs_wlevel_right_edge[g].ps,work_end); + + for(sr = 0; sr < NUM_SHADOW_REGS; sr++) { + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[sr][g].dqdqs_start, work_bgn); + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[sr][g].dqdqs_end, work_end); + } + + //USER center + + work_mid = (work_bgn + work_end) / 2; + + DPRINT(2, "wlevel: work_mid=%ld", work_mid); + + tmp_delay = 0; + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX + num_additional_fr_cycles*IO_DLL_CHAIN_LENGTH && tmp_delay < work_mid; p++, tmp_delay += IO_DELAY_PER_OPA_TAP); + + if (tmp_delay > work_mid) { + tmp_delay -= IO_DELAY_PER_OPA_TAP; + p--; + } + + while (p > IO_DQDQS_OUT_PHASE_MAX) { + tmp_delay -= IO_DELAY_PER_OPA_TAP; + p--; + } + + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + DPRINT(2, "wlevel: p=%lu tmp_delay=%lu left=%lu", p, tmp_delay, work_mid - tmp_delay); + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX && tmp_delay < work_mid; d++, tmp_delay += IO_DELAY_PER_DCHAIN_TAP); + + if (tmp_delay > work_mid) { + tmp_delay -= IO_DELAY_PER_DCHAIN_TAP; + d--; + } + + DPRINT(2, "wlevel: p=%lu d=%lu tmp_delay=%lu left=%lu", p, d, tmp_delay, work_mid - tmp_delay); + + scc_mgr_apply_group_all_out_delay_add_all_ranks (g, test_bgn, d); + + DPRINT(2, "wlevel: found middle: p=%lu d=%lu", p, d); + BFM_GBL_SET(dqs_wlevel_mid[g].p,p); + BFM_GBL_SET(dqs_wlevel_mid[g].d,d); + BFM_GBL_SET(dqs_wlevel_mid[g].ps,work_mid); + + return 1; +} + + +#else + +//USER Write Levelling -- Full Calibration +alt_u32 rw_mgr_mem_calibrate_wlevel (alt_u32 g, alt_u32 test_bgn) +{ + alt_u32 p, d; + t_btfld bit_chk; + alt_u32 work_bgn, work_end, work_mid; + alt_u32 tmp_delay; + + TRACE_FUNC("%lu %lu", g, test_bgn); + + //USER update info for sims + + reg_file_set_stage(CAL_STAGE_WLEVEL); + reg_file_set_sub_stage(CAL_SUBSTAGE_WORKING_DELAY); + + //USER maximum phases for the sweep + + //USER starting phases + + //USER update info for sims + + reg_file_set_group(g); + + //USER starting and end range where writes work + + work_bgn = 0; + work_end = 0; + + //USER step 1: find first working phase, increment in ptaps + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX; p++, work_bgn += IO_DELAY_PER_OPA_TAP) { + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + if (p > IO_DQDQS_OUT_PHASE_MAX) { + //USER fail, cannot find first working phase + + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_WORKING_DELAY); + + return 0; + } + + work_end = work_bgn + IO_DELAY_PER_OPA_TAP; + + reg_file_set_sub_stage(CAL_SUBSTAGE_LAST_WORKING_DELAY); + + //USER step 2: if we have room, back off by one and increment in dtaps + + if (p > 0) { + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p - 1); + + tmp_delay = work_bgn - IO_DELAY_PER_OPA_TAP; + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX && tmp_delay < work_bgn; d++, tmp_delay += IO_DELAY_PER_DCHAIN_TAP) { + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + work_bgn = tmp_delay; + break; + } + } + + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, 0); + } + + //USER step 3: go forward from working phase to non working phase, increment in ptaps + + for (p = p + 1; p <= IO_DQDQS_OUT_PHASE_MAX; p++, work_end += IO_DELAY_PER_OPA_TAP) { + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p); + + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + //USER step 4: back off one from last, increment in dtaps + + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p - 1); + + work_end -= IO_DELAY_PER_OPA_TAP; + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++, work_end += IO_DELAY_PER_DCHAIN_TAP) { + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, d); + + if (!rw_mgr_mem_calibrate_write_test_all_ranks (g, 0, PASS_ONE_BIT, &bit_chk)) { + break; + } + } + + scc_mgr_apply_group_all_out_delay_all_ranks (g, test_bgn, 0); + + if (work_end > work_bgn) { + //USER we have a working range + } else { + //USER nil range + + set_failing_group_stage(g, CAL_STAGE_WLEVEL, CAL_SUBSTAGE_LAST_WORKING_DELAY); + + return 0; + } + + //USER center + + work_mid = (work_bgn + work_end) / 2; + + tmp_delay = 0; + + for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && tmp_delay < work_mid; p++, tmp_delay += IO_DELAY_PER_OPA_TAP); + + tmp_delay -= IO_DELAY_PER_OPA_TAP; + + scc_mgr_set_dqdqs_output_phase_all_ranks(g, p - 1); + + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX && tmp_delay < work_mid; d++, tmp_delay += IO_DELAY_PER_DCHAIN_TAP); + + scc_mgr_apply_group_all_out_delay_add_all_ranks (g, test_bgn, d - 1); + + + return 1; +} + +#endif +#endif +#endif + +//USER center all windows. do per-bit-deskew to possibly increase size of certain windows + +#if NEWVERSION_WRDESKEW + +alt_u32 rw_mgr_mem_calibrate_writes_center (alt_u32 rank_bgn, alt_u32 write_group, alt_u32 test_bgn) +{ + alt_u32 i, p, min_index; + alt_32 d; + //USER Store these as signed since there are comparisons with signed numbers + t_btfld bit_chk; +#if QDRII + t_btfld tmp_bit_chk; + t_btfld tmp_mask; + t_btfld mask; +#endif + t_btfld sticky_bit_chk; + alt_32 left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS]; + alt_32 right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS]; + alt_32 mid; + alt_32 mid_min, orig_mid_min; + alt_32 new_dqs, start_dqs, shift_dq; +#if RUNTIME_CAL_REPORT + alt_32 new_dq[RW_MGR_MEM_DQ_PER_WRITE_DQS]; +#endif + alt_32 dq_margin, dqs_margin, dm_margin; + alt_u32 stop; + + TRACE_FUNC("%lu %lu", write_group, test_bgn); + BFM_STAGE("writes_center"); + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + dm_margin = 0; + + start_dqs = READ_SCC_DQS_IO_OUT1_DELAY(); + + select_curr_shadow_reg_using_rank(rank_bgn); + + //USER per-bit deskew + + //USER set the left and right edge of each bit to an illegal value + //USER use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value + sticky_bit_chk = 0; + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; + right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; + } + + //USER Search for the left edge of the window for each bit + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_dq_out1_delay (write_group, test_bgn, d); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + //USER Stop searching when the read test doesn't pass AND when we've seen a passing read on every bit + stop = !rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ONE_BIT, &bit_chk, 0); + sticky_bit_chk = sticky_bit_chk | bit_chk; + stop = stop && (sticky_bit_chk == param->write_correct_mask); + DPRINT(2, "write_center(left): dtap=%lu => " BTFLD_FMT " == " BTFLD_FMT " && %lu [bit_chk=" BTFLD_FMT "]", + d, sticky_bit_chk, param->write_correct_mask, stop, bit_chk); + + if (stop == 1) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + if (bit_chk & 1) { + //USER Remember a passing test as the left_edge + left_edge[i] = d; + } else { + //USER If a left edge has not been seen yet, then a future passing test will mark this edge as the right edge + if (left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[i] = -(d + 1); + } + } + DPRINT(2, "write_center[l,d=%lu): bit_chk_test=%d left_edge[%lu]: %ld right_edge[%lu]: %ld", + d, (int)(bit_chk & 1), i, left_edge[i], i, right_edge[i]); + bit_chk = bit_chk >> 1; + } + } + } + + //USER Reset DQ delay chains to 0 + scc_mgr_apply_group_dq_out1_delay (write_group, test_bgn, 0); + sticky_bit_chk = 0; + for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) { + + DPRINT(2, "write_center: left_edge[%lu]: %ld right_edge[%lu]: %ld", i, left_edge[i], i, right_edge[i]); + + //USER Check for cases where we haven't found the left edge, which makes our assignment of the the + //USER right edge invalid. Reset it to the illegal value. + if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) && (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) { + right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; + DPRINT(2, "write_center: reset right_edge[%lu]: %ld", i, right_edge[i]); + } + + //USER Reset sticky bit (except for bits where we have seen the left edge) + sticky_bit_chk = sticky_bit_chk << 1; + if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) { + sticky_bit_chk = sticky_bit_chk | 1; + } + + if (i == 0) + { + break; + } + } + + //USER Search for the right edge of the window for each bit + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) { + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d + start_dqs); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + + //USER Stop searching when the read test doesn't pass AND when we've seen a passing read on every bit + stop = !rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ONE_BIT, &bit_chk, 0); + if (stop) { + recover_mem_device_after_ck_dqs_violation(); + } + sticky_bit_chk = sticky_bit_chk | bit_chk; + stop = stop && (sticky_bit_chk == param->write_correct_mask); + + DPRINT(2, "write_center (right): dtap=%lu => " BTFLD_FMT " == " BTFLD_FMT " && %lu", d, sticky_bit_chk, param->write_correct_mask, stop); + + if (stop == 1) { + if (d == 0) { + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + //USER d = 0 failed, but it passed when testing the left edge, so it must be marginal, set it to -1 + if (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1 && left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[i] = -1; + } + } + } + break; + } else { + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + if (bit_chk & 1) { + //USER Remember a passing test as the right_edge + right_edge[i] = d; + } else { + if (d != 0) { + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + if (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) { + left_edge[i] = -(d + 1); + } + } else { + //USER d = 0 failed, but it passed when testing the left edge, so it must be marginal, set it to -1 + if (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1 && left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[i] = -1; + } + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + else if (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) { + left_edge[i] = -(d + 1); + } + } + } + DPRINT(2, "write_center[r,d=%lu): bit_chk_test=%d left_edge[%lu]: %ld right_edge[%lu]: %ld", + d, (int)(bit_chk & 1), i, left_edge[i], i, right_edge[i]); + bit_chk = bit_chk >> 1; + } + } + } + +#if ENABLE_TCL_DEBUG + // Store all observed margins + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + alt_u32 dq = write_group*RW_MGR_MEM_DQ_PER_WRITE_DQS + i; + + ALTERA_ASSERT(dq < RW_MGR_MEM_DATA_WIDTH); + + TCLRPT_SET(debug_cal_report->cal_dq_out_margins[curr_shadow_reg][dq].left_edge, left_edge[i]); + TCLRPT_SET(debug_cal_report->cal_dq_out_margins[curr_shadow_reg][dq].right_edge, right_edge[i]); + } +#endif + + //USER Check that all bits have a window + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + DPRINT(2, "write_center: left_edge[%lu]: %ld right_edge[%lu]: %ld", i, left_edge[i], i, right_edge[i]); + BFM_GBL_SET(dq_write_left_edge[write_group][i],left_edge[i]); + BFM_GBL_SET(dq_write_right_edge[write_group][i],right_edge[i]); + if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) || (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) { + set_failing_group_stage(test_bgn + i, CAL_STAGE_WRITES, CAL_SUBSTAGE_WRITES_CENTER); + return 0; + } + } + + //USER Find middle of window for each DQ bit + mid_min = left_edge[0] - right_edge[0]; + min_index = 0; + for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + mid = left_edge[i] - right_edge[i]; + if (mid < mid_min) { + mid_min = mid; + min_index = i; + } + } + + //USER -mid_min/2 represents the amount that we need to move DQS. If mid_min is odd and positive we'll need to add one to + //USER make sure the rounding in further calculations is correct (always bias to the right), so just add 1 for all positive values + if (mid_min > 0) { + mid_min++; + } + mid_min = mid_min / 2; + + DPRINT(1, "write_center: mid_min=%ld", mid_min); + + //USER Determine the amount we can change DQS (which is -mid_min) + orig_mid_min = mid_min; +#if ENABLE_DQS_OUT_CENTERING + if (DDRX || RLDRAMX) { + new_dqs = start_dqs - mid_min; + DPRINT(2, "write_center: new_dqs(1)=%ld", new_dqs); + if (new_dqs > IO_IO_OUT1_DELAY_MAX) { + new_dqs = IO_IO_OUT1_DELAY_MAX; + } else if (new_dqs < 0) { + new_dqs = 0; + } + mid_min = start_dqs - new_dqs; + + new_dqs = start_dqs - mid_min; + } else { + new_dqs = start_dqs; + mid_min = 0; + } +#else + new_dqs = start_dqs; + mid_min = 0; +#endif + + DPRINT(1, "write_center: start_dqs=%ld new_dqs=%ld mid_min=%ld", start_dqs, new_dqs, mid_min); + + //USER Initialize data for export structures + dqs_margin = IO_IO_OUT1_DELAY_MAX + 1; + dq_margin = IO_IO_OUT1_DELAY_MAX + 1; + + //USER add delay to bring centre of all DQ windows to the same "level" + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + //USER Use values before divide by 2 to reduce round off error + shift_dq = (left_edge[i] - right_edge[i] - (left_edge[min_index] - right_edge[min_index]))/2 + (orig_mid_min - mid_min); + + DPRINT(2, "write_center: before: shift_dq[%lu]=%ld", i, shift_dq); + + if (shift_dq + (alt_32)READ_SCC_DQ_OUT1_DELAY(i) > (alt_32)IO_IO_OUT1_DELAY_MAX) { + shift_dq = (alt_32)IO_IO_OUT1_DELAY_MAX - READ_SCC_DQ_OUT1_DELAY(i); + } else if (shift_dq + (alt_32)READ_SCC_DQ_OUT1_DELAY(i) < 0) { + shift_dq = -(alt_32)READ_SCC_DQ_OUT1_DELAY(i); + } +#if RUNTIME_CAL_REPORT + new_dq[i] = shift_dq; +#endif + DPRINT(2, "write_center: after: shift_dq[%lu]=%ld", i, shift_dq); + scc_mgr_set_dq_out1_delay(write_group, i, READ_SCC_DQ_OUT1_DELAY(i) + shift_dq); + scc_mgr_load_dq (i); + + DPRINT(2, "write_center: margin[%lu]=[%ld,%ld]", i, + left_edge[i] - shift_dq + (-mid_min), + right_edge[i] + shift_dq - (-mid_min)); + //USER To determine values for export structures + if (left_edge[i] - shift_dq + (-mid_min) < dq_margin) { + dq_margin = left_edge[i] - shift_dq + (-mid_min); + } + if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin) { + dqs_margin = right_edge[i] + shift_dq - (-mid_min); + } + } + + //USER Move DQS + if (QDRII) { + scc_mgr_set_group_dqs_io_and_oct_out1_gradual (write_group, new_dqs); + } else { + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, new_dqs); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + +#if RUNTIME_CAL_REPORT + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + RPRINT("Write Deskew ; DQ %2lu ; Rank %lu ; Left edge %3li ; Right edge %3li ; DQ delay %2li ; DQS delay %2li", write_group*RW_MGR_MEM_DQ_PER_WRITE_DQS + i, rank_bgn, left_edge[i], right_edge[i], new_dq[i], new_dqs); + } +#endif + + + ////////////////////// + ////////////////////// + //USER Centre DM + ////////////////////// + ////////////////////// + + BFM_STAGE("dm_center"); + + DPRINT(2, "write_center: DM"); + +#if RLDRAMX + + //Note: this is essentially the same as DDR with the exception of the dm_ global accounting + + //USER Determine if first group in device to initialize left and right edges + if (!is_write_group_enabled_for_dm(write_group)) + { + DPRINT(2, "dm_calib: skipping since not last in group"); + } + else + { + + // last in the group, so we need to do DM + DPRINT(2, "dm_calib: calibrating DM since last in group"); + + //USER set the left and right edge of each bit to an illegal value + //USER use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value + left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; + right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; + + sticky_bit_chk = 0; + //USER Search for the left edge of the window for the DM bit + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_dm_out1_delay (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + //USER Stop searching when the write test doesn't pass AND when we've seen a passing write before + if (rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) { + DPRINT(2, "dm_calib: left=%lu passed", d); + left_edge[0] = d; + } else { + DPRINT(2, "dm_calib: left=%lu failed", d); + //USER If a left edge has not been seen yet, then a future passing test will mark this edge as the right edge + if (left_edge[0] == IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[0] = -(d + 1); + } else { + //USER left edge has been seen, so this failure marks the left edge, and we are done + break; + } + } + DPRINT(2, "dm_calib[l,d=%lu]: left_edge: %ld right_edge: %ld", + d, left_edge[0], right_edge[0]); + } + + DPRINT(2, "dm_calib left done: left_edge: %ld right_edge: %ld", + left_edge[0], right_edge[0]); + + //USER Reset DM delay chains to 0 + scc_mgr_apply_group_dm_out1_delay (write_group, 0); + + //USER Check for cases where we haven't found the left edge, which makes our assignment of the the + //USER right edge invalid. Reset it to the illegal value. + if ((left_edge[0] == IO_IO_OUT1_DELAY_MAX + 1) && (right_edge[0] != IO_IO_OUT1_DELAY_MAX + 1)) { + right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; + DPRINT(2, "dm_calib: reset right_edge: %ld", right_edge[0]); + } + + //USER Search for the right edge of the window for the DM bit + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d++) { + // Note: This only shifts DQS, so are we limiting ourselve to + // width of DQ unnecessarily + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d + new_dqs); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + //USER Stop searching when the test fails and we've seen passing test already + if (rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) { + DPRINT(2, "dm_calib: right=%lu passed", d); + right_edge[0] = d; + } else { + recover_mem_device_after_ck_dqs_violation(); + + DPRINT(2, "dm_calib: right=%lu failed", d); + if (d != 0) { + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + if (right_edge[0] == IO_IO_OUT1_DELAY_MAX + 1) { + left_edge[0] = -(d + 1); + } else { + break; + } + } else { + //USER d = 0 failed, but it passed when testing the left edge, so it must be marginal, set it to -1 + if (right_edge[0] == IO_IO_OUT1_DELAY_MAX + 1 && left_edge[0] != IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[0] = -1; + // we're done + break; + } + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + else if (right_edge[0] == IO_IO_OUT1_DELAY_MAX + 1) { + left_edge[0] = -(d + 1); + } + } + } + DPRINT(2, "dm_calib[l,d=%lu]: left_edge: %ld right_edge: %ld", + d, left_edge[0], right_edge[0]); + } + + DPRINT(2, "dm_calib: left=%ld right=%ld", left_edge[0], right_edge[0]); +#if BFM_MODE + // need to update for all groups covered by this dm + for (i = write_group+1-(RW_MGR_MEM_IF_WRITE_DQS_WIDTH/RW_MGR_MEM_DATA_MASK_WIDTH); i <= write_group; i++) + { + DPRINT(3, "dm_calib: left[%d]=%ld right[%d]=%ld", i, left_edge[0], i, right_edge[0]); + BFM_GBL_SET(dm_left_edge[i][0],left_edge[0]); + BFM_GBL_SET(dm_right_edge[i][0],right_edge[0]); + } +#endif + + //USER Move DQS (back to orig) + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, new_dqs); + + //USER move DM + //USER Find middle of window for the DM bit + mid = (left_edge[0] - right_edge[0]) / 2; + if (mid < 0) { + mid = 0; + } + scc_mgr_apply_group_dm_out1_delay (write_group, mid); + + dm_margin = left_edge[0]; + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + DPRINT(2, "dm_calib: left=%ld right=%ld mid=%ld dm_margin=%ld", + left_edge[0], right_edge[0], mid, dm_margin); + } // end of DM calibration +#endif + + +#if DDRX + //USER set the left and right edge of each bit to an illegal value + //USER use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value + left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; + right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1; + alt_32 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; + alt_32 end_curr = IO_IO_OUT1_DELAY_MAX + 1; + alt_32 bgn_best = IO_IO_OUT1_DELAY_MAX + 1; + alt_32 end_best = IO_IO_OUT1_DELAY_MAX + 1; + alt_32 win_best = 0; + + //USER Search for the/part of the window with DM shift + for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d-=DELTA_D) { + scc_mgr_apply_group_dm_out1_delay (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) { + + //USE Set current end of the window + end_curr = -d; + //USER If a starting edge of our window has not been seen this is our current start of the DM window + if(bgn_curr == IO_IO_OUT1_DELAY_MAX + 1){ + bgn_curr = -d; + } + + //USER If current window is bigger than best seen. Set best seen to be current window + if((end_curr-bgn_curr+1) > win_best ){ + win_best = end_curr-bgn_curr+1; + bgn_best = bgn_curr; + end_best = end_curr; + } + } else { + //USER We just saw a failing test. Reset temp edge + bgn_curr=IO_IO_OUT1_DELAY_MAX + 1; + end_curr=IO_IO_OUT1_DELAY_MAX + 1; + } + + + } + + + //USER Reset DM delay chains to 0 + scc_mgr_apply_group_dm_out1_delay (write_group, 0); + + //USER Check to see if the current window nudges up aganist 0 delay. If so we need to continue the search by shifting DQS otherwise DQS search begins as a new search + if(end_curr!=0) { + bgn_curr=IO_IO_OUT1_DELAY_MAX + 1; + end_curr=IO_IO_OUT1_DELAY_MAX + 1; + } + + //USER Search for the/part of the window with DQS shifts + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d+=DELTA_D) { + // Note: This only shifts DQS, so are we limiting ourselve to + // width of DQ unnecessarily + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d + new_dqs); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) { + + //USE Set current end of the window + end_curr = d; + //USER If a beginning edge of our window has not been seen this is our current begin of the DM window + if(bgn_curr == IO_IO_OUT1_DELAY_MAX + 1){ + bgn_curr = d; + } + + //USER If current window is bigger than best seen. Set best seen to be current window + if((end_curr-bgn_curr+1) > win_best){ + win_best = end_curr-bgn_curr+1; + bgn_best = bgn_curr; + end_best = end_curr; + } + } else { + //USER We just saw a failing test. Reset temp edge + recover_mem_device_after_ck_dqs_violation(); + bgn_curr = IO_IO_OUT1_DELAY_MAX + 1; + end_curr = IO_IO_OUT1_DELAY_MAX + 1; + + //USER Early exit optimization: if ther remaining delay chain space is less than already seen largest window we can exit + if((win_best-1) > (IO_IO_OUT1_DELAY_MAX - new_dqs - d)){ + break; + } + + } + } + + //USER assign left and right edge for cal and reporting; + left_edge[0] = -1*bgn_best; + right_edge[0] = end_best; + + DPRINT(2, "dm_calib: left=%ld right=%ld", left_edge[0], right_edge[0]); + BFM_GBL_SET(dm_left_edge[write_group][0],left_edge[0]); + BFM_GBL_SET(dm_right_edge[write_group][0],right_edge[0]); + + //USER Move DQS (back to orig) + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, new_dqs); + + //USER Move DM + + //USER Find middle of window for the DM bit + mid = (left_edge[0] - right_edge[0]) / 2; + + //USER only move right, since we are not moving DQS/DQ + if (mid < 0) { + mid = 0; + } + + //dm_marign should fail if we never find a window + if(win_best==0){ + dm_margin = -1; + }else{ + dm_margin = left_edge[0] - mid; + } + + + + scc_mgr_apply_group_dm_out1_delay(write_group, mid); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + DPRINT(2, "dm_calib: left=%ld right=%ld mid=%ld dm_margin=%ld", + left_edge[0], right_edge[0], mid, dm_margin); +#endif + +#if QDRII + sticky_bit_chk = 0; + + //USER set the left and right edge of each bit to an illegal value + //USER use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + left_edge[i] = right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; + } + + mask = param->dm_correct_mask; + + //USER Search for the left edge of the window for the DM bit + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_dm_out1_delay (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + //USER Stop searching when the read test doesn't pass for all bits (as they've already been calibrated) + stop = !rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ONE_BIT, &bit_chk, 0); + DPRINT(2, "dm_calib[l,d=%lu] stop=%ld bit_chk=%llx sticky_bit_chk=%llx mask=%llx", + d, stop, bit_chk, sticky_bit_chk, param->write_correct_mask); + tmp_bit_chk = bit_chk; + tmp_mask = mask; + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + if ( (tmp_bit_chk & mask) == mask ) { + sticky_bit_chk = sticky_bit_chk | tmp_mask; + } + tmp_bit_chk = tmp_bit_chk >> (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + tmp_mask = tmp_mask << (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + } + stop = stop && (sticky_bit_chk == param->write_correct_mask); + + if (stop == 1) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + DPRINT(2, "dm_calib[l,i=%lu] d=%lu bit_chk&dm_mask=" BTFLD_FMT " == " BTFLD_FMT, i, d, + bit_chk & mask, mask); + if ((bit_chk & mask) == mask) { + DPRINT(2, "dm_calib: left[%lu]=%lu", i, d); + left_edge[i] = d; + } else { + //USER If a left edge has not been seen yet, then a future passing test will mark this edge as the right edge + if (left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[i] = -(d + 1); + } + } + bit_chk = bit_chk >> (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + } + } + } + + //USER Reset DM delay chains to 0 + scc_mgr_apply_group_dm_out1_delay (write_group, 0); + + //USER Check for cases where we haven't found the left edge, which makes our assignment of the the + //USER right edge invalid. Reset it to the illegal value. + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) && (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) { + right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1; + DPRINT(2, "dm_calib: reset right_edge: %d", right_edge[i]); + } + } + + //USER Search for the right edge of the window for the DM bit + for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d++) { + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d + new_dqs); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + rw_mgr_mem_dll_lock_wait(); + + //USER Stop searching when the read test doesn't pass for all bits (as they've already been calibrated) + stop = !rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ONE_BIT, &bit_chk, 0); + DPRINT(2, "dm_calib[l,d=%lu] stop=%ld bit_chk=%llx sticky_bit_chk=%llx mask=%llx", + d, stop, bit_chk, sticky_bit_chk, param->write_correct_mask); + tmp_bit_chk = bit_chk; + tmp_mask = mask; + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + if ( (tmp_bit_chk & mask) == mask ) { + sticky_bit_chk = sticky_bit_chk | tmp_mask; + } + tmp_bit_chk = tmp_bit_chk >> (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + tmp_mask = tmp_mask << (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + } + stop = stop && (sticky_bit_chk == param->write_correct_mask); + + if (stop == 1) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + DPRINT(2, "dm_calib[r,i=%lu] d=%lu bit_chk&dm_mask=" BTFLD_FMT " == " BTFLD_FMT, i, d, + bit_chk & mask, mask); + if ((bit_chk & mask) == mask) { + right_edge[i] = d; + } else { + //USER d = 0 failed, but it passed when testing the left edge, so it must be marginal, set it to -1 + if (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1 && left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1) { + right_edge[i] = -1; + // we're done + break; + } + //USER If a right edge has not been seen yet, then a future passing test will mark this edge as the left edge + else if (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) { + left_edge[i] = -(d + 1); + } + } + bit_chk = bit_chk >> (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + } + } + } + + //USER Move DQS (back to orig) + scc_mgr_set_group_dqs_io_and_oct_out1_gradual (write_group, new_dqs); + + //USER Move DM + dm_margin = IO_IO_OUT1_DELAY_MAX; + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + //USER Find middle of window for the DM bit + mid = (left_edge[i] - right_edge[i]) / 2; + DPRINT(2, "dm_calib[mid,i=%lu] left=%ld right=%ld mid=%ld", i, left_edge[i], right_edge[i], mid); + BFM_GBL_SET(dm_left_edge[write_group][i],left_edge[i]); + BFM_GBL_SET(dm_right_edge[write_group][i],right_edge[i]); + if (mid < 0) { + mid = 0; + } + scc_mgr_set_dm_out1_delay(write_group, i, mid); + scc_mgr_load_dm (i); + if ((left_edge[i] - mid) < dm_margin) { + dm_margin = left_edge[i] - mid; + } + } +#endif + + // Store observed DM margins +#if RLDRAMX + if (is_write_group_enabled_for_dm(write_group)) + { + TCLRPT_SET(debug_cal_report->cal_dm_margins[curr_shadow_reg][write_group][0].left_edge, left_edge[0]); + TCLRPT_SET(debug_cal_report->cal_dm_margins[curr_shadow_reg][write_group][0].right_edge, right_edge[0]); + } +#else + for (i = 0; i < RW_MGR_NUM_TRUE_DM_PER_WRITE_GROUP; i++) { + TCLRPT_SET(debug_cal_report->cal_dm_margins[curr_shadow_reg][write_group][i].left_edge, left_edge[i]); + TCLRPT_SET(debug_cal_report->cal_dm_margins[curr_shadow_reg][write_group][i].right_edge, right_edge[i]); + } +#endif + +#if RUNTIME_CAL_REPORT + for (i = 0; i < RW_MGR_NUM_TRUE_DM_PER_WRITE_GROUP; i++) { + RPRINT("DM Deskew ; Group %lu ; Left edge %3li; Right edge %3li; DM delay %2li", write_group, left_edge[i], right_edge[i], mid); + } +#endif + + //USER Export values + gbl->fom_out += dq_margin + dqs_margin; + + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[curr_shadow_reg][write_group].dqs_margin, dqs_margin); + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[curr_shadow_reg][write_group].dq_margin, dq_margin); + +#if RLDRAMX + if (is_write_group_enabled_for_dm(write_group)) + { + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[curr_shadow_reg][write_group].dm_margin, dm_margin); + } +#else + TCLRPT_SET(debug_cal_report->cal_dqs_out_margins[curr_shadow_reg][write_group].dm_margin, dm_margin); +#endif + TCLRPT_SET(debug_summary_report->fom_out, debug_summary_report->fom_out + (dq_margin + dqs_margin)); + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][write_group].fom_out, (dq_margin + dqs_margin)); + + DPRINT(2, "write_center: dq_margin=%ld dqs_margin=%ld dm_margin=%ld", dq_margin, dqs_margin, dm_margin); + + //USER Do not remove this line as it makes sure all of our decisions have been applied + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0); +} + +#else // !NEWVERSION_WRDESKEW + +alt_u32 rw_mgr_mem_calibrate_writes_center (alt_u32 rank_bgn, alt_u32 write_group, alt_u32 test_bgn) +{ + alt_u32 i, p, d; + alt_u32 mid; + t_btfld bit_chk, sticky_bit_chk; + alt_u32 max_working_dq[RW_MGR_MEM_DQ_PER_WRITE_DQS]; + alt_u32 max_working_dm[RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH]; + alt_u32 dq_margin, dqs_margin, dm_margin; + alt_u32 start_dqs; + alt_u32 stop; + + TRACE_FUNC("%lu %lu", write_group, test_bgn); + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + //USER per-bit deskew + + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + max_working_dq[i] = 0; + } + + for (d = 1; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_dq_out1_delay (write_group, test_bgn, d); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (!rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ONE_BIT, &bit_chk, 0)) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + if (bit_chk & 1) { + max_working_dq[i] = d; + } + bit_chk = bit_chk >> 1; + } + } + } + + scc_mgr_apply_group_dq_out1_delay (write_group, test_bgn, 0); + + //USER determine minimum of maximums + + dq_margin = IO_IO_OUT1_DELAY_MAX; + + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + if (max_working_dq[i] < dq_margin) { + dq_margin = max_working_dq[i]; + } + } + + //USER add delay to center DQ windows + + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + if (max_working_dq[i] > dq_margin) { + scc_mgr_set_dq_out1_delay(write_group, i, max_working_dq[i] - dq_margin); + } else { + scc_mgr_set_dq_out1_delay(write_group, i, 0); + } + + scc_mgr_load_dq (p, i); + } + + //USER sweep DQS window, may potentially have more window due to per-bit-deskew + + start_dqs = READ_SCC_DQS_IO_OUT1_DELAY(); + + for (d = start_dqs + 1; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, d); + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + + if (!rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ALL_BITS, &bit_chk, 0)) { + break; + } + } + + scc_mgr_set_dqs_out1_delay(write_group, start_dqs); + scc_mgr_set_oct_out1_delay(write_group, start_dqs); + + dqs_margin = d - start_dqs - 1; + + //USER time to center, +1 so that we don't go crazy centering DQ + + mid = (dq_margin + dqs_margin + 1) / 2; + + gbl->fom_out += dq_margin + dqs_margin; + TCLRPT_SET(debug_summary_report->fom_out, debug_summary_report->fom_out + (dq_margin + dqs_margin)); + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][grp].fom_out, (dq_margin + dqs_margin)); + +#if ENABLE_DQS_OUT_CENTERING + //USER center DQS ... if the headroom is setup properly we shouldn't need to + if (DDRX) { + if (dqs_margin > mid) { + scc_mgr_set_dqs_out1_delay(write_group, READ_SCC_DQS_IO_OUT1_DELAY() + dqs_margin - mid); + scc_mgr_set_oct_out1_delay(write_group, READ_SCC_OCT_OUT1_DELAY(write_group) + dqs_margin - mid); + } + } +#endif + + scc_mgr_load_dqs_io (); + scc_mgr_load_dqs_for_write_group (write_group); + + //USER center dq + + if (dq_margin > mid) { + for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) { + scc_mgr_set_dq_out1_delay(write_group, i, READ_SCC_DQ_OUT1_DELAY(i) + dq_margin - mid); + scc_mgr_load_dq (p, i); + } + dqs_margin += dq_margin - mid; + dq_margin -= dq_margin - mid; + } + + //USER do dm centering + + if (!RLDRAMX) { + dm_margin = IO_IO_OUT1_DELAY_MAX; + + if (QDRII) { + sticky_bit_chk = 0; + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + max_working_dm[i] = 0; + } + } + + for (d = 1; d <= IO_IO_OUT1_DELAY_MAX; d++) { + scc_mgr_apply_group_dm_out1_delay (write_group, d); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + if (DDRX) { + if (rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) { + max_working_dm[0] = d; + } else { + break; + } + } else { + stop = !rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0); + sticky_bit_chk = sticky_bit_chk | bit_chk; + stop = stop && (sticky_bit_chk == param->read_correct_mask); + + if (stop == 1) { + break; + } else { + for (i = 0; i < RW_MGR_MEM_DATA_MASK_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + if ((bit_chk & param->dm_correct_mask) == param->dm_correct_mask) { + max_working_dm[i] = d; + } + bit_chk = bit_chk >> (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH); + } + } + } + } + + i = 0; + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { + if (max_working_dm[i] > mid) { + scc_mgr_set_dm_out1_delay(write_group, i, max_working_dm[i] - mid); + } else { + scc_mgr_set_dm_out1_delay(write_group, i, 0); + } + + scc_mgr_load_dm (i); + + if (max_working_dm[i] < dm_margin) { + dm_margin = max_working_dm[i]; + } + } + } else { + dm_margin = 0; + } + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + return (dq_margin + dqs_margin) > 0; +} + +#endif + +//USER calibrate the write operations + +alt_u32 rw_mgr_mem_calibrate_writes (alt_u32 rank_bgn, alt_u32 g, alt_u32 test_bgn) +{ + //USER update info for sims + + TRACE_FUNC("%lu %lu", g, test_bgn); + + reg_file_set_stage(CAL_STAGE_WRITES); + reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER); + + //USER starting phases + + //USER update info for sims + + reg_file_set_group(g); + + if (!rw_mgr_mem_calibrate_writes_center (rank_bgn, g, test_bgn)) { + set_failing_group_stage(g, CAL_STAGE_WRITES, CAL_SUBSTAGE_WRITES_CENTER); + return 0; + } + + + return 1; +} + +// helpful for creating eye diagrams +// TODO: This is for the TCL DBG... but obviously it serves no purpose... +// Decide what to do with it! + +void rw_mgr_mem_calibrate_eye_diag_aid (void) +{ + // no longer exists +} + +// TODO: This needs to be update to properly handle the number of failures +// Right now it only checks if the write test was successful or not +alt_u32 rw_mgr_mem_calibrate_full_test (alt_u32 min_correct, t_btfld *bit_chk, alt_u32 test_dm) +{ + alt_u32 g; + alt_u32 success = 0; + alt_u32 run_groups = ~param->skip_groups; + + TRACE_FUNC("%lu %lu", min_correct, test_dm); + + for (g = 0; g < RW_MGR_MEM_IF_READ_DQS_WIDTH; g++) { + if (run_groups & ((1 << RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1)) + { + success = rw_mgr_mem_calibrate_write_test_all_ranks (g, test_dm, PASS_ALL_BITS, bit_chk); + } + run_groups = run_groups >> RW_MGR_NUM_DQS_PER_WRITE_GROUP; + } + + return success; +} + +#if ENABLE_TCL_DEBUG +// see how far we can push a particular DQ pin before complete failure on input and output sides +// NOTE: if ever executing a run_*_margining function outside of calibration context you must first issue IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 1); +void run_dq_margining (alt_u32 rank_bgn, alt_u32 write_group) +{ + alt_u32 test_num; + alt_u32 read_group; + alt_u32 read_test_bgn; + alt_u32 subdq; + alt_u32 dq; + alt_u32 delay; + alt_u32 calibrated_delay; + alt_u32 working_cnt; + t_btfld bit_chk; + t_btfld bit_chk_test = 0; + t_btfld bit_chk_mask; + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + select_curr_shadow_reg_using_rank(rank_bgn); + + // Load the read patterns + rw_mgr_mem_calibrate_read_load_patterns (rank_bgn, 0); + + // sweep input delays + for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH, read_test_bgn = 0; + read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH; + read_group++, read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) + { + + ALTERA_ASSERT(read_group < RW_MGR_MEM_IF_READ_DQS_WIDTH); + + for (subdq = 0; subdq < RW_MGR_MEM_DQ_PER_READ_DQS; subdq++) + { + dq = read_group*RW_MGR_MEM_DQ_PER_READ_DQS + subdq; + + ALTERA_ASSERT(dq < RW_MGR_MEM_DATA_WIDTH); + + calibrated_delay = debug_cal_report->cal_dq_settings[curr_shadow_reg][dq].dq_in_delay; + + working_cnt = 0; + + bit_chk_test = 0; + + // Find the left edge + for (delay = calibrated_delay; delay <= IO_IO_IN_DELAY_MAX; delay++) + { + WRITE_SCC_DQ_IN_DELAY((subdq + read_test_bgn), delay); + scc_mgr_load_dq (subdq + read_test_bgn); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + for (test_num = 0; test_num < NUM_READ_TESTS; test_num++) + { + rw_mgr_mem_calibrate_read_test (rank_bgn, read_group, 1, PASS_ONE_BIT, &bit_chk, 0, 0); + if (test_num == 0) + { + bit_chk_test = bit_chk; + } + else + { + bit_chk_test &= bit_chk; + } + } + + // Check only the bit we are testing + bit_chk_mask = (bit_chk_test & (((t_btfld) 1) << ((t_btfld) subdq))); + if (bit_chk_mask == 0) + { + break; + } + + working_cnt++; + } + + // Restore the settings + WRITE_SCC_DQ_IN_DELAY((subdq + read_test_bgn), calibrated_delay); + scc_mgr_load_dq (subdq + read_test_bgn); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + // Store the setting + TCLRPT_SET(debug_margin_report->margin_dq_in_margins[curr_shadow_reg][dq].min_working_setting, working_cnt); + + // Find the right edge + calibrated_delay = debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][read_group].dqs_bus_in_delay; + + working_cnt = 0; + for (delay = calibrated_delay; delay <= IO_DQS_IN_DELAY_MAX; delay++) + { + WRITE_SCC_DQS_IN_DELAY(read_group, delay); + scc_mgr_load_dqs(read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + for (test_num = 0; test_num < NUM_READ_TESTS; test_num++) + { + rw_mgr_mem_calibrate_read_test (rank_bgn, read_group, 1, PASS_ONE_BIT, &bit_chk, 0, 0); + if (test_num == 0) + { + bit_chk_test = bit_chk; + } + else + { + bit_chk_test &= bit_chk; + } + } + + // Check only the bit we are testing + bit_chk_mask = (bit_chk_test & (((t_btfld)1) << ((t_btfld)(subdq)))); + if (bit_chk_mask == 0) + { + break; + } + + working_cnt++; + } + + // Restore the settings + WRITE_SCC_DQS_IN_DELAY(read_group, calibrated_delay); + scc_mgr_load_dqs(read_group); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + // Store the setting + TCLRPT_SET(debug_margin_report->margin_dq_in_margins[curr_shadow_reg][dq].max_working_setting, working_cnt); + + } + } + + // sweep output delays + for (subdq = 0; subdq < RW_MGR_MEM_DQ_PER_WRITE_DQS; subdq++) + { + dq = write_group*RW_MGR_MEM_DQ_PER_WRITE_DQS + subdq; + + calibrated_delay = debug_cal_report->cal_dq_settings[curr_shadow_reg][dq].dq_out_delay1; + working_cnt = 0; + + // Find the left edge + for (delay = calibrated_delay; delay <= IO_IO_OUT1_DELAY_MAX; delay++) + { + WRITE_SCC_DQ_OUT1_DELAY(subdq, delay); + scc_mgr_load_dq (subdq); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + for (test_num = 0; test_num < NUM_WRITE_TESTS; test_num++) + { + rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ALL_BITS, &bit_chk, 0); + if (test_num == 0) + { + bit_chk_test = bit_chk; + } + else + { + bit_chk_test &= bit_chk; + } + } + + // Check only the bit we are testing + bit_chk_mask = (bit_chk_test & (((t_btfld)1) << ((t_btfld)subdq))); + if (bit_chk_mask == 0) + { + break; + } + + working_cnt++; + } + + // Restore the settings + WRITE_SCC_DQ_OUT1_DELAY(subdq, calibrated_delay); + scc_mgr_load_dq (subdq); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + // Store the setting + TCLRPT_SET(debug_margin_report->margin_dq_out_margins[curr_shadow_reg][dq].min_working_setting, working_cnt); + + // Find the right edge + calibrated_delay = debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].dqs_out_delay1; + + working_cnt = 0; + for (delay = calibrated_delay; delay <= IO_IO_OUT1_DELAY_MAX; delay++) + { + WRITE_SCC_DQS_IO_OUT1_DELAY(delay); + scc_mgr_load_dqs_io(); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + + for (test_num = 0; test_num < NUM_WRITE_TESTS; test_num++) + { + rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 0, PASS_ONE_BIT, &bit_chk, 0); + if (test_num == 0) + { + bit_chk_test = bit_chk; + } + else + { + bit_chk_test &= bit_chk; + } + } + + // Check only the bit we are testing + bit_chk_mask = (bit_chk_test & (((t_btfld)1) << ((t_btfld)subdq))); + if (bit_chk_mask == 0) + { + break; + } + + working_cnt++; + } + + //USER Restore the settings + if (QDRII) { + scc_mgr_set_group_dqs_io_and_oct_out1_gradual (write_group, calibrated_delay); + } else { + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, calibrated_delay); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + + // Store the setting + TCLRPT_SET(debug_margin_report->margin_dq_out_margins[curr_shadow_reg][dq].max_working_setting, working_cnt); + } +} +#endif + +#if ENABLE_TCL_DEBUG +// NOTE: if ever executing a run_*_margining function outside of calibration context you must first issue IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 1); +void run_dm_margining (alt_u32 rank_bgn, alt_u32 write_group) +{ + alt_u32 test_status; + alt_u32 test_num; + alt_u32 dm; + alt_u32 delay; + alt_u32 calibrated_delay; + alt_u32 working_cnt; + t_btfld bit_chk; + + ALTERA_ASSERT(write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH); + + select_curr_shadow_reg_using_rank(rank_bgn); + + // sweep output delays + for (dm = 0; dm < RW_MGR_NUM_DM_PER_WRITE_GROUP; dm++) + { + + calibrated_delay = debug_cal_report->cal_dm_settings[curr_shadow_reg][write_group][dm].dm_out_delay1; + working_cnt = 0; + + // Find the left edge + for (delay = calibrated_delay; delay <= IO_IO_OUT1_DELAY_MAX; delay++) + { + WRITE_SCC_DM_IO_OUT1_DELAY(dm, delay); + scc_mgr_load_dm (dm); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + test_status = 1; + for (test_num = 0; test_num < NUM_WRITE_TESTS; test_num++) + { + if (!rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) + { + test_status = 0; + break; + } + } + + if (test_status == 0) + { + break; + } + + working_cnt++; + } + + // Restore the settings + WRITE_SCC_DM_IO_OUT1_DELAY(dm, calibrated_delay); + scc_mgr_load_dm (dm); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + // Store the setting + TCLRPT_SET(debug_margin_report->margin_dm_margins[curr_shadow_reg][write_group][dm].min_working_setting, working_cnt); + + // Find the right edge + calibrated_delay = debug_cal_report->cal_dqs_out_settings[curr_shadow_reg][write_group].dqs_out_delay1; + + working_cnt = 0; + for (delay = calibrated_delay; delay <= IO_IO_OUT1_DELAY_MAX; delay++) + { + WRITE_SCC_DQS_IO_OUT1_DELAY(delay); + scc_mgr_load_dqs_io(); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + if (QDRII) + { + rw_mgr_mem_dll_lock_wait(); + } + + test_status = 1; + for (test_num = 0; test_num < NUM_WRITE_TESTS; test_num++) + { + if (!rw_mgr_mem_calibrate_write_test (rank_bgn, write_group, 1, PASS_ALL_BITS, &bit_chk, 0)) + { + test_status = 0; + break; + } + } + + if (test_status == 0) + { + break; + } + + working_cnt++; + } + + //USER Restore the settings + if (QDRII) { + scc_mgr_set_group_dqs_io_and_oct_out1_gradual (write_group, calibrated_delay); + } else { + scc_mgr_apply_group_dqs_io_and_oct_out1 (write_group, calibrated_delay); + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + + // Store the setting + TCLRPT_SET(debug_margin_report->margin_dm_margins[curr_shadow_reg][write_group][dm].max_working_setting, working_cnt); + + } +} +#endif + + +//USER precharge all banks and activate row 0 in bank "000..." and bank "111..." +#if DDRX +void mem_precharge_and_activate (void) +{ + alt_u32 r; + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF); + + //USER precharge all banks ... + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x0F); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_ACTIVATE_0_AND_1_WAIT1); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x0F); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_ACTIVATE_0_AND_1_WAIT2); + + //USER activate rows + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_ACTIVATE_0_AND_1); + } +} +#endif + +#if QDRII || RLDRAMX +void mem_precharge_and_activate (void) {} +#endif + +//USER perform all refreshes necessary over all ranks +#if (ENABLE_NON_DESTRUCTIVE_CALIB || ENABLE_NON_DES_CAL) +// Only have DDR3 version for now +#if DDR3 +alt_u32 mem_refresh_all_ranks (alt_u32 no_validate) +{ + const alt_u32 T_REFI_NS = 3900; // JEDEC spec refresh interval in ns (industrial temp) +// const alt_u32 T_RFC_NS = 350; // Worst case REFRESH-REFRESH or REFRESH-ACTIVATE wait time in ns + // Alternatively, we could extract T_RFC from uniphy_gen.tcl + const alt_u32 T_RFC_AFI = 350 * AFI_CLK_FREQ / 1000; // T_RFC expressed in mem clk cycles (will be less than 256) +#if (ENABLE_NON_DESTRUCTIVE_CALIB) + const alt_u32 NUM_REFRESH_POSTING = 8192; // Number of consecutive refresh commands supported by Micron DDR3 devices +#else + const alt_u32 NUM_REFRESH_POSTING = 8; +#endif + alt_u32 i; + alt_u32 elapsed_time; // In AVL clock cycles + +#if (ENABLE_NON_DESTRUCTIVE_CALIB) + //USER Reset the refresh interval timer + elapsed_time = IORD_32DIRECT (BASE_TIMER, 0); + IOWR_32DIRECT (BASE_TIMER, 0, 0x00); + + //USER Validate that maximum refresh interval is not exceeded + if ( !no_validate ) { + if (!(~elapsed_time) || elapsed_time > (NUM_REFRESH_POSTING * T_REFI_NS * AVL_CLK_FREQ / 1000) ) { + // Non-destructive calibration failure + return 0; + } + } + +#endif + //USER set CS and ODT mask + if ( RDIMM || LRDIMM ) { + if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) { + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); + } + else { + // Only single-rank DIMM supported for non-destructive cal + return 0; + } + } + else { // UDIMM + // Issue refreshes to all ranks simultaneously + IOWR_32DIRECT (RW_MGR_SET_CS_AND_ODT_MASK, 0, RW_MGR_RANK_ALL); + } + + //USER Precharge all banks + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_PRECHARGE_ALL); + // Wait for tRP = 15ns before issuing REFRESH commands + // No need to insert explicit delay; simulation shows more than 1000 ns between PRECHARGE and first REFRESH + + //USER Issue refreshes + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_REFRESH_ALL); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_REFRESH_DELAY); + for (i = 0; i < NUM_REFRESH_POSTING; i += 256) { + // Issue 256 REFRESH commands, waiting t_RFC between consecutive refreshes + +#if (ENABLE_NON_DESTRUCTIVE_CALIB) + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0xFF); +#else + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x07); +#endif + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, T_RFC_AFI); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_REFRESH_ALL); + } + + //USER Re-activate all banks + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x00); // No need to wait between commands to activate different banks (since ACTIVATE is preceded by tRFC wait) + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x0F); // Wait for ACTIVATE to complete + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_ACTIVATE_0_AND_1_WAIT1); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_ACTIVATE_0_AND_1_WAIT2); + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_ACTIVATE_0_AND_1); + + return 1; +} +#else +alt_u32 mem_refresh_all_ranks (alt_u32 no_validate) +{ + return 1; +} +#endif +#endif + +//USER Configure various memory related parameters. + +#if DDRX +void mem_config (void) +{ + alt_u32 rlat, wlat; + alt_u32 rw_wl_nop_cycles; + alt_u32 max_latency; +#if CALIBRATE_BIT_SLIPS + alt_u32 i; +#endif + + TRACE_FUNC(); + + //USER read in write and read latency + + wlat = IORD_32DIRECT (MEM_T_WL_ADD, 0); +#if HARD_PHY + wlat += IORD_32DIRECT (DATA_MGR_MEM_T_ADD, 0); /* WL for hard phy does not include additive latency */ + + #if DDR3 || DDR2 + // YYONG: add addtional write latency to offset the address/command extra clock cycle + // YYONG: We change the AC mux setting causing AC to be delayed by one mem clock cycle + // YYONG: only do this for DDR3 + wlat = wlat + 1; + #endif +#endif + + rlat = IORD_32DIRECT (MEM_T_RL_ADD, 0); + + if (QUARTER_RATE_MODE) { + //USER In Quarter-Rate the WL-to-nop-cycles works like this + //USER 0,1 -> 0 + //USER 2,3,4,5 -> 1 + //USER 6,7,8,9 -> 2 + //USER etc... + rw_wl_nop_cycles = (wlat + 6) / 4 - 1; + } + else if(HALF_RATE_MODE) { + //USER In Half-Rate the WL-to-nop-cycles works like this + //USER 0,1 -> -1 + //USER 2,3 -> 0 + //USER 4,5 -> 1 + //USER etc... + if(wlat % 2) + { + rw_wl_nop_cycles = ((wlat - 1) / 2) - 1; + } + else + { + rw_wl_nop_cycles = (wlat / 2) - 1; + } + } + else { + rw_wl_nop_cycles = wlat - 2; +#if LPDDR2 + rw_wl_nop_cycles = rw_wl_nop_cycles + 1; +#endif + } +#if MULTIPLE_AFI_WLAT + for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + gbl->rw_wl_nop_cycles_per_group[i] = rw_wl_nop_cycles; + } +#endif + gbl->rw_wl_nop_cycles = rw_wl_nop_cycles; + +#if ARRIAV || CYCLONEV + //USER For AV/CV, lfifo is hardened and always runs at full rate + //USER so max latency in AFI clocks, used here, is correspondingly smaller + if (QUARTER_RATE_MODE) { + max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/4 - 1; + } else if (HALF_RATE_MODE) { + max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/2 - 1; + } else { + max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1; + } +#else + max_latency = (1<<MAX_LATENCY_COUNT_WIDTH) - 1; +#endif + //USER configure for a burst length of 8 + + if (QUARTER_RATE_MODE) { + //USER write latency + wlat = (wlat + 5) / 4 + 1; + + //USER set a pretty high read latency initially + gbl->curr_read_lat = (rlat + 1) / 4 + 8; + } else if (HALF_RATE_MODE) { + //USER write latency + wlat = (wlat - 1) / 2 + 1; + + //USER set a pretty high read latency initially + gbl->curr_read_lat = (rlat + 1) / 2 + 8; + } else { + //USER write latency +#if HARD_PHY + // Adjust Write Latency for Hard PHY + wlat = wlat + 1; +#if LPDDR2 + // Add another one in hard for LPDDR2 since this value is raw from controller + // assume tdqss is one + wlat = wlat + 1; +#endif +#endif + + //USER set a pretty high read latency initially + gbl->curr_read_lat = rlat + 16; + } + + if (gbl->curr_read_lat > max_latency) { + gbl->curr_read_lat = max_latency; + } + IOWR_32DIRECT (PHY_MGR_PHY_RLAT, 0, gbl->curr_read_lat); + + //USER advertise write latency + gbl->curr_write_lat = wlat; +#if MULTIPLE_AFI_WLAT + for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { +#if HARD_PHY + IOWR_32DIRECT (PHY_MGR_AFI_WLAT, i*4, wlat - 2); +#else + IOWR_32DIRECT (PHY_MGR_AFI_WLAT, i*4, wlat - 1); +#endif + } +#else +#if HARD_PHY + IOWR_32DIRECT (PHY_MGR_AFI_WLAT, 0, wlat - 2); +#else + IOWR_32DIRECT (PHY_MGR_AFI_WLAT, 0, wlat - 1); +#endif +#endif + + //USER initialize bit slips +#if CALIBRATE_BIT_SLIPS + for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + IOWR_32DIRECT (PHY_MGR_FR_SHIFT, i*4, 0); + } +#endif + + + mem_precharge_and_activate (); +} +#endif + +#if QDRII || RLDRAMX +void mem_config (void) +{ + alt_u32 wlat, nop_cycles, max_latency; + + TRACE_FUNC(); + + max_latency = (1<<MAX_LATENCY_COUNT_WIDTH) - 1; + + if (QUARTER_RATE_MODE) { + // TODO_JCHOI: verify confirm + gbl->curr_read_lat = (IORD_32DIRECT (MEM_T_RL_ADD, 0) + 1) / 4 + 8; + } else if (HALF_RATE_MODE) { + gbl->curr_read_lat = (IORD_32DIRECT (MEM_T_RL_ADD, 0) + 1) / 2 + 8; + } else { + gbl->curr_read_lat = IORD_32DIRECT (MEM_T_RL_ADD, 0) + 16; + } + if (gbl->curr_read_lat > max_latency) { + gbl->curr_read_lat = max_latency; + } + IOWR_32DIRECT (PHY_MGR_PHY_RLAT, 0, gbl->curr_read_lat); + + if (RLDRAMX) + { + //USER read in write and read latency + wlat = IORD_32DIRECT (MEM_T_WL_ADD, 0); + + if (QUARTER_RATE_MODE) + { + // TODO_JCHOI Verify + nop_cycles = ((wlat - 1) / 4) - 1; + } + else if (HALF_RATE_MODE) + { +#if HR_DDIO_OUT_HAS_THREE_REGS + nop_cycles = (wlat / 2) - 2; +#else + #if RLDRAM3 + // RLDRAM3 uses all AFI phases to issue commands + nop_cycles = (wlat / 2) - 2; + #else + nop_cycles = ((wlat + 1) / 2) - 2; + #endif +#endif + } + else + { + nop_cycles = wlat - 1; + } + gbl->rw_wl_nop_cycles = nop_cycles; + } +} +#endif + +//USER Set VFIFO and LFIFO to instant-on settings in skip calibration mode + +void mem_skip_calibrate (void) +{ + alt_u32 vfifo_offset; + alt_u32 i, j, r; +#if HCX_COMPAT_MODE && DDR3 + alt_u32 v; +#if (RDIMM || LRDIMM) + alt_u32 increment = 2; +#else + alt_u32 wlat = IORD_32DIRECT (PHY_MGR_MEM_T_WL, 0); + alt_u32 rlat = IORD_32DIRECT (PHY_MGR_MEM_T_RL, 0); + alt_u32 increment = rlat - wlat*2 + 1; +#endif +#endif + + TRACE_FUNC(); + + // Need to update every shadow register set used by the interface + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r += NUM_RANKS_PER_SHADOW_REG) { + + // Strictly speaking this should be called once per group to make + // sure each group's delay chains are refreshed from the SCC register file, + // but since we're resetting all delay chains anyway, we can save some + // runtime by calling select_shadow_regs_for_update just once to switch rank. + select_shadow_regs_for_update(r, 0, 1); + + //USER Set output phase alignment settings appropriate for skip calibration + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + +#if STRATIXV || ARRIAV || CYCLONEV || ARRIAVGZ + scc_mgr_set_dqs_en_phase(i, 0); +#else +#if IO_DLL_CHAIN_LENGTH == 6 + scc_mgr_set_dqs_en_phase(i, (IO_DLL_CHAIN_LENGTH >> 1) - 1); +#else + scc_mgr_set_dqs_en_phase(i, (IO_DLL_CHAIN_LENGTH >> 1)); +#endif +#endif +#if HCX_COMPAT_MODE && DDR3 + v = 0; + for (j = 0; j < increment; j++) { + rw_mgr_incr_vfifo(i, &v); + } + +#if IO_DLL_CHAIN_LENGTH == 6 + scc_mgr_set_dqdqs_output_phase(i, 6); +#else + scc_mgr_set_dqdqs_output_phase(i, 7); +#endif +#else + #if HCX_COMPAT_MODE + // in this mode, write_clk doesn't always lead mem_ck by 90 deg, and so + // the enhancement in case:33398 can't be applied. + scc_mgr_set_dqdqs_output_phase(i, (IO_DLL_CHAIN_LENGTH - IO_DLL_CHAIN_LENGTH / 3)); + #else + // Case:33398 + // + // Write data arrives to the I/O two cycles before write latency is reached (720 deg). + // -> due to bit-slip in a/c bus + // -> to allow board skew where dqs is longer than ck + // -> how often can this happen!? + // -> can claim back some ptaps for high freq support if we can relax this, but i digress... + // + // The write_clk leads mem_ck by 90 deg + // The minimum ptap of the OPA is 180 deg + // Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay + // The write_clk is always delayed by 2 ptaps + // + // Hence, to make DQS aligned to CK, we need to delay DQS by: + // (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH)) + // + // Dividing the above by (360 / IO_DLL_CHAIN_LENGTH) gives us the number of ptaps, which simplies to: + // + // (1.25 * IO_DLL_CHAIN_LENGTH - 2) + scc_mgr_set_dqdqs_output_phase(i, (1.25 * IO_DLL_CHAIN_LENGTH - 2)); + #endif +#endif + } + + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, 0xff); + IOWR_32DIRECT (SCC_MGR_DQS_IO_ENA, 0, 0xff); + + for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) { + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, i); + IOWR_32DIRECT (SCC_MGR_DQ_ENA, 0, 0xff); + IOWR_32DIRECT (SCC_MGR_DM_ENA, 0, 0xff); + } + +#if USE_SHADOW_REGS + //USER in shadow-register mode, SCC_UPDATE is done on a per-group basis + //USER unless we explicitly ask for a multicast via the group counter + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, 0xFF); +#endif + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + } + +#if ARRIAV || CYCLONEV + // Compensate for simulation model behaviour + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + scc_mgr_set_dqs_bus_in_delay(i, 10); + scc_mgr_load_dqs (i); + } + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); +#endif + +#if ARRIAV || CYCLONEV + //ArriaV has hard FIFOs that can only be initialized by incrementing in sequencer + vfifo_offset = CALIB_VFIFO_OFFSET; + for (j = 0; j < vfifo_offset; j++) { + if(HARD_PHY) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_HARD_PHY, 0, 0xff); + } else { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR, 0, 0xff); + } + } +#else +// Note, this is not currently supported; changing this might significantly +// increase the size of the ROM +#if SUPPORT_DYNAMIC_SKIP_CALIBRATE_ACTIONS + if ((DYNAMIC_CALIB_STEPS) & CALIB_IN_RTL_SIM) { + //USER VFIFO is reset to the correct settings in RTL simulation + } else { + vfifo_offset = IORD_32DIRECT (PHY_MGR_CALIB_VFIFO_OFFSET, 0); + + if (QUARTER_RATE_MODE) { + while (vfifo_offset > 3) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_QR, 0, 0xff); + vfifo_offset -= 4; + } + + if (vfifo_offset == 3) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR_HR, 0, 0xff); + } else if (vfifo_offset == 2) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_HR, 0, 0xff); + } else if (vfifo_offset == 1) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR, 0, 0xff); + } + } else { + while (vfifo_offset > 1) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_HR, 0, 0xff); + vfifo_offset -= 2; + } + + if (vfifo_offset == 1) { + IOWR_32DIRECT (PHY_MGR_CMD_INC_VFIFO_FR, 0, 0xff); + } + } + } +#endif +#endif + + + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + +#if ARRIAV || CYCLONEV + // For ACV with hard lfifo, we get the skip-cal setting from generation-time constant + gbl->curr_read_lat = CALIB_LFIFO_OFFSET; +#else + gbl->curr_read_lat = IORD_32DIRECT (PHY_MGR_CALIB_LFIFO_OFFSET, 0); +#endif + IOWR_32DIRECT (PHY_MGR_PHY_RLAT, 0, gbl->curr_read_lat); +} + + +#if BFM_MODE +void print_group_settings(alt_u32 group, alt_u32 dq_begin) +{ + int i; + + fprintf(bfm_gbl.outfp, "Group %lu (offset %lu)\n", group, dq_begin); + + fprintf(bfm_gbl.outfp, "Output:\n"); + fprintf(bfm_gbl.outfp, "dqdqs_out_phase: %2u\n", READ_SCC_DQDQS_OUT_PHASE(group)); + fprintf(bfm_gbl.outfp, "dqs_out1_delay: %2u\n", READ_SCC_DQS_IO_OUT1_DELAY()); + fprintf(bfm_gbl.outfp, "dqs_out2_delay: %2u\n", READ_SCC_DQS_IO_OUT2_DELAY()); + fprintf(bfm_gbl.outfp, "oct_out1_delay: %2u\n", READ_SCC_OCT_OUT1_DELAY(group)); + fprintf(bfm_gbl.outfp, "oct_out2_delay: %2u\n", READ_SCC_OCT_OUT2_DELAY(group)); + fprintf(bfm_gbl.outfp, "dm_out1: "); + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { + fprintf(bfm_gbl.outfp, "%2u ", READ_SCC_DM_IO_OUT1_DELAY(i)); + } + fprintf(bfm_gbl.outfp, "\n"); + fprintf(bfm_gbl.outfp, "dm_out2: "); + for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) { + fprintf(bfm_gbl.outfp, "%2u ", READ_SCC_DM_IO_OUT2_DELAY(i)); + } + fprintf(bfm_gbl.outfp, "\n"); + fprintf(bfm_gbl.outfp, "dq_out1: "); + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + fprintf(bfm_gbl.outfp, "%2u ", READ_SCC_DQ_OUT1_DELAY(i)); + } + fprintf(bfm_gbl.outfp, "\n"); + fprintf(bfm_gbl.outfp, "dq_out2: "); + for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) { + fprintf(bfm_gbl.outfp, "%2u ", READ_SCC_DQ_OUT2_DELAY(i)); + } + fprintf(bfm_gbl.outfp, "\n"); + + fprintf(bfm_gbl.outfp, "Input:\n"); + fprintf(bfm_gbl.outfp, "dqs_en_phase: %2u\n", READ_SCC_DQS_EN_PHASE(group)); + fprintf(bfm_gbl.outfp, "dqs_en_delay: %2u\n", READ_SCC_DQS_EN_DELAY(group)); + fprintf(bfm_gbl.outfp, "dqs_in_delay: %2u\n", READ_SCC_DQS_IN_DELAY(group)); + fprintf(bfm_gbl.outfp, "dq_in: "); + for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) { + fprintf(bfm_gbl.outfp, "%2u ", READ_SCC_DQ_IN_DELAY(i)); + } + fprintf(bfm_gbl.outfp, "\n"); + + fprintf(bfm_gbl.outfp, "\n"); + + fflush(bfm_gbl.outfp); +} + +#endif + +#if RUNTIME_CAL_REPORT +void print_report(alt_u32 pass) +{ + RPRINT("Calibration Summary"); + char *stage_name, *substage_name; + + if(pass) { + RPRINT("Calibration Passed"); + RPRINT("FOM IN = %lu", gbl->fom_in); + RPRINT("FOM OUT = %lu", gbl->fom_out); + } else { + RPRINT("Calibration Failed"); + switch (gbl->error_stage) { + case CAL_STAGE_NIL: + stage_name = "NIL"; + substage_name = "NIL"; + case CAL_STAGE_VFIFO: + stage_name = "VFIFO"; + switch (gbl->error_substage) { + case CAL_SUBSTAGE_GUARANTEED_READ: + substage_name = "GUARANTEED READ"; + break; + case CAL_SUBSTAGE_DQS_EN_PHASE: + substage_name = "DQS ENABLE PHASE"; + break; + case CAL_SUBSTAGE_VFIFO_CENTER: + substage_name = "Read Per-Bit Deskew"; + break; + default: + substage_name = "NIL"; + } + break; + case CAL_STAGE_WLEVEL: + stage_name = "WRITE LEVELING"; + switch (gbl->error_substage) { + case CAL_SUBSTAGE_WORKING_DELAY: + substage_name = "DQS Window Left Edge"; //need a more descriptive name + break; + case CAL_SUBSTAGE_LAST_WORKING_DELAY: + substage_name = "DQS Window Right Edge"; + break; + case CAL_SUBSTAGE_WLEVEL_COPY: + substage_name = "WRITE LEVEL COPY"; + break; + default: + substage_name = "NIL"; + } + break; + case CAL_STAGE_LFIFO: + stage_name = "LFIFO"; + substage_name = "READ LATENCY"; + break; + case CAL_STAGE_WRITES: + stage_name = "WRITES"; + substage_name = "Write Per-Bit Deskew"; + break; + case CAL_STAGE_FULLTEST: + stage_name = "FULL TEST"; + substage_name = "FULL TEST"; + break; + case CAL_STAGE_REFRESH: + stage_name = "REFRESH"; + substage_name = "REFRESH"; + break; + case CAL_STAGE_CAL_SKIPPED: + stage_name = "SKIP CALIBRATION"; //hw: is this needed + substage_name = "SKIP CALIBRATION"; + break; + case CAL_STAGE_CAL_ABORTED: + stage_name = "ABORTED CALIBRATION"; //hw: hum??? + substage_name = "ABORTED CALIBRATION"; + break; + case CAL_STAGE_VFIFO_AFTER_WRITES: + stage_name = "READ Fine-tuning"; + switch (gbl->error_substage) { + case CAL_SUBSTAGE_GUARANTEED_READ: + substage_name = "GUARANTEED READ"; + break; + case CAL_SUBSTAGE_DQS_EN_PHASE: + substage_name = "DQS ENABLE PHASE"; + break; + case CAL_SUBSTAGE_VFIFO_CENTER: + substage_name = "VFIFO CENTER"; + break; + default: + substage_name = "NIL"; + } + break; + default: + stage_name = "NIL"; + substage_name = "NIL"; + } + RPRINT("Error Stage : %lu - %s", gbl->error_stage, stage_name); + RPRINT("Error Substage: %lu - %s", gbl->error_substage, substage_name); + RPRINT("Error Group : %lu", gbl->error_group); + } +} +#endif //RUNTIME_CAL_REPORT + +//USER Memory calibration entry point + +alt_u32 mem_calibrate (void) +{ + alt_u32 i; + alt_u32 rank_bgn, sr; + alt_u32 write_group, write_test_bgn; + alt_u32 read_group, read_test_bgn; + alt_u32 run_groups, current_run; + alt_u32 failing_groups = 0; + alt_u32 group_failed = 0; + alt_u32 sr_failed = 0; + + TRACE_FUNC(); + + // Initialize the data settings + DPRINT(1, "Preparing to init data"); +#if ENABLE_TCL_DEBUG + tclrpt_initialize_data(); +#endif + DPRINT(1, "Init complete"); + + gbl->error_substage = CAL_SUBSTAGE_NIL; + gbl->error_stage = CAL_STAGE_NIL; + gbl->error_group = 0xff; + gbl->fom_in = 0; + gbl->fom_out = 0; + + TCLRPT_SET(debug_summary_report->cal_read_latency, 0); + TCLRPT_SET(debug_summary_report->cal_write_latency, 0); + + mem_config (); + + if(ARRIAV || CYCLONEV) { + alt_u32 bypass_mode = (HARD_PHY) ? 0x1 : 0x0; + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, i); + scc_set_bypass_mode (i, bypass_mode); + } + } + + if (((DYNAMIC_CALIB_STEPS) & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) { + //USER Set VFIFO and LFIFO to instant-on settings in skip calibration mode + + mem_skip_calibrate (); + } else { + for (i = 0; i < NUM_CALIB_REPEAT; i++) { + + //USER Zero all delay chain/phase settings for all groups and all shadow register sets + scc_mgr_zero_all (); + +#if ENABLE_SUPER_QUICK_CALIBRATION + for (write_group = 0, write_test_bgn = 0; write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++, write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) + { + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, write_group); + scc_mgr_zero_group (write_group, write_test_bgn, 0); + } +#endif + + run_groups = ~param->skip_groups; + + for (write_group = 0, write_test_bgn = 0; write_group < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++, write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) + { + // Initialized the group failure + group_failed = 0; + + // Mark the group as being attempted for calibration +#if ENABLE_TCL_DEBUG + tclrpt_set_group_as_calibration_attempted(write_group); +#endif + +#if RLDRAMX || QDRII + //Note: + // It seems that with rldram and qdr vfifo starts at max (not sure for ddr) + // also not sure if max is really vfifo_size-1 or vfifo_size + BFM_GBL_SET(vfifo_idx,VFIFO_SIZE-1); +#else + BFM_GBL_SET(vfifo_idx,0); +#endif + current_run = run_groups & ((1 << RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1); + run_groups = run_groups >> RW_MGR_NUM_DQS_PER_WRITE_GROUP; + + if (current_run == 0) + { + continue; + } + + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, write_group); +#if !ENABLE_SUPER_QUICK_CALIBRATION + scc_mgr_zero_group (write_group, write_test_bgn, 0); +#endif + + for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH, read_test_bgn = 0; + read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH && group_failed == 0; + read_group++, read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) { + + //USER Calibrate the VFIFO + if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_VFIFO)) { + if (!rw_mgr_mem_calibrate_vfifo (read_group, read_test_bgn)) { + group_failed = 1; + + if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS)) { + return 0; + } + } + } + } + + //USER level writes (or align DK with CK for RLDRAMX) + if (group_failed == 0) + { + if ((DDRX || RLDRAMII) && !(ARRIAV || CYCLONEV)) + { + if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_WLEVEL)) { + if (!rw_mgr_mem_calibrate_wlevel (write_group, write_test_bgn)) { + group_failed = 1; + + if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS)) { + return 0; + } + } + } + } + } + + //USER Calibrate the output side + if (group_failed == 0) + { + for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { + sr_failed = 0; + if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES)) { + if ((STATIC_CALIB_STEPS) & CALIB_SKIP_DELAY_SWEEPS) { + //USER not needed in quick mode! + } else { + //USER Determine if this set of ranks should be skipped entirely + if (! param->skip_shadow_regs[sr]) { + + //USER Select shadow register set + select_shadow_regs_for_update(rank_bgn, write_group, 1); + + if (!rw_mgr_mem_calibrate_writes (rank_bgn, write_group, write_test_bgn)) { + sr_failed = 1; + if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS)) { + return 0; + } + } + } + } + } + if(sr_failed == 0) { + TCLRPT_SET(debug_cal_report->cal_status_per_group[sr][write_group].error_stage, CAL_STAGE_NIL); + } else { + group_failed = 1; + } + } + } + +#if READ_AFTER_WRITE_CALIBRATION + if (group_failed == 0) + { + for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH, read_test_bgn = 0; + read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH / RW_MGR_MEM_IF_WRITE_DQS_WIDTH && group_failed == 0; + read_group++, read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) { + + if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES)) { + if (!rw_mgr_mem_calibrate_vfifo_end (read_group, read_test_bgn)) { + group_failed = 1; + + if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS)) { + return 0; + } + } + } + } + } +#endif + + if (group_failed == 0) + { + +#if BFM_MODE + + // TODO: should just update global BFM structure with all data + // and print all out at the end + print_group_settings(write_group, write_test_bgn); +#endif + +#if STATIC_IN_RTL_SIM +#if ENABLE_TCL_DEBUG && BFM_MODE + tclrpt_populate_fake_margin_data(); +#endif +#else +#if ENABLE_TCL_DEBUG + if (gbl->phy_debug_mode_flags & PHY_DEBUG_ENABLE_MARGIN_RPT) + { + // Run margining + for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS; rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) { + + //USER Determine if this set of ranks should be skipped entirely + if (! param->skip_shadow_regs[sr]) { + + //USER Select shadow register set + select_shadow_regs_for_update(rank_bgn, write_group, 1); + + run_dq_margining(rank_bgn, write_group); +#if DDRX + if (RW_MGR_NUM_TRUE_DM_PER_WRITE_GROUP > 0) + { + run_dm_margining(rank_bgn, write_group); + } +#endif +#if QDRII + run_dm_margining(rank_bgn, write_group); +#endif +#if RLDRAMX + if (is_write_group_enabled_for_dm(write_group)) + { + run_dm_margining(rank_bgn, write_group); + } +#endif + } + } + } +#endif +#endif + } + + if (group_failed != 0) + { + failing_groups++; + } + +#if ENABLE_NON_DESTRUCTIVE_CALIB + if (gbl->phy_debug_mode_flags & PHY_DEBUG_ENABLE_NON_DESTRUCTIVE_CALIBRATION) { + // USER Refresh the memory + if (!mem_refresh_all_ranks(0)) { + set_failing_group_stage(write_group, CAL_STAGE_REFRESH, CAL_SUBSTAGE_REFRESH); + TCLRPT_SET(debug_cal_report->cal_status_per_group[curr_shadow_reg][write_group].error_stage, CAL_STAGE_REFRESH); + return 0; + } + } +#endif + +#if ENABLE_NON_DESTRUCTIVE_CALIB + // USER Check if synchronous abort has been asserted + if (abort_cal) { + set_failing_group_stage(write_group, CAL_STAGE_CAL_ABORTED, CAL_SUBSTAGE_NIL); + return 0; + } +#endif + } + + // USER If there are any failing groups then report the failure + if (failing_groups != 0) + { + return 0; + } + + //USER Calibrate the LFIFO + if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) { + //USER If we're skipping groups as part of debug, don't calibrate LFIFO + if (param->skip_groups == 0) + { + if (!rw_mgr_mem_calibrate_lfifo ()) { + return 0; + } + } + } + } + } + + TCLRPT_SET(debug_summary_report->cal_write_latency, IORD_32DIRECT (MEM_T_WL_ADD, 0)); + if (QUARTER_RATE == 1) { + // The read latency is in terms of AFI cycles so we multiply by 4 in quarter + // rate to get the memory cycles. + TCLRPT_SET(debug_summary_report->cal_read_latency, gbl->curr_read_lat * 4); + } + else if (HALF_RATE == 1) { + // The read latency is in terms of AFI cycles so we multiply by 2 in half + // rate to get the memory cycles. + TCLRPT_SET(debug_summary_report->cal_read_latency, gbl->curr_read_lat * 2); + } + else { + TCLRPT_SET(debug_summary_report->cal_read_latency, gbl->curr_read_lat); + } + + + //USER Do not remove this line as it makes sure all of our decisions have been applied + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + return 1; +} +#if ENABLE_NON_DES_CAL +alt_u32 run_mem_calibrate(alt_u32 non_des_mode) { + +#else +alt_u32 run_mem_calibrate(void) { +#endif + + alt_u32 pass; + alt_u32 debug_info; + + // Initialize the debug status to show that calibration has started. + // This should occur before anything else +#if ENABLE_TCL_DEBUG + tclrpt_initialize_debug_status(); + + // Set that calibration has started + debug_data->status |= 1 << DEBUG_STATUS_CALIBRATION_STARTED; +#endif + // Reset pass/fail status shown on afi_cal_success/fail + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, PHY_MGR_CAL_RESET); + + TRACE_FUNC(); + + BFM_STAGE("calibrate"); +#if USE_DQS_TRACKING +#if HHP_HPS + //stop tracking manger + alt_u32 ctrlcfg = IORD_32DIRECT(CTRL_CONFIG_REG,0); + + IOWR_32DIRECT(CTRL_CONFIG_REG, 0, ctrlcfg & 0xFFBFFFFF); +#else + // we need to stall tracking + IOWR_32DIRECT (TRK_STALL, 0, TRK_STALL_REQ_VAL); + // busy wait for tracking manager to ack stall request + while (IORD_32DIRECT (TRK_STALL, 0) != TRK_STALL_ACKED_VAL) { + } +#endif +#endif + + initialize(); + +#if ENABLE_NON_DESTRUCTIVE_CALIB + if (gbl->phy_debug_mode_flags & PHY_DEBUG_ENABLE_NON_DESTRUCTIVE_CALIBRATION) { + if (no_init) { + rw_mgr_mem_initialize_no_init(); + // refresh is done as part of rw_mgr_mem_initialize_no_init() + } else { + rw_mgr_mem_initialize (); + mem_refresh_all_ranks(1); + } + } else { + rw_mgr_mem_initialize (); + } +#else +#if ENABLE_NON_DES_CAL + if (non_des_mode) + rw_mgr_mem_initialize_no_init(); + else + rw_mgr_mem_initialize (); + +#else + rw_mgr_mem_initialize (); +#endif +#endif + + +#if ENABLE_BRINGUP_DEBUGGING + do_bringup_test(); +#endif + + pass = mem_calibrate (); + +#if ENABLE_NON_DESTRUCTIVE_CALIB + if( (gbl->phy_debug_mode_flags & PHY_DEBUG_ENABLE_NON_DESTRUCTIVE_CALIBRATION) ) { + if (!mem_refresh_all_ranks(0)) { + set_failing_group_stage(RW_MGR_MEM_IF_WRITE_DQS_WIDTH, CAL_STAGE_REFRESH, CAL_SUBSTAGE_REFRESH); + pass = 0; + } + } else { + mem_precharge_and_activate (); + } +#else + mem_precharge_and_activate (); +#endif + + //pe_checkout_pattern(); + + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + + if (pass) { + TCLRPT_SET(debug_summary_report->error_stage, CAL_STAGE_NIL); + + + BFM_STAGE("handoff"); + +#ifdef TEST_SIZE + if (!check_test_mem(0)) { + gbl->error_stage = 0x92; + gbl->error_group = 0x92; + } +#endif + } + +#if TRACKING_ERROR_TEST + if (IORD_32DIRECT(REG_FILE_TRK_SAMPLE_CHECK, 0) == 0xFE) { + poll_for_sample_check(); + } +#endif + + //USER Handoff +#if ENABLE_NON_DES_CAL + + if (non_des_mode) + { + alt_u32 took_too_long = 0; + IOWR_32DIRECT (RW_MGR_ENABLE_REFRESH, 0, 0); // Disable refresh engine + took_too_long = IORD_32DIRECT (RW_MGR_ENABLE_REFRESH, 0); + + if (took_too_long != 0) + { + pass = 0; // force a failure + set_failing_group_stage(RW_MGR_MEM_IF_WRITE_DQS_WIDTH, CAL_STAGE_REFRESH, CAL_SUBSTAGE_REFRESH); + } + } +#endif + + + //USER Don't return control of the PHY back to AFI when in debug mode + if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) { + rw_mgr_mem_handoff (); + +#if HARD_PHY + // In Hard PHY this is a 2-bit control: + // 0: AFI Mux Select + // 1: DDIO Mux Select + IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 0x2); +#else + IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 0); +#endif + } +#if USE_DQS_TRACKING +#if HHP_HPS + IOWR_32DIRECT(CTRL_CONFIG_REG, 0, ctrlcfg); +#else + // clear tracking stall flags + IOWR_32DIRECT (TRK_STALL, 0, 0); +#endif +#endif + +#if FAKE_CAL_FAIL + if (0) { +#else + if (pass) { +#endif + IPRINT("CALIBRATION PASSED"); + + gbl->fom_in /= 2; + gbl->fom_out /= 2; + + if (gbl->fom_in > 0xff) { + gbl->fom_in = 0xff; + } + + if (gbl->fom_out > 0xff) { + gbl->fom_out = 0xff; + } + +#if BFM_MODE + // duplicated because we want it after updating gbl, but before phy + // is informed that calibration has completed + print_gbl(); + fini_outfile(); +#endif + + // Update the FOM in the register file + debug_info = gbl->fom_in; + debug_info |= gbl->fom_out << 8; + IOWR_32DIRECT (REG_FILE_FOM, 0, debug_info); + + IOWR_32DIRECT (PHY_MGR_CAL_DEBUG_INFO, 0, debug_info); + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, PHY_MGR_CAL_SUCCESS); + + } else { + + IPRINT("CALIBRATION FAILED"); + + debug_info = gbl->error_stage; + debug_info |= gbl->error_substage << 8; + debug_info |= gbl->error_group << 16; + +#if BFM_MODE + // duplicated because we want it after updating gbl, but before phy + // is informed that calibration has completed + print_gbl(); + fini_outfile(); +#endif + + IOWR_32DIRECT (REG_FILE_FAILING_STAGE, 0, debug_info); + IOWR_32DIRECT (PHY_MGR_CAL_DEBUG_INFO, 0, debug_info); + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, PHY_MGR_CAL_FAIL); + + // Update the failing group/stage in the register file + debug_info = gbl->error_stage; + debug_info |= gbl->error_substage << 8; + debug_info |= gbl->error_group << 16; + IOWR_32DIRECT (REG_FILE_FAILING_STAGE, 0, debug_info); + + } + +#if RUNTIME_CAL_REPORT + print_report(pass); +#endif + + + // Mark the reports as being ready to read + TCLRPT_SET(debug_summary_report->report_flags, debug_summary_report->report_flags |= DEBUG_REPORT_STATUS_REPORT_READY); + TCLRPT_SET(debug_cal_report->report_flags, debug_cal_report->report_flags |= DEBUG_REPORT_STATUS_REPORT_READY); + TCLRPT_SET(debug_margin_report->report_flags, debug_margin_report->report_flags |= DEBUG_REPORT_STATUS_REPORT_READY); + + // Set the debug status to show that calibration has ended. + // This should occur after everything else +#if ENABLE_TCL_DEBUG + debug_data->status |= 1 << DEBUG_STATUS_CALIBRATION_ENDED; +#endif + return pass; + +} + +#if HCX_COMPAT_MODE || ENABLE_INST_ROM_WRITE +void hc_initialize_rom_data(void) +{ + alt_u32 i; + + for(i = 0; i < inst_rom_init_size; i++) + { + alt_u32 data = inst_rom_init[i]; + IOWR_32DIRECT (RW_MGR_INST_ROM_WRITE, (i << 2), data); + } + + for(i = 0; i < ac_rom_init_size; i++) + { + alt_u32 data = ac_rom_init[i]; + IOWR_32DIRECT (RW_MGR_AC_ROM_WRITE, (i << 2), data); + } +} +#endif + +#if BFM_MODE +void init_outfile(void) +{ + const char *filename = getenv("SEQ_OUT_FILE"); + + if (filename == NULL) { + filename = "sequencer.out"; + } + + if ((bfm_gbl.outfp = fopen(filename, "w")) == NULL) { + printf("ERROR: Failed to open %s for writing; using stdout\n", filename); + bfm_gbl.outfp = stdout; + } + + fprintf(bfm_gbl.outfp, "%s%s %s ranks=%lu cs/dimm=%lu dq/dqs=%lu,%lu vg/dqs=%lu,%lu dqs=%lu,%lu dq=%lu dm=%lu " + "ptap_delay=%lu dtap_delay=%lu dtap_dqsen_delay=%lu dll=%lu\n", + RDIMM ? "r" : (LRDIMM ? "l" : ""), + DDR2 ? "DDR2" : (DDR3 ? "DDR3" : (QDRII ? "QDRII" : (RLDRAMII ? "RLDRAMII" : (RLDRAM3 ? "RLDRAM3" : "??PROTO??")))), + FULL_RATE ? "FR" : (HALF_RATE ? "HR" : (QUARTER_RATE ? "QR" : "??RATE??")), + RW_MGR_MEM_NUMBER_OF_RANKS, + RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM, + RW_MGR_MEM_DQ_PER_READ_DQS, + RW_MGR_MEM_DQ_PER_WRITE_DQS, + RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS, + RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS, + RW_MGR_MEM_IF_READ_DQS_WIDTH, + RW_MGR_MEM_IF_WRITE_DQS_WIDTH, + RW_MGR_MEM_DATA_WIDTH, + RW_MGR_MEM_DATA_MASK_WIDTH, + IO_DELAY_PER_OPA_TAP, + IO_DELAY_PER_DCHAIN_TAP, + IO_DELAY_PER_DQS_EN_DCHAIN_TAP, + IO_DLL_CHAIN_LENGTH); + fprintf(bfm_gbl.outfp, "max values: en_p=%lu dqdqs_p=%lu en_d=%lu dqs_in_d=%lu io_in_d=%lu io_out1_d=%lu io_out2_d=%lu " + "dqs_in_reserve=%lu dqs_out_reserve=%lu\n", + IO_DQS_EN_PHASE_MAX, + IO_DQDQS_OUT_PHASE_MAX, + IO_DQS_EN_DELAY_MAX, + IO_DQS_IN_DELAY_MAX, + IO_IO_IN_DELAY_MAX, + IO_IO_OUT1_DELAY_MAX, + IO_IO_OUT2_DELAY_MAX, + IO_DQS_IN_RESERVE, + IO_DQS_OUT_RESERVE); + fprintf(bfm_gbl.outfp, "\n"); + // repeat these in a format that can be easily parsed + fprintf(bfm_gbl.outfp, "ptap_delay: %lu\n", IO_DELAY_PER_OPA_TAP); + fprintf(bfm_gbl.outfp, "dtap_delay: %lu\n", IO_DELAY_PER_DCHAIN_TAP); + fprintf(bfm_gbl.outfp, "ptap_per_cycle: %lu\n", IO_DLL_CHAIN_LENGTH); + fprintf(bfm_gbl.outfp, "ptap_max: %lu\n", IO_DQDQS_OUT_PHASE_MAX); + fprintf(bfm_gbl.outfp, "dtap_max: %lu\n", IO_IO_OUT1_DELAY_MAX); + fprintf(bfm_gbl.outfp, "vfifo_size: %lu\n", VFIFO_SIZE); +} + +void fini_outfile(void) +{ + if (bfm_gbl.outfp != stdout && bfm_gbl.outfp != NULL) { + // just flush, in case we calibrate again + fflush(bfm_gbl.outfp); + } +} + +void print_u32_array(const char *label, alt_u32 *val, alt_u32 size) +{ + int i; + + fprintf(bfm_gbl.outfp, "%s", label); + for (i = 0; i < size; i++) { + fprintf(bfm_gbl.outfp, "%lu ", val[i]); + } + fprintf(bfm_gbl.outfp, "\n"); +} + +void print_s32_array(const char *label, alt_32 *val, alt_u32 size) +{ + int i; + + fprintf(bfm_gbl.outfp, "%s", label); + for (i = 0; i < size; i++) { + fprintf(bfm_gbl.outfp, "%ld ", val[i]); + } + fprintf(bfm_gbl.outfp, "\n"); +} + +void print_dqs_array(const char *label, alt_u32 *dqs) +{ + print_u32_array(label, dqs, MAX_DQS); +} + +void print_read_dq_array(const char *label, alt_32 *dq) +{ + print_s32_array(label, dq, RW_MGR_MEM_IF_READ_DQS_WIDTH*RW_MGR_MEM_DQ_PER_READ_DQS); +} + +void print_write_dq_array(const char *label, alt_32 *dq) +{ + print_s32_array(label, dq, RW_MGR_MEM_IF_WRITE_DQS_WIDTH*RW_MGR_MEM_DQ_PER_WRITE_DQS); +} + +void print_dm_array(const char *label, alt_32 *dq) +{ + print_s32_array(label, dq, RW_MGR_MEM_IF_WRITE_DQS_WIDTH*RW_MGR_NUM_DM_PER_WRITE_GROUP); +} + +void print_dqs_pos_array(const char *fmt, dqs_pos_t *dqs, int has_v, int has_ps) +{ + int i; + + if (has_v) { + fprintf(bfm_gbl.outfp, fmt, "_v: "); + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + fprintf(bfm_gbl.outfp, "%lu ", dqs[i].v); + } + fprintf(bfm_gbl.outfp, "\n"); + } + fprintf(bfm_gbl.outfp, fmt, "_p: "); + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + fprintf(bfm_gbl.outfp, "%lu ", dqs[i].p); + } + fprintf(bfm_gbl.outfp, "\n"); + fprintf(bfm_gbl.outfp, fmt, "_d: "); + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + fprintf(bfm_gbl.outfp, "%lu ", dqs[i].d); + } + fprintf(bfm_gbl.outfp, "\n"); + if (has_ps) { + fprintf(bfm_gbl.outfp, fmt, "_ps: "); + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + fprintf(bfm_gbl.outfp, "%lu ", dqs[i].ps); + } + fprintf(bfm_gbl.outfp, "\n"); + } +} + +void print_gbl(void) +{ + int i; + + fprintf(bfm_gbl.outfp, "Globals\n"); + fprintf(bfm_gbl.outfp, "bfm_stage: %s\n", BFM_GBL_GET(stage)); + // TODO: may want to do this per group, like other values + print_dqs_pos_array( "dqse_left%s ", BFM_GBL_GET(dqs_enable_left_edge), 1, 1); + print_dqs_pos_array( "dqse_right%s ", BFM_GBL_GET(dqs_enable_right_edge), 1, 1); + print_dqs_pos_array( "dqse_mid%s ", BFM_GBL_GET(dqs_enable_mid), 1, 1); + print_dqs_pos_array( "gwrite_pos%s ", BFM_GBL_GET(gwrite_pos), 0, 0); + print_dqs_pos_array( "dqswl_left%s ", BFM_GBL_GET(dqs_wlevel_left_edge), 0, 1); + print_dqs_pos_array( "dqswl_right%s", BFM_GBL_GET(dqs_wlevel_right_edge), 0, 1); + print_dqs_pos_array( "dqswl_mid%s ", BFM_GBL_GET(dqs_wlevel_mid), 0, 1); + print_read_dq_array( "dq_read_l: ", BFM_GBL_GET(dq_read_left_edge)); + print_read_dq_array( "dq_read_r: ", BFM_GBL_GET(dq_read_right_edge)); + print_write_dq_array( "dq_write_l: ", BFM_GBL_GET(dq_write_left_edge)); + print_write_dq_array( "dq_write_r: ", BFM_GBL_GET(dq_write_right_edge)); + print_dm_array( "dm_l: ", BFM_GBL_GET(dm_left_edge)); + print_dm_array( "dm_r: ", BFM_GBL_GET(dm_right_edge)); + + fprintf(bfm_gbl.outfp, "curr_read_lat: %lu\n", gbl->curr_read_lat); + fprintf(bfm_gbl.outfp, "error_stage: %lu\n", gbl->error_stage); + fprintf(bfm_gbl.outfp, "error_group: %lu\n", gbl->error_group); + fprintf(bfm_gbl.outfp, "fom_in: %lu\n", gbl->fom_in); + fprintf(bfm_gbl.outfp, "fom_out: %lu\n", gbl->fom_out); + fflush(bfm_gbl.outfp); +}; + +void bfm_set_globals_from_config() +{ + const char *filename = "board_delay_config.txt"; + const char *seq_c_prefix = "seq_c_"; + FILE *fp; + char line[1024]; + char name[64]; + int value; + + if ((fp = fopen(filename, "r")) == NULL) { + DPRINT(0, "Failed to open %s for reading; skipping config\n", filename); + return; + } + + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "%s %ld", name, &value) != 2) { + continue; + } + // for some unknown reason, sscanf of 'name' doesn't seem to work when linked into modelsim, + // so we take a different approach for the name part, by just looking at the original line + if (strncmp(line, seq_c_prefix, strlen(seq_c_prefix)) != 0) { + // not a line targetted for us + continue; + } + + if (strncmp(line, "seq_c_skip_guaranteed_write", strlen("seq_c_skip_guaranteed_write")) == 0) { + BFM_GBL_SET(bfm_skip_guaranteed_write,value); + DPRINT(0, "bfm_skip_guaranteed_write => %ld", value); + } else if (strncmp(line, "seq_c_trk_sample_count", strlen("seq_c_trk_sample_count")) == 0) { + BFM_GBL_SET(trk_sample_count,value); + DPRINT(0, "trk_sample_count => %ld", value); + } else if (strncmp(line, "seq_c_trk_long_idle_updates", strlen("seq_c_trk_long_idle_updates")) == 0) { + BFM_GBL_SET(trk_long_idle_updates,value); + DPRINT(0, "trk_long_idle_updates => %ld", value); + } else if (strncmp(line, "seq_c_lfifo_margin", strlen("seq_c_lfifo_margin")) == 0) { + BFM_GBL_SET(lfifo_margin,value/AFI_RATE_RATIO); + DPRINT(0, "lfifo_margin => %ld", value); + } else { + DPRINT(0, "Unknown Sequencer setting in line: %s\n", line); + } + } + + fclose(fp); +} +#endif + +void initialize_reg_file(void) +{ + // Initialize the register file with the correct data + IOWR_32DIRECT (REG_FILE_SIGNATURE, 0, REG_FILE_INIT_SEQ_SIGNATURE); + IOWR_32DIRECT (REG_FILE_DEBUG_DATA_ADDR, 0, 0); + IOWR_32DIRECT (REG_FILE_CUR_STAGE, 0, 0); + IOWR_32DIRECT (REG_FILE_FOM, 0, 0); + IOWR_32DIRECT (REG_FILE_FAILING_STAGE, 0, 0); + IOWR_32DIRECT (REG_FILE_DEBUG1, 0, 0); + IOWR_32DIRECT (REG_FILE_DEBUG2, 0, 0); +} + +#if HPS_HW +void initialize_hps_phy(void) +{ + // These may need to be included also: + // wrap_back_en (false) + // atpg_en (false) + // pipelineglobalenable (true) + + alt_u32 reg; + // Tracking also gets configured here because it's in the same register + alt_u32 trk_sample_count = 7500; + alt_u32 trk_long_idle_sample_count = (10 << 16) | 100; // Format is number of outer loops in the 16 MSB, sample count in 16 LSB. + + reg = 0; +#if DDR3 || DDR2 + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2); +#else + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(1); +#endif + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1); + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1); + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1); + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0); +#if LPDDR2 + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(0); +#else + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1); +#endif + // Fix for long latency VFIFO + // This field selects the intrinsic latency to RDATA_EN/FULL path. 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles. + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0); + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(trk_sample_count); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(trk_sample_count >> SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH); + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(trk_long_idle_sample_count); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(trk_long_idle_sample_count >> SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_OFFSET, reg); +} +#endif + +#if USE_DQS_TRACKING + +#if HHP_HPS +void initialize_tracking(void) +{ + alt_u32 concatenated_longidle = 0x0; + alt_u32 concatenated_delays = 0x0; + alt_u32 concatenated_rw_addr = 0x0; + alt_u32 concatenated_refresh = 0x0; + alt_u32 dtaps_per_ptap; + alt_u32 tmp_delay; + + // compute usable version of value in case we skip full computation later + dtaps_per_ptap = 0; + tmp_delay = 0; + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DCHAIN_TAP; + } + dtaps_per_ptap--; + +#if BFM_MODE + concatenated_longidle = concatenated_longidle ^ (bfm_gbl.trk_long_idle_updates > 0 ? bfm_gbl.trk_long_idle_updates : 10); //longidle outer loop + concatenated_longidle = concatenated_longidle << 16; + concatenated_longidle = concatenated_longidle ^ (bfm_gbl.trk_sample_count > 0 ? bfm_gbl.trk_sample_count : 100); //longidle sample count +#else + concatenated_longidle = concatenated_longidle ^ 10; //longidle outer loop + concatenated_longidle = concatenated_longidle << 16; + concatenated_longidle = concatenated_longidle ^ 100; //longidle sample count +#endif + + concatenated_delays = concatenated_delays ^ 243; // trfc, worst case of 933Mhz 4Gb + concatenated_delays = concatenated_delays << 8; + concatenated_delays = concatenated_delays ^ 14; // trcd, worst case + concatenated_delays = concatenated_delays << 8; + concatenated_delays = concatenated_delays ^ 10; // vfifo wait + concatenated_delays = concatenated_delays << 8; + concatenated_delays = concatenated_delays ^ 4; // mux delay + +#if DDR3 || LPDDR2 + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_IDLE; + concatenated_rw_addr = concatenated_rw_addr << 8; + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_ACTIVATE_1; + concatenated_rw_addr = concatenated_rw_addr << 8; + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_SGLE_READ; + concatenated_rw_addr = concatenated_rw_addr << 8; + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_PRECHARGE_ALL; +#endif + +#if DDR3 || LPDDR2 + concatenated_refresh = concatenated_refresh ^ __RW_MGR_REFRESH_ALL; +#else + concatenated_refresh = concatenated_refresh ^ 0; +#endif + concatenated_refresh = concatenated_refresh << 24; + concatenated_refresh = concatenated_refresh ^ 1000; // trefi + + // Initialize the register file with the correct data + IOWR_32DIRECT (REG_FILE_DTAPS_PER_PTAP, 0, dtaps_per_ptap); +#if BFM_MODE + IOWR_32DIRECT (REG_FILE_TRK_SAMPLE_COUNT, 0, bfm_gbl.trk_sample_count > 0 ? bfm_gbl.trk_sample_count : 7500); +#else + IOWR_32DIRECT (REG_FILE_TRK_SAMPLE_COUNT, 0, 7500); +#endif + IOWR_32DIRECT (REG_FILE_TRK_LONGIDLE, 0, concatenated_longidle); + IOWR_32DIRECT (REG_FILE_DELAYS, 0, concatenated_delays); + IOWR_32DIRECT (REG_FILE_TRK_RW_MGR_ADDR, 0, concatenated_rw_addr); + IOWR_32DIRECT (REG_FILE_TRK_READ_DQS_WIDTH, 0, RW_MGR_MEM_IF_READ_DQS_WIDTH); + IOWR_32DIRECT (REG_FILE_TRK_RFSH, 0, concatenated_refresh); +} + +#else + +void initialize_tracking(void) +{ + alt_u32 concatenated_longidle = 0x0; + alt_u32 concatenated_delays = 0x0; + alt_u32 concatenated_rw_addr = 0x0; + alt_u32 concatenated_refresh = 0x0; + alt_u32 dtaps_per_ptap; + alt_u32 tmp_delay; + + // compute usable version of value in case we skip full computation later + dtaps_per_ptap = 0; + tmp_delay = 0; + while (tmp_delay < IO_DELAY_PER_OPA_TAP) { + dtaps_per_ptap++; + tmp_delay += IO_DELAY_PER_DCHAIN_TAP; + } + dtaps_per_ptap--; + +#if BFM_MODE + concatenated_longidle = concatenated_longidle ^ (bfm_gbl.trk_long_idle_updates > 0 ? bfm_gbl.trk_long_idle_updates : 10); //longidle outer loop + concatenated_longidle = concatenated_longidle << 16; + concatenated_longidle = concatenated_longidle ^ (bfm_gbl.trk_sample_count > 0 ? bfm_gbl.trk_sample_count : 100); //longidle sample count +#else + concatenated_longidle = concatenated_longidle ^ 10; //longidle outer loop + concatenated_longidle = concatenated_longidle << 16; + concatenated_longidle = concatenated_longidle ^ 100; //longidle sample count +#endif + +#if FULL_RATE + concatenated_delays = concatenated_delays ^ 60; // trfc +#endif +#if HALF_RATE + concatenated_delays = concatenated_delays ^ 30; // trfc +#endif +#if QUARTER_RATE + concatenated_delays = concatenated_delays ^ 15; // trfc +#endif + concatenated_delays = concatenated_delays << 8; +#if FULL_RATE + concatenated_delays = concatenated_delays ^ 4; // trcd +#endif +#if HALF_RATE + concatenated_delays = concatenated_delays ^ 2; // trcd +#endif +#if QUARTER_RATE + concatenated_delays = concatenated_delays ^ 0; // trcd +#endif + concatenated_delays = concatenated_delays << 8; +#if FULL_RATE + concatenated_delays = concatenated_delays ^ 5; // vfifo wait +#endif +#if HALF_RATE + concatenated_delays = concatenated_delays ^ 3; // vfifo wait +#endif +#if QUARTER_RATE + concatenated_delays = concatenated_delays ^ 1; // vfifo wait +#endif + concatenated_delays = concatenated_delays << 8; +#if FULL_RATE + concatenated_delays = concatenated_delays ^ 4; // mux delay +#endif +#if HALF_RATE + concatenated_delays = concatenated_delays ^ 2; // mux delay +#endif +#if QUARTER_RATE + concatenated_delays = concatenated_delays ^ 0; // mux delay +#endif + +#if DDR3 || LPDDR2 + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_IDLE; + concatenated_rw_addr = concatenated_rw_addr << 8; + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_ACTIVATE_1; + concatenated_rw_addr = concatenated_rw_addr << 8; + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_SGLE_READ; + concatenated_rw_addr = concatenated_rw_addr << 8; + concatenated_rw_addr = concatenated_rw_addr ^ __RW_MGR_PRECHARGE_ALL; +#endif + +#if DDR3 || LPDDR2 + concatenated_refresh = concatenated_refresh ^ __RW_MGR_REFRESH_ALL; +#else + concatenated_refresh = concatenated_refresh ^ 0; +#endif + concatenated_refresh = concatenated_refresh << 24; + concatenated_refresh = concatenated_refresh ^ 546; // trefi + + IOWR_32DIRECT (TRK_DTAPS_PER_PTAP, 0, dtaps_per_ptap); +#if BFM_MODE + IOWR_32DIRECT (TRK_SAMPLE_COUNT, 0, bfm_gbl.trk_sample_count > 0 ? bfm_gbl.trk_sample_count : 7500); +#else + IOWR_32DIRECT (TRK_SAMPLE_COUNT, 0, 7500); +#endif + IOWR_32DIRECT (TRK_LONGIDLE, 0, concatenated_longidle); + IOWR_32DIRECT (TRK_DELAYS, 0, concatenated_delays); + IOWR_32DIRECT (TRK_RW_MGR_ADDR, 0, concatenated_rw_addr); + IOWR_32DIRECT (TRK_READ_DQS_WIDTH, 0, RW_MGR_MEM_IF_READ_DQS_WIDTH); + IOWR_32DIRECT (TRK_RFSH, 0, concatenated_refresh); +} +#endif /* HHP_HPS */ +#endif /* USE_DQS_TRACKING */ + +#if HHP_HPS_SIMULATION +void initialize_hps_controller(void) +{ + alt_u32 reg; + alt_u32 memtype; + alt_u32 ecc; + alt_u32 ctrl_width; + alt_u32 mem_bl; + + if (DDR2) { + memtype = 1; + } else if (DDR3) { + memtype = 2; + } else if (LPDDR1) { + memtype = 3; + } else if (LPDDR2) { + memtype = 4; + } else { + // should never happen + memtype = 0; + } + + if (RW_MGR_MEM_DATA_WIDTH == 24 || RW_MGR_MEM_DATA_WIDTH == 40) { + // we have ecc + ecc = 1; + } else { + ecc = 0; + } + + reg = 0; + reg |= SDR_CTRLGRP_CTRLCFG_MEMTYPE_SET(memtype); + reg |= SDR_CTRLGRP_CTRLCFG_MEMBL_SET(MEM_BURST_LENGTH); + reg |= SDR_CTRLGRP_CTRLCFG_ADDRORDER_SET(ADDR_ORDER); + reg |= SDR_CTRLGRP_CTRLCFG_ECCEN_SET(ecc); + reg |= SDR_CTRLGRP_CTRLCFG_ECCCORREN_SET(0); + reg |= SDR_CTRLGRP_CTRLCFG_CFG_ENABLE_ECC_CODE_OVERWRITES_SET(0); + reg |= SDR_CTRLGRP_CTRLCFG_GENSBE_SET(0); + reg |= SDR_CTRLGRP_CTRLCFG_GENDBE_SET(0); + reg |= SDR_CTRLGRP_CTRLCFG_REORDEREN_SET(1); + reg |= SDR_CTRLGRP_CTRLCFG_STARVELIMIT_SET(0x8); + reg |= SDR_CTRLGRP_CTRLCFG_DQSTRKEN_SET(USE_DQS_TRACKING); // Do we want this? +#if DM_PINS_ENABLED + reg |= SDR_CTRLGRP_CTRLCFG_NODMPINS_SET(0); +#else + reg |= SDR_CTRLGRP_CTRLCFG_NODMPINS_SET(1); +#endif + reg |= SDR_CTRLGRP_CTRLCFG_BURSTINTREN_SET(0); + reg |= SDR_CTRLGRP_CTRLCFG_BURSTTERMEN_SET(0); + reg |= SDR_CTRLGRP_CTRLCFG_OUTPUTREG_SET(0); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_CTRLCFG_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_DRAMTIMING1_TCWL_SET(MEM_WTCL_INT); + reg |= SDR_CTRLGRP_DRAMTIMING1_TAL_SET(MEM_ATCL_INT); + reg |= SDR_CTRLGRP_DRAMTIMING1_TCL_SET(MEM_TCL); + reg |= SDR_CTRLGRP_DRAMTIMING1_TRRD_SET(MEM_TRRD); + reg |= SDR_CTRLGRP_DRAMTIMING1_TFAW_SET(MEM_TFAW); + reg |= SDR_CTRLGRP_DRAMTIMING1_TRFC_SET(MEM_TRFC); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMTIMING1_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_DRAMTIMING2_TREFI_SET(MEM_TREFI); + reg |= SDR_CTRLGRP_DRAMTIMING2_TRCD_SET(MEM_TRCD); + reg |= SDR_CTRLGRP_DRAMTIMING2_TRP_SET(MEM_TRP); + reg |= SDR_CTRLGRP_DRAMTIMING2_TWTR_SET(MEM_TWTR); + reg |= SDR_CTRLGRP_DRAMTIMING2_TWR_SET(MEM_TWR); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMTIMING2_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_DRAMTIMING3_TRTP_SET(MEM_TRTP); + reg |= SDR_CTRLGRP_DRAMTIMING3_TRAS_SET(MEM_TRAS); + reg |= SDR_CTRLGRP_DRAMTIMING3_TRC_SET(MEM_TRC); + reg |= SDR_CTRLGRP_DRAMTIMING3_TMRD_SET(MEM_TMRD_CK); + reg |= SDR_CTRLGRP_DRAMTIMING3_TCCD_SET(CFG_TCCD); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMTIMING3_OFFSET, reg); + + + // These values don't really matter for the HPS simulation + reg = 0; + reg |= SDR_CTRLGRP_DRAMTIMING4_SELFRFSHEXIT_SET(512); + reg |= SDR_CTRLGRP_DRAMTIMING4_PWRDOWNEXIT_SET(10); + reg |= SDR_CTRLGRP_DRAMTIMING4_MINPWRSAVECYCLES_SET(0); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMTIMING4_OFFSET, reg); + + // These values don't really matter for the HPS simulation + reg = 0; + reg |= SDR_CTRLGRP_LOWPWRTIMING_AUTOPDCYCLES_SET(0); + reg |= SDR_CTRLGRP_LOWPWRTIMING_CLKDISABLECYCLES_SET(0); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_LOWPWRTIMING_OFFSET, reg); + + + // These values don't really matter for the HPS simulation + reg = 0; + reg |= SDR_CTRLGRP_DRAMODT_CFG_WRITE_ODT_CHIP_SET(0); + reg |= SDR_CTRLGRP_DRAMODT_CFG_READ_ODT_CHIP_SET(0); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMODT_OFFSET, reg); + + + reg = 0; + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_ACT_TO_RDWR_SET(INTG_EXTRA_CTL_CLK_ACT_TO_RDWR); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_ACT_TO_PCH_SET(INTG_EXTRA_CTL_CLK_RD_TO_PCH); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_ACT_TO_ACT_SET(INTG_EXTRA_CTL_CLK_ACT_TO_ACT); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_RD_TO_RD_SET(INTG_EXTRA_CTL_CLK_RD_TO_RD); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_RD_TO_RD_DIFF_CHIP_SET(INTG_EXTRA_CTL_CLK_RD_TO_RD_DIFF_CHIP); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_RD_TO_WR_SET(INTG_EXTRA_CTL_CLK_RD_TO_WR); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_RD_TO_WR_BC_SET(INTG_EXTRA_CTL_CLK_RD_TO_WR_BC); + reg |= SDR_CTRLGRP_EXTRATIME1_CFG_EXTRA_CTL_CLK_RD_TO_WR_DIFF_CHIP_SET(INTG_EXTRA_CTL_CLK_RD_TO_WR_DIFF_CHIP); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_EXTRATIME1_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_RD_TO_PCH_SET(INTG_EXTRA_CTL_CLK_RD_TO_PCH); + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_RD_AP_TO_VALID_SET(INTG_EXTRA_CTL_CLK_RD_AP_TO_VALID); + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_WR_TO_WR_SET(INTG_EXTRA_CTL_CLK_WR_TO_WR); + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_WR_TO_WR_DIFF_CHIP_SET(INTG_EXTRA_CTL_CLK_WR_TO_WR_DIFF_CHIP); + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_WR_TO_RD_SET(INTG_EXTRA_CTL_CLK_WR_TO_RD); + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_WR_TO_RD_BC_SET(INTG_EXTRA_CTL_CLK_WR_TO_RD_BC); + reg |= SDR_CTRLGRP_EXTRATIME2_CFG_EXTRA_CTL_CLK_WR_TO_RD_DIFF_CHIP_SET(INTG_EXTRA_CTL_CLK_WR_TO_RD_DIFF_CHIP); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_EXTRATIME2_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_WR_TO_PCH_SET(INTG_EXTRA_CTL_CLK_WR_TO_PCH); + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_WR_AP_TO_VALID_SET(INTG_EXTRA_CTL_CLK_WR_AP_TO_VALID); + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_PCH_TO_VALID_SET(INTG_EXTRA_CTL_CLK_PCH_TO_VALID); + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_PCH_ALL_TO_VALID_SET(INTG_EXTRA_CTL_CLK_PCH_ALL_TO_VALID); + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_ACT_TO_ACT_DIFF_BANK_SET(INTG_EXTRA_CTL_CLK_ACT_TO_ACT_DIFF_BANK); + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_FOUR_ACT_TO_ACT_SET(INTG_EXTRA_CTL_CLK_FOUR_ACT_TO_ACT); + reg |= SDR_CTRLGRP_EXTRATIME3_CFG_EXTRA_CTL_CLK_ARF_TO_VALID_SET(INTG_EXTRA_CTL_CLK_ARF_TO_VALID); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_EXTRATIME3_OFFSET, reg); + + + reg = 0; + reg |= SDR_CTRLGRP_EXTRATIME4_CFG_EXTRA_CTL_CLK_PDN_TO_VALID_SET(INTG_EXTRA_CTL_CLK_PDN_TO_VALID); + reg |= SDR_CTRLGRP_EXTRATIME4_CFG_EXTRA_CTL_CLK_SRF_TO_VALID_SET(INTG_EXTRA_CTL_CLK_SRF_TO_VALID); + reg |= SDR_CTRLGRP_EXTRATIME4_CFG_EXTRA_CTL_CLK_SRF_TO_ZQ_CAL_SET(INTG_EXTRA_CTL_CLK_SRF_TO_ZQ_CAL); + reg |= SDR_CTRLGRP_EXTRATIME4_CFG_EXTRA_CTL_CLK_ARF_PERIOD_SET(INTG_EXTRA_CTL_CLK_ARF_PERIOD); + reg |= SDR_CTRLGRP_EXTRATIME4_CFG_EXTRA_CTL_CLK_PDN_PERIOD_SET(INTG_EXTRA_CTL_CLK_PDN_PERIOD); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_EXTRATIME4_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_DRAMADDRW_COLBITS_SET(MEM_IF_COL_ADDR_WIDTH); + reg |= SDR_CTRLGRP_DRAMADDRW_ROWBITS_SET(MEM_IF_ROW_ADDR_WIDTH); + reg |= SDR_CTRLGRP_DRAMADDRW_BANKBITS_SET(MEM_IF_BANKADDR_WIDTH); + reg |= SDR_CTRLGRP_DRAMADDRW_CSBITS_SET(MEM_IF_CS_WIDTH > 1 ? MEM_IF_CHIP_BITS : 0); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMADDRW_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_DRAMIFWIDTH_IFWIDTH_SET(RW_MGR_MEM_DATA_WIDTH); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMIFWIDTH_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_DRAMDEVWIDTH_DEVWIDTH_SET(RW_MGR_MEM_DQ_PER_READ_DQS); // should always be 8 + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_DRAMDEVWIDTH_OFFSET, reg); + + switch (RW_MGR_MEM_DATA_WIDTH) { + case 8: ctrl_width = 0; break; + case 16: // FALLTHROUGH + case 24: ctrl_width = 1; break; + case 32: // FALLTHROUGH + case 40: ctrl_width = 2; break; + default: ctrl_width = 0; break; /* shouldn't happen */ + } + + reg = 0; + reg |= SDR_CTRLGRP_CTRLWIDTH_CTRLWIDTH_SET(ctrl_width); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_CTRLWIDTH_OFFSET, reg); + + // hard-coded values taken from test bench + reg = 0; + // 30'b111111111111010001000010001000 + reg |= SDR_CTRLGRP_MPPRIORITY_USERPRIORITY_SET(0x3FFD1088); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_MPPRIORITY_OFFSET, reg); + + // hard-coded values taken from test bench + reg = 0; + // first 32 bits of 50'b01111011111000010000100001000010000100001000010000 + reg |= SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_0_STATICWEIGHT_31_0_SET(0x21084210); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_0_OFFSET, reg); + + // hard-coded values taken from test bench + reg = 0; + // first next 18 bits of 50'b01111011111000010000100001000010000100001000010000 + reg |= SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_1_STATICWEIGHT_49_32_SET(0x1EF84); + // first 14 of 64'b0011111000000000000000000000000000000000001000000010000000100000 + reg |= SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_1_SUMOFWEIGHTS_13_0_SET(0x2002); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_1_OFFSET, reg); + + // hard-coded values taken from test bench + reg = 0; + // next 32 bits of 64'b0011111000000000000000000000000000000000001000000010000000100000 + reg |= SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_2_SUMOFWEIGHTS_45_14_SET(0x80); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_2_OFFSET, reg); + + // hard-coded values taken from test bench + reg = 0; + // next 18 bits of 64'b0011111000000000000000000000000000000000001000000010000000100000 + reg |= SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_3_SUMOFWEIGHTS_63_46_SET(0xF800); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_MPWEIGHT_MPWEIGHT_3_OFFSET, reg); + + switch (MEM_BURST_LENGTH) { + case 2: mem_bl = 0; break; + case 4: mem_bl = 1; break; + case 8: mem_bl = 2; break; + default: mem_bl = 2; break; // should never happen + } + reg = 0; + reg |= SDR_CTRLGRP_STATICCFG_MEMBL_SET(mem_bl); + reg |= SDR_CTRLGRP_STATICCFG_USEECCASDATA_SET(0); /* allow fpga to access ecc bits; not supported */ + reg |= SDR_CTRLGRP_STATICCFG_APPLYCFG_SET(1); /* apply all of the configs here and above */ + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_STATICCFG_OFFSET, reg); + + reg = 0; + reg |= SDR_CTRLGRP_FPGAPORTRST_PORTRSTN_SET(~0); + IOWR_32DIRECT (BASE_MMR, SDR_CTRLGRP_FPGAPORTRST_OFFSET, reg); +} +#endif + +void user_init_cal_req(void) +{ + alt_u32 scc_afi_reg; + + scc_afi_reg = IORD_32DIRECT (SCC_MGR_AFI_CAL_INIT, 0); + + if (scc_afi_reg == 1 || scc_afi_reg == 16) {// 1 is initialization request + initialize(); + rw_mgr_mem_initialize (); + rw_mgr_mem_handoff (); + IOWR_32DIRECT (PHY_MGR_MUX_SEL, 0, 0); + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, PHY_MGR_CAL_SUCCESS); + } else if (scc_afi_reg == 2 || scc_afi_reg == 32) { +#if ENABLE_NON_DES_CAL + run_mem_calibrate (0); +#else + run_mem_calibrate(); +#endif + } +#if ENABLE_NON_DES_CAL + else if (scc_afi_reg == 4) { + //non destructive mem init + IOWR_32DIRECT (SCC_MGR_AFI_CAL_INIT, 0, scc_afi_reg & ~(1 << 2)); + + rw_mgr_mem_initialize_no_init(); + + + } else if (scc_afi_reg == 8) { + //non destructive mem calibrate + IOWR_32DIRECT (SCC_MGR_AFI_CAL_INIT, 0, scc_afi_reg & ~(1 << 3)); + + run_mem_calibrate (1); + IOWR_32DIRECT (RW_MGR_ENABLE_REFRESH, 0, 0); // Disable refresh engine + } +#endif + +} + +#if TRACKING_WATCH_TEST || TRACKING_ERROR_TEST +void decrement_dqs_en_phase (alt_u32 group) { + alt_u32 phase = 0; + alt_u32 v; + phase = READ_SCC_DQS_EN_PHASE(group); + + if (phase == 0) { + rw_mgr_decr_vfifo(group, &v); + scc_mgr_set_dqs_en_phase(group, IO_DQS_EN_PHASE_MAX); + scc_mgr_set_dqs_en_delay(group, IO_DQS_EN_DELAY_MAX); + return; + } + + scc_mgr_set_dqs_en_phase(group, phase - 1); + scc_mgr_set_dqs_en_delay(group, IO_DQS_EN_DELAY_MAX); +} + +void read_samples (void) +{ + alt_u32 group = 0; + alt_32 sample_count = 0; + alt_u32 delay = 0; + alt_u32 phase = 0; + + alt_u32 dtaps_per_ptap = 0; + + dtaps_per_ptap = IORD_32DIRECT(0xD0000, 0); + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][group].dtaps_per_ptap, dtaps_per_ptap); + +#if TRACKING_WATCH_TEST + for (;;) { + // Stall tracking to ensure accurate reading + IOWR_32DIRECT (TRK_STALL, 0, TRK_STALL_REQ_VAL); + // Wait for tracking manager to ack stall request + while (IORD_32DIRECT (TRK_STALL, 0) != TRK_STALL_ACKED_VAL) { + } + + for (group = 0; group < RW_MGR_MEM_IF_READ_DQS_WIDTH; group++) { + // Read sample counter + sample_count = IORD_32DIRECT(0x58F00, group << 2); + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][group].sample_count, sample_count); + delay = READ_SCC_DQS_EN_DELAY(group); + phase = READ_SCC_DQS_EN_PHASE(group); + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][group].dqs_en_phase, (delay | (phase << 16))); + } + + // Release stall + IOWR_32DIRECT(TRK_STALL, 0, 0); + } +#endif + +#if TRACKING_ERROR_TEST + for (group = 0; group < RW_MGR_MEM_IF_READ_DQS_WIDTH; group++) { + // Read sample counter + sample_count = IORD_32DIRECT(0x58F00, group << 2); + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][group].sample_count, sample_count); + delay = READ_SCC_DQS_EN_DELAY(group); + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][group].dqs_en_delay, delay); + phase = READ_SCC_DQS_EN_PHASE(group); + TCLRPT_SET(debug_cal_report->cal_dqs_in_settings[curr_shadow_reg][group].dqs_en_phase, phase); + } +#endif +} + +void tracking_sample_check (void) +{ + alt_u32 group = 0; + alt_u32 t11_d = 0; + alt_u32 read_val = 0; + + alt_u32 num_samples = 0; + alt_u32 num_samples_max = 7500; + + alt_u32 bit_chk = 0; + alt_u32 test_status = 0; + + for (group = 0; group < RW_MGR_MEM_IF_READ_DQS_WIDTH; group++) + { + // TODO: Figure out whether the sample counter and sample run + // values should be defined somewhere, or just leave them + // hardcoded. + IOWR_32DIRECT(0x58F00, group << 2, 0x00); + } + + for (num_samples = 0; num_samples < num_samples_max; num_samples++) { + //do a read + //test_status = rw_mgr_mem_calibrate_read_test_all_ranks (group, 1, PASS_ONE_BIT, &bit_chk, 0); + //do a write + test_status = rw_mgr_mem_calibrate_write_test_all_ranks (group, 0, PASS_ONE_BIT, &bit_chk); + + // do a sample + IOWR_32DIRECT(0x58FFC, 0, 0xFF); + } + + read_samples(); +} + +void poll_for_sample_check (void) +{ + alt_u32 check_status = 2; + alt_u32 delay = 0; + alt_u32 group = 0; + + alt_u32 READY_FOR_READ = 0xFE; + alt_u32 READ_FINISHED = 0xFD; + alt_u32 EXIT_LOOP = 0x00; + alt_u32 FINISHED_SIGNAL = 0xFF; + + for (;;) { + check_status = IORD_32DIRECT(REG_FILE_TRK_SAMPLE_CHECK, 0); + + if (check_status == READY_FOR_READ) { + for (group = 0; group < RW_MGR_MEM_IF_READ_DQS_WIDTH; group++) { + + delay = READ_SCC_DQS_EN_DELAY(group); + + if (delay == 0) { + decrement_dqs_en_phase(group); + } else { + delay--; + scc_mgr_set_dqs_en_delay(group, delay); + } + + IOWR_32DIRECT (SCC_MGR_DQS_ENA, 0, group); + } + + IOWR_32DIRECT (SCC_MGR_UPD, 0, 0); + + tracking_sample_check(); + check_status = IORD_32DIRECT(REG_FILE_TRK_SAMPLE_CHECK, 0); + if (check_status != EXIT_LOOP) { + IOWR_32DIRECT(REG_FILE_TRK_SAMPLE_CHECK, 0, READ_FINISHED); + } + } + if (check_status == EXIT_LOOP) { + IOWR_32DIRECT(REG_FILE_TRK_SAMPLE_CHECK, 0, FINISHED_SIGNAL); + break; + } + } +} +#endif // TRACKING_WATCH_TEST || TRACKING_ERROR_TEST + +#if BFM_MODE +int seq_main(void) +#elif HPS_HW +int sdram_calibration(void) +#else +int main(void) +#endif +{ + param_t my_param; + gbl_t my_gbl; + alt_u32 pass; + alt_u32 i; + + param = &my_param; + gbl = &my_gbl; + + // Initialize the debug mode flags + gbl->phy_debug_mode_flags = 0; + // Set the calibration enabled by default + gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT; + // Only enable margining by default if requested +#if ENABLE_MARGIN_REPORT_GEN + gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_MARGIN_RPT; +#endif + // Only sweep all groups (regardless of fail state) by default if requested +#if ENABLE_SWEEP_ALL_GROUPS + gbl->phy_debug_mode_flags |= PHY_DEBUG_SWEEP_ALL_GROUPS; +#endif + //Set enabled read test by default +#if DISABLE_GUARANTEED_READ + gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ; +#endif +#if ENABLE_NON_DESTRUCTIVE_CALIB + gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_NON_DESTRUCTIVE_CALIBRATION; +#endif + +#if BFM_MODE + init_outfile(); + bfm_set_globals_from_config(); +#endif + + // Initialize the register file + initialize_reg_file(); + +#if HPS_HW + // Initialize any PHY CSR + initialize_hps_phy(); +#endif + +#if HHP_HPS + scc_mgr_initialize(); +#endif + +#if USE_DQS_TRACKING + initialize_tracking(); +#endif + + // Initialize the TCL report. This must occur before any printf + // but after the debug mode flags and register file +#if ENABLE_TCL_DEBUG + tclrpt_initialize(&my_debug_data); +#endif + + // USER Enable all ranks, groups + for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++) { + param->skip_ranks[i] = 0; + } + for (i = 0; i < NUM_SHADOW_REGS; ++i) { + param->skip_shadow_regs[i] = 0; + } + param->skip_groups = 0; + + IPRINT("Preparing to start memory calibration"); + + TRACE_FUNC(); + DPRINT(1, "%s%s %s ranks=%lu cs/dimm=%lu dq/dqs=%lu,%lu vg/dqs=%lu,%lu dqs=%lu,%lu dq=%lu dm=%lu " + "ptap_delay=%lu dtap_delay=%lu dtap_dqsen_delay=%lu, dll=%lu", + RDIMM ? "r" : (LRDIMM ? "l" : ""), + DDR2 ? "DDR2" : (DDR3 ? "DDR3" : (QDRII ? "QDRII" : (RLDRAMII ? "RLDRAMII" : (RLDRAM3 ? "RLDRAM3" : "??PROTO??")))), + FULL_RATE ? "FR" : (HALF_RATE ? "HR" : (QUARTER_RATE ? "QR" : "??RATE??")), + (long unsigned int)RW_MGR_MEM_NUMBER_OF_RANKS, + (long unsigned int)RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM, + (long unsigned int)RW_MGR_MEM_DQ_PER_READ_DQS, + (long unsigned int)RW_MGR_MEM_DQ_PER_WRITE_DQS, + (long unsigned int)RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS, + (long unsigned int)RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS, + (long unsigned int)RW_MGR_MEM_IF_READ_DQS_WIDTH, + (long unsigned int)RW_MGR_MEM_IF_WRITE_DQS_WIDTH, + (long unsigned int)RW_MGR_MEM_DATA_WIDTH, + (long unsigned int)RW_MGR_MEM_DATA_MASK_WIDTH, + (long unsigned int)IO_DELAY_PER_OPA_TAP, + (long unsigned int)IO_DELAY_PER_DCHAIN_TAP, + (long unsigned int)IO_DELAY_PER_DQS_EN_DCHAIN_TAP, + (long unsigned int)IO_DLL_CHAIN_LENGTH); + DPRINT(1, "max values: en_p=%lu dqdqs_p=%lu en_d=%lu dqs_in_d=%lu io_in_d=%lu io_out1_d=%lu io_out2_d=%lu" + "dqs_in_reserve=%lu dqs_out_reserve=%lu", + (long unsigned int)IO_DQS_EN_PHASE_MAX, + (long unsigned int)IO_DQDQS_OUT_PHASE_MAX, + (long unsigned int)IO_DQS_EN_DELAY_MAX, + (long unsigned int)IO_DQS_IN_DELAY_MAX, + (long unsigned int)IO_IO_IN_DELAY_MAX, + (long unsigned int)IO_IO_OUT1_DELAY_MAX, + (long unsigned int)IO_IO_OUT2_DELAY_MAX, + (long unsigned int)IO_DQS_IN_RESERVE, + (long unsigned int)IO_DQS_OUT_RESERVE); + +#if HCX_COMPAT_MODE || ENABLE_INST_ROM_WRITE + hc_initialize_rom_data(); +#endif + +#if !HARD_PHY + // Hard PHY does not support soft reset + IOWR_32DIRECT (RW_MGR_SOFT_RESET, 0, 0); +#endif + + //USER update info for sims + reg_file_set_stage(CAL_STAGE_NIL); + reg_file_set_group(0); + + // Load global needed for those actions that require + // some dynamic calibration support +#if HARD_PHY + dyn_calib_steps = STATIC_CALIB_STEPS; +#else + dyn_calib_steps = IORD_32DIRECT(PHY_MGR_CALIB_SKIP_STEPS, 0); +#endif + + // Load global to allow dynamic selection of delay loop settings + // based on calibration mode + if (!((DYNAMIC_CALIB_STEPS) & CALIB_SKIP_DELAY_LOOPS)) { + skip_delay_mask = 0xff; + } else { + skip_delay_mask = 0x0; + } + + +#ifdef TEST_SIZE + if (!check_test_mem(1)) { + IOWR_32DIRECT (PHY_MGR_CAL_DEBUG_INFO, 0, 0x9090); + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, PHY_MGR_CAL_FAIL); + } + write_test_mem(); + if (!check_test_mem(0)) { + IOWR_32DIRECT (PHY_MGR_CAL_DEBUG_INFO, 0, 0x9191); + IOWR_32DIRECT (PHY_MGR_CAL_STATUS, 0, PHY_MGR_CAL_FAIL); + } +#endif + + +#if HHP_HPS_SIMULATION + // configure controller + initialize_hps_controller(); +#endif + +#if ENABLE_TCL_DEBUG && USE_USER_RDIMM_VALUE + tclrpt_loop(); +#endif + + +#if ENABLE_NON_DES_CAL_TEST + rw_mgr_mem_initialize (); + +// IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_IDLE); +// IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_SELF_REFRESH); + + pass = run_mem_calibrate (1); + +#else + +#if ENABLE_NON_DES_CAL +#if ENABLE_TCL_DEBUG + tclrpt_loop(); +#else + pass = run_mem_calibrate (1); +#endif +#else + pass = run_mem_calibrate (); +#endif + +#endif + +#if TRACKING_WATCH_TEST + if (IORD_32DIRECT(REG_FILE_TRK_SAMPLE_CHECK, 0) == 0xEE) { + read_samples(); + } +#endif + +#if ENABLE_PRINTF_LOG + IPRINT("Calibration complete"); + // Send the end of transmission character + IPRINT("%c", 0x4); +#endif + + + + +#if BFM_MODE +#if ENABLE_TCL_DEBUG + tclrpt_dump_internal_data(); +#endif + bfm_sequencer_is_done(); +#elif HHP_HPS_SIMULATION + // nothing to do for HPS simulation following calibration + while (1) { + } +#elif ENABLE_TCL_DEBUG + #if HPS_HW + // EMPTY + #else + tclrpt_loop(); + #endif +#else + #if HPS_HW + // EMPTY + #else + while (1) { + user_init_cal_req(); + } + + #endif +#endif + + +return pass; +} + + +#if ENABLE_BRINGUP_DEBUGGING + +/////////////////////////////////////////////////////////////////////////////////////// +// Bring-Up test Support +/////////////////////////////////////////////////////////////////////////////////////// + + +void do_bringup_test_guaranteed_write (void) +{ + alt_u32 r; + + TRACE_FUNC(); + + for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) { + if (param->skip_ranks[r]) { + //USER request to skip the rank + + continue; + } + + //USER set rank + set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE); + + //USER Load up a constant bursts + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_GUARANTEED_WRITE_0_1_A_5_WAIT0); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_GUARANTEED_WRITE_0_1_A_5_WAIT1); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_2, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_2, 0, __RW_MGR_GUARANTEED_WRITE_0_1_A_5_WAIT2); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_3, 0, 0x20); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_3, 0, __RW_MGR_GUARANTEED_WRITE_0_1_A_5_WAIT3); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, 0, __RW_MGR_GUARANTEED_WRITE_0_1_A_5); + } + + set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF); +} + +void do_bringup_test_clear_di_buf (alt_u32 group) +{ + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 128); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_DO_CLEAR_DI_BUF); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, group << 2, __RW_MGR_DO_CLEAR_DI_BUF); +} + +void do_bringup_test_guaranteed_read (alt_u32 group) +{ + IOWR_32DIRECT (PHY_MGR_CMD_FIFO_RESET, 0, 0); + IOWR_32DIRECT (RW_MGR_RESET_READ_DATAPATH, 0, 0); + + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_0, 0, 16); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_0, 0, __RW_MGR_DO_TEST_READ); + IOWR_32DIRECT (RW_MGR_LOAD_CNTR_1, 0, 16); + IOWR_32DIRECT (RW_MGR_LOAD_JUMP_ADD_1, 0, __RW_MGR_DO_TEST_READ_POST_WAIT); + + IOWR_32DIRECT (RW_MGR_RUN_SINGLE_GROUP, group << 2, __RW_MGR_DO_TEST_READ); +} + +void do_bringup_test () +{ + int i; + alt_u32 group; + alt_u32 v = 0; + + group = 0; + + mem_config (); + + // 15 is the maximum latency (should make dependent on actual design + IOWR_32DIRECT (PHY_MGR_PHY_RLAT, 0, 15); /* lfifo setting */ + +#if ARRIAV || CYCLONEV + for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) { + IOWR_32DIRECT (SCC_MGR_GROUP_COUNTER, 0, i); + scc_set_bypass_mode(i, 0); + } +#endif + + // initialize global buffer to something known + for (i = 0; i < sizeof(di_buf_gbl); i++) { + di_buf_gbl[i] = 0xee; + } + + // pre-increment vfifo to ensure not at max value + rw_mgr_incr_vfifo(group, &v); + rw_mgr_incr_vfifo(group, &v); + + do_bringup_test_clear_di_buf(group); + + while (1) { + do_bringup_test_guaranteed_write(); + do_bringup_test_guaranteed_read(group); + load_di_buf_gbl(); + rw_mgr_incr_vfifo(group, &v); + } +} + + +#endif // ENABLE_BRINGUP_DEBUGGING + +#if ENABLE_ASSERT +void err_report_internal_error +( + const char* description, + const char* module, + const char* file, + int line +) +{ + void *array[10]; + size_t size; + char **strings; + size_t i; + + fprintf(stderr, ERR_IE_TEXT, module, file, line, description, "\n"); + + size = backtrace (array, 10); + strings = backtrace_symbols (array, size); + + fprintf (stderr, "Obtained %zd stack frames.\n", size); + + for (i = 0; i < size; i++) + { + fprintf (stderr, "%s\n", strings[i]); + } + + free (strings); +} +#endif |