;------------------------------------------------------------------------------- ; ; Unit Name : l_HndlMissData.pro ; ; Purpose : Fill in missing packets with a fill value of NO_DATA. ; ; ; Development History: ; Author Date Build Description of Change ; -------------- --------- -------- ------------------------- ; ELH v1.0 Original implementation ; ELH v1.0.7 Added interpolation of UDF ; times that are missing ; ELH 11/21/01 v1.0.7 Fixed conflicting assignment ; of UDF time structure to ; array structure from ConvSec2Arr ; ELH 01/27/2004 v1.0.11 fixed if begin/end fill conditions ; . arithmetic calcuations were ; incorrectly done in condition ; Added SyncData2Time ; ELH 12/2006 Changed floor to round in calculating ; missing spins ; ; File Revision Number: %I% ; File Last Modified : %E% %T% ; ;------------------------------------------------------------------------------- ;------------------------------------------------------------------------------- ; Procedure: l_HndlMissData ; ; Description: Determines data gaps and fills with NO_DATA value. ; ; Return Value: type = ; Value Description ; ------------------------- ------------------------------ ; None ; ; Argument List: ; Name Type Use Description ; ----------------- ------ --- --------------------------- ; orig_tme double[] I original data time in seconds ; filled_time double[] O interpolated times for data gaps ; nspin long I total number of spins of data ; orig_data double[] I original data, no data gaps filled ; filled_data double[] O data with data gaps filled ; data_format long I a one dim array [spins*spin sector] or ; a two dim array [spins, spin sector] ; center_sector long I trace element sector location ; l_setup struct I generic setup parameters ; adjFlag long I flag to indicate if data needs to ; call the algorithm to wrap data ; (adjust data to zero degree spin ; angle) ; l_oa struct * I OA data parameters ; orig_start_udf_time struct [] I start UDF time, non-filled ; filled_start_udf_time struct [] O start UDF time, filled ; extra_data float [] I extra data spins ; spin_ang_zero_bin long I sector location to spin angle zero ; ; ; External Variables: ; Source Name Type Use Description ; --------------- --------- ---- ---- -------------------- ; l_LenaPlot.pro ; PLOTDEFS MAX_PLOTS int G max # plots per page ; MAX_WEB_SPINS int G max # spins per plot ; when using web interface ; NO_CNTS double G no counts indicator ; NO_DATA double G no data indicator ; ; ; Development History: ; Author Date Build Description of Change ; -------------- --------- -------- ------------------------- ; E. Lee 5/00 v1.0 Original implementation ; E. Lee 8/01 v1.0.7 Interpolated UDF times ; E. Lee 5/04 v1.0.11 Added SyncData2Time ;------------------------------------------------------------------------------- PRO l_HndlMissData, orig_tme, filled_time, nspin, orig_data, filled_data, $ data_format, center_sector, l_setup, adjFlag, l_oa, $ orig_start_udf_time, filled_start_udf_time, extra_data, spin_ang_zero_bin COMMON PLOTDEFS, MAX_PLOTS, MAX_WEB_SPINS, NO_CNTS, NO_DATA ;openw, ofd, 'UDFTimes.txt', /get_lun ;nitems = n_elements (orig_start_udf_time) - 1L ;for i = 0, nitems do begin ;printf, ofd, 'orig udf data = ', orig_start_udf_time[i] ;endfor ;close, ofd ;free_lun, ofd ustart_time = l_setup.l_start_dtime ustop_time = l_setup.l_stop_dtime year = l_setup.l_start_dtime[0] ;------------------------------------------------------------ ; the fill value used for data gaps ;------------------------------------------------------------ dg_value = NO_DATA ;------------------------------------------------------------ ; the spin interval of 2 minutes per spin ;------------------------------------------------------------ spin_interval = 120.0 ;------------------------------------------------------------ ; the spin interval of 2 minutes + threshold ;------------------------------------------------------------ spin_threshold = 120.0 + 10.0 ;------------------------------------------------------------ ; get the user's start and stop time requested ; get the data start and stop time for the requested user ; time ;------------------------------------------------------------ ;indx = where (orig_tme gt 0.0, wcnt) ;if (wcnt gt 0) then begin ; start_time = min(orig_tme[indx]) ;endif else begin ; start_time = min(orig_tme) ;endelse ;stop_time = max (orig_tme) tmp = intarr (6) tmp [0] = orig_start_udf_time[0].year tmp [1] = orig_start_udf_time[0].doy tmp [2] = orig_start_udf_time[0].hour tmp [3] = orig_start_udf_time[0].min tmp [4] = orig_start_udf_time[0].sec tmp [5] = orig_start_udf_time[0].msec start_time = ConvArrTimeToSecs (tmp) nitems = n_elements (orig_start_udf_time) - 1L tmp [0] = orig_start_udf_time[nitems].year tmp [1] = orig_start_udf_time[nitems].doy tmp [2] = orig_start_udf_time[nitems].hour tmp [3] = orig_start_udf_time[nitems].min tmp [4] = orig_start_udf_time[nitems].sec tmp [5] = orig_start_udf_time[nitems].msec stop_time = ConvArrTimeToSecs (tmp) user_st_time = ConvArrTimeToSecs (ustart_time) user_sp_time = ConvArrTimeToSecs (ustop_time) ;------------------------------------------------------------ ; set the number of spin sectors ;------------------------------------------------------------ sector_format = data_format spin_sector = sector_format time_sector_format = sector_format time_diff = stop_time - start_time spins = round (double(time_diff)/spin_interval) ;------------------------------------------------------------ ; determine the number of missing packets within the user ; request time ;------------------------------------------------------------ total_missing = 0L for ii = 1L, nspin-1L do begin spin_diff = orig_tme[ii*time_sector_format] - $ orig_tme[(ii*time_sector_format)-(time_sector_format)] ;if (spin_diff gt spin_threshold) then begin ;print, 'spin_diff = ', orig_tme[ii*time_sector_format], $ ; orig_tme[(ii*time_sector_format)-(time_sector_format)], spin_diff ;endif if spin_diff gt spin_threshold then begin ;spin_diff = spin_diff - spin_interval missing = round (spin_diff/spin_interval) - 1L ;print, 'missing = ', missing total_missing = total_missing + missing endif endfor ;print, 'total_missing = ', total_missing ;------------------------------------------------------------ ; set up the correct array size filling it with gray for ; no data ;------------------------------------------------------------ ele = nspin + total_missing adj_spin_avg = fltarr (sector_format, nspin) spin_tme = dblarr (time_sector_format, ele) udf_tme = replicate ({UDF_TIMES_STRUCT}, ele) adj_spin_avg[*,*] = dg_value ;------------------------------------------------------------ ; if data is available fill new array with the original ; data else if there are missing spins, fill with fill ; data. Extrapolate the time. ;------------------------------------------------------------ adj_spin_avg[*,0] = orig_data[0:(sector_format-1)] ;---------------------------------------------------------- ; reformat data array to 2 dim array ;---------------------------------------------------------- idx = 1L for ii = 1L, nspin-1L do begin adj_spin_avg[*,idx] = $ orig_data[(ii*sector_format):((ii*sector_format)+(sector_format-1L))] idx = idx + 1L endfor ;---------------------------------------------------------- ; adjust the bins so that zero spin angle is either the ; earth or the sun ; this MUST be done before missing data is filled ;---------------------------------------------------------- flag = STRUPCASE (adjFlag) if (adjFlag eq 'YES') then begin if ((l_setup.l_angle_zero eq 'S') OR (l_setup.l_angle_zero eq 'M') OR $ (l_setup.l_angle_zero eq 'E')) then begin iarr_times = orig_tme iarr = adj_spin_avg ;----------------------------------------------- ; roll data ;----------------------------------------------- l_Adj2SunOrEarth, center_sector, iarr, iarr_times, $ oarr, sector_format, l_setup.l_angle_zero, $ l_oa, extra_data, spin_ang_zero_bin ;l_oa, new_extra_data, spin_ang_zero_bin adj_spin_avg = oarr endif endif if ((l_setup.l_angle_zero eq 'S') OR (l_setup.l_angle_zero eq 'M')) $ AND (sector_format ne 1L) AND (adjFlag eq 'YES') then begin sector_format = 360L endif spin_avg = fltarr (sector_format, ele) spin_avg[*,*] = dg_value if (sector_format eq 1L) then begin begin_time = orig_tme[0] next_time = orig_tme[1] endif else begin begin_time = orig_tme[0] next_time = orig_tme[45] endelse ;---------------------------------------------------------- ; if a year cross over, the actual start time which is the ; data at the end of the year (12/31) is a time larger then ; the new year time (01/01). This results in an invalid ; plot. Interpolation of the beginning time is needed. ; **** NEED TO HANDLE THIS FOR MORE THAN 1 SPIN ***** ;---------------------------------------------------------- if (begin_time gt next_time) then begin spin_tme[*,0] = next_time - spin_interval udf_tme[0] = ConvSec2Struct((next_time - spin_interval), year) endif else begin spin_tme[*,0] = orig_tme [0:(time_sector_format-1)] udf_tme[0] = orig_start_udf_time[0] endelse spin_avg[*,0] = adj_spin_avg[*,0] idx = 1 ;print, 'nspin = ', nspin ;help, orig_tme ;help, udf_tme for ii = 1L, nspin-1L do begin ;----------------------------------- ; determine the time difference between ; current spin and previous spin ;----------------------------------- spin_diff = orig_tme[ii*time_sector_format] - $ orig_tme[(ii*time_sector_format)-(time_sector_format)] ;----------------------------------- ; a data gap is found ;----------------------------------- ;print, 'spin_diff = ', spin_diff, orig_tme[(ii*time_sector_format)-time_sector_format], orig_tme[ii*time_sector_format] if spin_diff gt spin_threshold then begin ;----------------------------------- ; determine the number of missing spins ;----------------------------------- ;spin_diff = spin_diff - spin_interval missing = round(spin_diff/spin_interval) - 1L ;print, 'missing = ', missing idx = idx + missing ;----------------------------------- ; fill in the spin time from original to ; correct spin location ;----------------------------------- spin_avg[*,idx] = adj_spin_avg[*,ii] spin_tme[*,idx] = $ orig_tme[(ii*time_sector_format):((ii*time_sector_format)+$ (time_sector_format-1))] udf_tme[idx] = orig_start_udf_time[ii] ;print, 'fill udf time = ', udf_tme[idx] ;----------------------------------- ; skip the number of missing spins and ; store data in correct spin location ;----------------------------------- ;spin_avg[*,idx] = adj_spin_avg[*,ii] ;----------------------------------- ; fill in the missing spin times ; interpolating missing times ;----------------------------------- if (missing le 0) then missing = 1L ;if (missing gt 1) then begin mt_adj = spin_diff/(missing+1) ;endif ;endif else begin ; mt_adj = spin_diff/missing ;endelse if (missing gt 0) then begin miss_idx = idx - missing miss_tme = orig_tme [((ii-1)*time_sector_format):$ (((ii-1)*time_sector_format)+ $ (time_sector_format-1))] + mt_adj for jj=0L, missing-1L do begin spin_tme[*,miss_idx] = miss_tme udf_tme[miss_idx] = ConvSec2Struct(miss_tme[0], year) spin_avg[*,miss_idx] = NO_DATA miss_tme = miss_tme + mt_adj miss_idx = miss_idx + 1 endfor ;idx = miss_idx endif endif else begin ;----------------------------------- ; no missing data, fill with original ;----------------------------------- spin_avg[*,idx] = adj_spin_avg[*,ii] ;help, orig_tme ;help, spin_tme spin_tme[*,idx] = $ orig_tme[(ii*time_sector_format):((ii*time_sector_format)+$ (time_sector_format-1))] udf_tme[idx] = orig_start_udf_time[ii] ;print, 'udf_tme = ', udf_tme[idx] endelse idx = idx + 1 endfor ;---------------------------------------------------------- ; change the spin_tme format back to a 1 dimensional array ;---------------------------------------------------------- index = reform (spin_tme, n_elements(spin_tme), /OVERWRITE) ;------------------------------------------------------------ ; if there is data missing for both the beginning and the end ; of the user requested time ; ***** NEED TO CORRECT TIME LABELING TO REFLECT ACTUAL DATA ; AVAILABLE AND NOT USER REQUESTED TIME ********** ;------------------------------------------------------------ missing_data = 0 diff_st_time = start_time - user_st_time diff_sp_time = user_sp_time - stop_time ;print, 'start_time = ', start_time ;print, 'user_st_time = ', user_st_time ;print, 'user_sp_time = ', user_sp_time ;print, 'stop_time = ', stop_time ;print, 'diff_st_time = ', diff_st_time ;print, 'diff_sp_time = ', diff_sp_time if (start_time gt user_st_time) and $ (user_sp_time gt stop_time) then begin if (diff_st_time ge spin_threshold) or $ (diff_sp_time ge spin_threshold) then begin ;print, 'FILL BEGIN AND END' missing_data = 1 ;------------------------------------------------ ; calculate # of missing spin packets ;------------------------------------------------ b_miss_pkts = ceil((start_time - user_st_time)/spin_interval) - 1L e_miss_pkts = ceil((user_sp_time - stop_time)/spin_interval) - 1L total_mps = b_miss_pkts + e_miss_pkts + ele ;------------------------------------------------ ; fill with fill data ;------------------------------------------------ new_data = fltarr (sector_format, total_mps) new_data[*,*] = dg_value ;------------------------------------------------ ; interpolate time ;------------------------------------------------ tele = n_elements (spin_tme) tot_time_ele = (time_sector_format*(b_miss_pkts+ $ e_miss_pkts))+tele new_time = fltarr (tot_time_ele) new_time[*] = dg_value new_udf_time = replicate ({UDF_TIMES_STRUCT}, total_mps) ;------------------------------------------------ ; interpolate times at beginning ;------------------------------------------------ interp_time = spin_tme[0] beg_start = long(b_miss_pkts) - 1L for ii = beg_start, 0L, -1 do begin start_indx = ii * time_sector_format stop_indx = ii * time_sector_format + time_sector_format - 1L interp_time = interp_time - 120. new_time[start_indx:stop_indx] = interp_time new_udf_time[ii] = ConvSec2Struct(interp_time, year) endfor ;------------------------------------------------ ; fill with read data in between ;------------------------------------------------ new_data[*,b_miss_pkts:((b_miss_pkts+ele)-1)] = spin_avg[*,*] new_udf_time[b_miss_pkts:((b_miss_pkts+ele)-1)] = udf_tme[*] fill_end = tot_time_ele - e_miss_pkts*time_sector_format new_time[b_miss_pkts*time_sector_format:fill_end-1] = $ spin_tme[*] ;------------------------------------------------ ; interpolate times at end ;------------------------------------------------ nspin_times = n_elements(spin_tme) - 1L interp_time = spin_tme[nspin_times] end_start = long(total_mps - e_miss_pkts) - 1L ;******** add + 1L , chnged 7/16/04 ******* for jj = end_start, total_mps - 1L do begin start_indx = jj * time_sector_format stop_indx = jj * time_sector_format + time_sector_format - 1L new_time[start_indx:stop_indx] = interp_time new_udf_time[jj] = ConvSec2Struct(interp_time, year) interp_time = interp_time + 120. endfor endif endif else begin ;------------------------------------------------------------ ; fill the beginning of the user data set ; there is no data for the time the user requested ;------------------------------------------------------------ if (start_time gt user_st_time) then begin if (diff_st_time ge spin_threshold) then begin ;print, 'FILL BEGIN ONLY' missing_data = 1 miss_pkts = ceil((start_time - user_st_time)/spin_interval) - 1L mpkts = miss_pkts + ele new_data = fltarr (sector_format, mpkts) new_data[*,*] = dg_value new_data[*,miss_pkts:mpkts-1] = spin_avg[*,*] tele = n_elements (spin_tme) tot_time_ele = (time_sector_format*miss_pkts)+tele new_time = fltarr (tot_time_ele) new_time[*] = dg_value new_udf_time = replicate ({UDF_TIMES_STRUCT}, mpkts) ;------------------------------------------------ ; interpolate times at beginning ;------------------------------------------------ interp_time = spin_tme[0] for ii = long(miss_pkts), 0L, -1L do begin start_indx = ii * time_sector_format stop_indx = ii * time_sector_format + time_sector_format - 1L interp_time = interp_time - 120. new_time[start_indx:stop_indx] = interp_time new_udf_time[ii] = ConvSec2Struct(interp_time, year) endfor ;------------------------------------------------ ; fill the rest of the data in ;------------------------------------------------ new_time[miss_pkts*time_sector_format:mpkts*time_sector_format-1] = $ spin_tme[*] new_udf_time[miss_pkts:mpkts-1] = udf_tme[*] endif endif else begin ;------------------------------------------------------------ ; fill the end of the user data set ; there is no data for the time the user requested ;------------------------------------------------------------ if (stop_time lt user_sp_time) then begin if (diff_sp_time ge spin_threshold) then begin ;print, 'FILL END ONLY' missing_data = 1 miss_pkts = ceil((user_sp_time - stop_time)/spin_interval) - 1 mpkts = miss_pkts + ele tele = n_elements (spin_tme) new_data = fltarr (sector_format, mpkts) new_data[*,*] = dg_value new_udf_time = replicate ({UDF_TIMES_STRUCT}, mpkts) tot_time_ele = (time_sector_format*miss_pkts)+tele new_time = fltarr (tot_time_ele) new_time[*] = dg_value ;------------------------------------------------ ; fill in the real data ;------------------------------------------------ new_data[*,0:ele-1] = spin_avg[*,*] new_udf_time[0:ele-1] = udf_tme[*] new_time [0:tele-1] = spin_tme[*] ;------------------------------------------------ ; interpolate times at end ;------------------------------------------------ interp_time = spin_tme[tele - 1L] end_start = long(mpkts - miss_pkts) - 1L ;******* added -1L, changed 7/16/04 for jj = end_start, mpkts - 1L do begin start_indx = jj * time_sector_format stop_indx = jj * time_sector_format + time_sector_format - 1L new_time[start_indx:stop_indx] = interp_time new_udf_time[jj] = ConvSec2Struct(interp_time, year) interp_time = interp_time + 120. endfor endif endif ;--- stop time is less than user stop time endelse ;--- end if only fill end --- endelse ;--- end if not both begin and end fill --- if (missing_data) then begin SyncData2Time, new_data, new_time, new_udf_time, l_setup filled_data = new_data filled_time = new_time filled_start_udf_time = new_udf_time endif else begin SyncData2Time, spin_avg, spin_tme, udf_tme, l_setup filled_data = spin_avg filled_time = spin_tme filled_start_udf_time = udf_tme endelse RETURN END