SMA Data File Format
 
When taking interferometer data, the SMA records data in a set of files all contained in
a single directory.   This document describes the contents of those files.
It is important to remember these files were written by a big endian processor!
PowerPCs are big endian (by default).   If you directly process these data on a little
endian machine, such as an x86 linux box, you will need to do some byte swapping
to account for this.

The files are listed here alphabetically, by file name.

antennas
This file contains the X, Y and Z coordinates of each antenna, in meters.
The coordinate system is defined, with these letters, in Thompson, Moran and Swenson's
text "Interferometry and Synthesis in Radio Astronomy".

This file is written to once, immediately after the data directory is opened.   If the
antenna coordinates are updated during a track, those new coordinates will NOT
appear in this file.

bl_read
This file contains the baseline information.   There is one fixed-length
record written per receiver per sideband per baseline per scan.

The order of the loops writing out these records is:
receiver
    sideband
        polarization
            baseline

so baseline ID is the most rapidly varying index.   For most tracks
receiver and polarization only have a single value, and the loops
collapse to:
sideband
    baseline

Each record contains:

typedef struct __attribute__((packed)) blhDef {
  int 	blhid     ; /*	proj. baseline id #       */
                    /* This is a unique identifier*/
                    /* for the baseline           */
  int 	inhid     ; /*	integration id #          */
  short	isb       ; /*	sideband int code         */
  short	ipol      ; /*	polarization int code     */
                    /*  0 = Unknown               */
                    /*  1 = RR                    */
                    /*  2 = RL                    */
                    /*  3 = LR                    */
                    /*  4 = LL                    */
  float	pa        ; /*	polarization angle        */
                    /*  UNUSED - set to 0.0       */
  short	iaq       ; /*	amp qual int code         */
                    /*  This flag is meaningful   */
                    /*  only in Dual Rx Polar     */
                    /*  mode, when it is used to  */
                    /*  specify the receiver on   */
                    /*  used on the lower number- */
                    /*  receiver on the baseline. */
                    /*  0 = 345 Rx                */
                    /*  1 = 400 Rx                */
                    /*  anything else is bad data.*/
  short	ibq       ; /*	baseline qual int code    */
                    /*  In full polarization mode */
                    /*  ibq encodes the high ant  */
                    /*  number Rx, and ibq encodes*/
                    /*  the higher ant number Rx. */
                    /*  0 = 345 Rx                */
                    /*  1 = 400 Rx                */
  short	icq       ; /*	coherence qual int code   */
  short	ioq       ; /*	offset qual int code      */
                    /*  = 1 for no offsets        */
                    /*  = 0 otherwise             */
  short	irec      ; /*	receiver int code         */
                    /*  0 = 230                   */
                    /*  1 = 345                   */
                    /*  2 = 400                   */
                    /*  3 = 600                   */
                    /* -1 = Rx unknown            */
  short	iifc      ; /*	if channel int code       */
                    /*  UNUSED, set to 0          */
  float	u         ; /*	u coord for bsl (klambda) */
  float	v         ; /*	v coord. for bsl          */
  float	w         ; /*	w coord. for bsl          */
  float	prbl      ; /*	projected baseline        */
  float	angres    ; /*	baseline resolution       */
                    /*  UNUSED, set to            */
                    /*  16 + baseline int code    */
                    /*  For no reason at all      */
  float	vis       ; /*	fractional visibility     */
                    /*  UNUSED, set to            */
                    /*  17 + baseline int code    */
                    /*  For no reason at all      */
  float	coh       ; /*	coherence estimate        */
  float	sigcoh    ; /*	sigma on coh.             */
                    /*  UNUSED, set to 0.0        */
  float	csnr      ; /*	cont. SNR                 */
                    /*  UNUSED, set to            */
                    /*  20 + baseline int code    */
                    /*  For no reason at all      */
  float	vflux     ; /*	visibility flux           */
                    /*  UNUSED, set to            */
                    /*  21 + baseline int code    */
                    /*  For no reason at all      */
  float	cnoise    ; /*	continuum noise           */
                    /*  UNUSED, set to            */
                    /*  22 + baseline int code    */
                    /*  For no reason at all      */
  double avedhrs  ; /*	hrs offset from ref-time  */
  float	ampave    ; /*	ave continuum amp         */
  float	phaave    ; /*	ave continuum phase       */
  float	tpvar     ; /*	total power stability     */
                    /*  UNUSED, set to            */
                    /*  26 + baseline int code    */
                    /*  For no reason at all      */
  int 	blsid     ; /*	physical baseline id #    */
  short	itel1     ; /*	tel 1 int code            */
  short	itel2     ; /*	tel 2 int code            */
  short	iblcd     ; /*	baseline int code         */
  float	ble       ; /*	bsl east vector           */
  float	bln       ; /*	bsl north vector          */
  float	blu       ; /*	bsl up vector             */
  int 	soid      ; /*	bsl soln id #             */
                    /*  UNUSED, set to 0          */
} blhDef;
/* the size of blhDef is 118 bytes */

codes_read
The various header blocks that comprise the header information for
each scan generally do not contain string values for such things
as the name of the source being observed.   Instead, this file
contains all such strings, along with integer index values.
The header blocks then contain the integer index values, rather than
the strings themselves.   As soon as the data files are created,
certain strings, such ans "l" and "u" used to name sidebands, are
written to this file.   After that, new strings are added on an 
as-needed basis, for example, when a new source is observed.

The following codes are always written to the file, regardless of
the array's configuration:

v_name       icode        code (string)   ncode     Comments

"tq"           0         "v01"              1       OVRO intended to use this
                                                    to specify the correlator
                                                    configuration.   We do not
                                                    use this at all.

"vctype"       0         "vlsr"             1       Velocity definition
                                                    We use this value, always.
               1          "cz"              1       But these other codes
               2          "vhel"            1       exist.
               3          "pla"             1

"sb"           0          "l"               1       Sideband designator - LSB
               1          "u"               1       USB

"aq"           0          " "               1       These are amplitude qualifiers
               1          "1"               1       and we don't use them at all
               2          "2"               1

"bq"           0          " "               1       Baseline quaifier - we don't
               1          "b"               1       use this at all.

"cq"           0          " "               1       Coherence qualifier - we don't
               1          "c"               1       use this at all.


"oq"           0          " "               1       Offset qualifier - we don't use 
               1          "o"               1       this at all.

"rec"          0          "230"             1       Receiver code - 230 insert
               1          "345"             1       345 Rx
               2          "400"             1       400 Rx
               3          "690"             1       690 Rx

"ifc"          0          "1"               1       IF channel - we don't use this
               1          "2"               1

"tel1"         0          "0"               1       First antenna on a baseline
              ...         ...              ...
              10         "10"               1       Antenna 10 = CSO

"tel2"         0          "0"               1       Second antenna on a baseline
              ...         ...              ...
              10         "10"               1       Antenna 10 = CSO

"gq"           0          " "               1       Gain qualifier
               1          "g"               1

"pq"           0          " "               1       Passband qualifier
               1          "p"

"band"         0          "c1"              1       Band qualifier - only this one is
                                                    always written.   Bands Sxx are written
                                                    later, if needed.

"pstate"       0          "0"               1       OVRO may have intended to use this for
                                                    some kind of polarization information.
                                                    We don't use it.

"taper"        0          "u"               1       Uniform taper?   We don't use this.
               1          "h"               1       Or this either.

"trans"        0      "unspecified"         1       We don't use this.

"pos"          0      "unspecified"         1       We don't use this.

"offtype"      0      "ra-dec"              1       Offset type, we don't use this.
               1      "az-el"               1

The following codes are written when observing in dual Rx polarization mode:

"pol"          0          "Unknown"         1
               1          "RR"              1
               2          "RL"              1
               3          "LR"              1
               4          "LL"              1
               5          "LH"              1
               6          "LV"              1
               7          "RH"              1
               8          "RV"              1
               9          "HR"              1
              10          "HL"              1
              11          "HH"              1
              12          "HV"              1
              13          "VR"              1
              14          "VL"              1
              15          "VH"              1
              16          "VV"              1

The followin codes are written when we are not in dual Rx polarization mode:

"pol"          0          "hh"              1
               1          "vv"              1
               2          "hv"              1
               3          "vh"              1

Additional codes values are written as needed:

There is a "ref_time" code written each UT day.   Most files will only have one
of these, becasue the UT day rolls over at 2:00 PM HST, and we are not usually
observing at that time of the day.  The format for the string associated with
this type of record is "Mon dd yyyy".   icode counts from 1 to the total number
of "ref_time" records.

Each baseline has a "blcd" code record written.   The icode value counts
sequentially through the baselines.   The string holds "A-B" where A and B
are antenna numbers.

Every spectral chunk has a "band" code written.   icode is 0 for the continuum
band (always written) and icode counts the existing chunks sequentially after
that.   If no chunks are discarded, icode is equal to the chunk number.  The
string format is "sxx" where xx is the chunk number, always written as a two
digit integer, perhaps with a leading zero.

Each source has a "source" record written.   The icode value counts from 1 to
the total number of sources.   The string part of the record contains the
ASCII text name of the source, truncated if necessary after 25 characters.

There is a "ut" record written for each scan.   icode for this type of record
equals the scan number.   The string has "Mon dd yyy hh:mm:ss.000{AM/PM}".
The is an "ra" and a "dec" record written each scan.   In both cases icode
holds the scan number.   The "ra" string format is "hh:mm:ss.ss" and the "dec"
string format is "{+/-}dd:mm:ss.s". A "vrad" record is written for each
scan, and the string holds the radial velocity in m/sec.
 
The file consists of fixed length record with the format

typedef struct __attribute__((packed)) codehDef {
  
  char v_name[12];/* label 			*/
  short  icode	; /* index for a code word	*/
  char code[26]	; /* the code word		*/
  short ncode	; /* # chars in code word	*/
  
} codehDef;
/* the size of codeh is 42 bytes */

codeVersion
This file contains the dates at which certain critical pieces of the online
software were compiled.   It is intended to help track bug fixes that need to be applied
to certain data sets.   It is written to once, immediately after the
data directory is created.  The format is program name, followed by compilation
date in the form YYMMDD.   Here's an example:

dDSServer 091014
int_server 091113
dataCatcher 091113
setLO 090819
eng_read This file contains fixed-length records. One record is written for each antenna (not including the CSO and JCMT) after each scan completes. The format is typedef struct __attribute__((packed)) antEngDef { int antennaNumber; int padNumber; int antennaStatus; /* Antenna is ON or OFF LINE */ int trackStatus; /* Track is running or not */ int commStatus; /* Data for this integration is valid or not */ int inhid ; /* integration id # */ int ints ; /* integration # */ double dhrs ; /* hrs from ref_time */ double ha ; /* hour angle */ double lst ; /* lst */ double pmdaz ; /* pointing model correction */ double pmdel ; double tiltx ; double tilty ; double actual_az ; double actual_el ; double azoff ; double eloff ; double az_tracking_error ; double el_tracking_error ; double refraction ; double chopper_x ; double chopper_y ; double chopper_z ; double chopper_angle ; double tsys ; double tsys_rx2 ; double ambient_load_temperature ; } antEngDef; /* The size of antEngDef is 196 bytes */ in_read This file contains scan header information. It consists of fixed-length records. There is one record written for each scan. The format is typedef struct __attribute__((packed)) inhDef { int conid ; /* config id # */ /* UNUSED - set to 0 */ short icocd ; /* config int code */ /* UNUSED - set to 0 */ int traid ; /* track id # */ /* Set to the Project ID */ int inhid ; /* integration id # */ int ints ; /* integration # */ /* In reality, same as inhid */ short itq ; /* tuning int code */ /* UNUSED - set to 0 */ float az ; /* azimuth (degrees) */ float el ; /* elevation (degrees) */ float ha ; /* hour angle (hours) */ short iut ; /* ut int code */ short iref_time ; /* ref_time int code */ double dhrs ; /* hrs from ref_time */ float vc ; /* vcorr for vctype */ short ivctype ; /* vctype int code */ /* UNUSED - set to 0 */ double sx ; /* x vec. for bsl. */ double sy ; /* y vec. for bsl. */ double sz ; /* z vec. for bsl. */ float rinteg ; /* actual int time */ int proid ; /* project id # */ int souid ; /* source id # */ short isource ; /* source int code */ short ipos ; /* position int code */ /* UNUSED - set to 0 */ float offx ; /* offset in x */ /* used for RA offset */ float offy ; /* offset in y */ /* used for Dec offset */ short iofftype ; /* offset int code */ /* UNUSED - set to 0 */ short ira ; /* ra int code */ short idec ; /* dec int code */ double rar ; /* ra (radians) */ double decr ; /* declination (radians) */ float epoch ; /* epoch for coord. */ /* always set to 2000.0 */ float sflux ; /* flux */ /* UNUSED - set to 0.0 */ float size ; /* source size (arcsec) */ } inhDef; /* The size of inhDef is 132 bytes */ modeInfo is an ASCII file containing two integers on a single line. The first integer specifies the number of active receivers. It will always be "1" or "2". The second integer specifies the bandwidth, in GHz, and it will always be "2" or "4". This file is written to once, immediately after the data directory is created. plot_me_5_rx{n} There is one of these files for each active receiver. This file is not used by mir or miriad at all - it is used by corrPlotter to produce the "mir display". It is a text file, with one (very long) line per scan. Each line contains: Token Type Description 1 string Source name 2 float UT at mid-scan in hours 3 float Hour Angle at mid-scan in hours 4 float Declination at mid-scan in radians 5 float LO frequency in GHz 6 int Source type 7 int First antenna on baseline 8 int Second antenna on baseline 9 int Flag (0 = bad, 1 = good) 10 float Pseudo-continuum amplitude 11 float Pseudo-continuum phase 12 float Coherence Items 7->12 are repeated for each sideband and each baseline. All lower sideband baselines are prined first, followed by the upper sidebands. So the order will typically be: 1-2 LSB 1-3 LSB ... 7-8 LSB 1-2 USB 1-3 USB ... 7-8 USB At the very end of the line, a code is written for the antenna specifying the polarization of each antenna for that scan. The format is three bits per antenna, with antenna 1 occupying bits 3->5 (counting from bit 0), antenna 2 is specified in bits 6->8, etc. Treating these three bits as an integer, R = 1, L = 2, V = 3 and H = 4. In nonpolarization observations, this whole integer is set to 0. projectInfo This file simply contains the name of the project PI. It exists only to help track the data through our retrieval system. It is writen to once immediately after the data directory is created. sch_read This is the file that actually contains the visibility data. Unlike most of the files containing binary data, the records in this file are of wildly varying length, depending upon the resolution of the spectrometer band. There is one record written for each scan, and it contains the packed visibility data for all chunks and the pseudocontinuum channels. Here's the format: typedef struct __attribute__((packed)) schDef { int inhid ; /* integration id # */ char form[4] ; /* indicates the format of the data, always I2-C */ int nbyt ; /* the number of bytes in one integration of data */ int nbyt_pack ; /* ? It's not clear why this exists. It is always equal to nbyt */ short *packdata; /* integer array containing the data in the format above */ } schDef; The spectral bands are stored in packdata as follows: In double bandwidth mode, the nested loops are sideband polarization baseline spectral band in all other observing modes the nested loops are receiver sideband polarization baseline spectral band so spectral band is always the most rapidly varying index The pseudocontinuum band is the first band stored, followed by the spectral bands (usually 24 or 48 of them). SMAshLog_{YYYYMMDD_HHMMSS} This file contains all of the SMAsh commands which were issued while the array was using this directory to store data. In some cases there may be more than one of these files in the directory. sp_read This file contains the header information for each individual spectral band of data. A spectral band can be either a correlator chunk or the pseudo-continuum channel. So, for example, if your file contains 2814 scans, and you had two receivers active and all 24 correlator chunks and all 28 baselines, the total number of records in this file would be 2814 * 28 * (24+1) * 2 * 2 = 7879200. The loop order for writing this file is scan receiver sideband polarization baseline band Here's what each record contains: typedef struct __attribute__((packed)) sphDef { int sphid ; /* spectrum id # */ int blhid ; /* proj. baseline id # */ int inhid ; /* integration id # */ short igq ; /* gain qual int code */ short ipq ; /* passband qual int code */ short iband ; /* spectral band int code */ short ipstate ; /* pol state int code */ float tau0 ; /* Tau at 225 GHz from the */ /* CSO tau-o-meter */ double vel ; /* velocity (vctype) (km/s) */ float vres ; /* velocity res. */ short ivtype ; /* velocity type int code */ /* UNUSED - set to 0 */ double fsky ; /* center sky freq. */ float fres ; /* frequency res. (MHz) */ float tssb ; /* tsys (ssb) */ float integ ; /* integration time */ float wt ; /* weight (sec/tssb**2) */ /* set to -(sec/tssb**2) if */ /* the data is flagged bad. */ short itaper ; /* spectrum taper int code */ /* UNUSED - set to 0 */ float snoise ; /* th. noise in spectrum */ /* SET TO CATALOG VELOCITY! */ short nch ; /* # channels in spectrum */ short nrec ; /* # of records w/i inh# */ /* Always set to 1 */ int dataoff ; /* byte offset for data */ int linid ; /* spectral line id # */ /* UNUSED - set to 0 */ short itrans ; /* transition int code */ /* UNUSED - set to 0 */ double rfreq ; /* rest frequency (GHz) */ short pasid ; /* passband fn id # */ /* UNUSED - set to 0 */ short gaiidamp ; /* Correlator block number */ /* 0 for c1 chunk */ short gaiidpha ; /* Correlator chunk number */ /* NOT the sxx chunk name - */ /* this value is always 1->4, */ /* except for the c1 chunk */ short flcid ; /* flux cal id # */ /* UNUSED - set to 0 */ short atmid ; /* atmospheric coh id # */ /* UNUSED - set to 0 */ } sphDef; /* the size of sphDef is 100 bytes */ tsys_read contains antenna-based Tsys values for both sidebands of each chunk. One record is written to this file for each scan. The file is packed big-endian binary, with records of the form typedef struct __attribute__((packed)) tsysRecordDef { int inhid ; /* integration id #, following the definition */ /* in the standard mir format (inh_def). */ double dhrs ; /* Hours from ref time, following the definition */ /* in the standard mir format (inh_def). */ short ipol ; /* polarization int code, following the definition */ /* in the standard mir format (blh_def). */ short itel ; /* antenna int code, following the definition */ /* in the standard mir format (blh_def). */ float tsys[2][48] ; /* Tsys following the definition of tssb in the */ /* standard mir (sph_def) but with an antenna-based */ /* value instead of a baseline-based mean. */ /* The first index loops over sidebands (0 = LSB) */ /* and the second index loops over chunks. */ /* In 4 GHz mode, elements 0 -> 47 correspond to */ /* chunks s01 -> s47. In the 2 GHz modes, */ /* elements 0 -> 23 correspond to s01 -> s24 of the */ /* low frequency receiver, and elements 24 -> 47 */ /* contain Tsys for the high frequency receiver */ /* If one of the receivers is not in use, its Tsys */ /* values are filled with NaNs */ /* The stored Tsys is currently just twice the */ /* measured DSB Tsys, in order to simulate a SSB */ /* Tsys */ } tsysRecordDef; /* The size of tsysRecordDef is 400 bytes */ we_read This file contains antenna status flags, and weather information for each scan. It is a binary file. For the 11 element arrays, element 0 contains observatory-wide values (for example the temperature measured from the hangar weather station) and the other elemets contain the values measured at the individual antennas. If a particular antenna does not have hardware to measure a certain quantity, than -1.0 is written for the quantity's value at that antenna. One record is written to this file for each scan. typedef struct __attribute__((packed)) wehDef { /* Note the following arrays contain 11 elements. Element n is used for antenna n, except in the case of n=0, which stores the values from the observatory's weather station on the hangar. */ int scanNumber; /* The scan number for which this info applies */ int flags[11]; /* Flagging information from statusServer. */ /* Here are the flags which have been defined: */ #define SFLAG_UNLOCKED_PLL 0x00000001 #define SFLAG_CAL_VANE 0x00000002 #define SFLAG_BAD_SAMPLES 0x00000004 #define SFLAG_COORD_MISMATCH 0x00000008 #define SFLAG_DEWAR_WARM 0x00000010 #define SFLAG_DRIVES_OFF 0x00000020 #define SFLAG_FEED_MISMATCH 0x00000040 #define SFLAG_IRIG_TIME 0x00000080 #define SFLAG_M3_CLOSED 0x00000100 #define SFLAG_OPTICAL 0x00000200 #define SFLAG_SOURCE_MISMATCH 0x00000400 #define SFLAG_TRACK_STALE 0x00000800 #define SFLAG_WACKY_OFFSETS 0x00001000 #define SFLAG_CHOPPER_POS 0x00002000 #define SFLAG_AVE_TRACKING 0x00004000 #define SFLAG_PEAK_TRACKING 0x00008000 #define SFLAG_SOURCE_CHANGE 0x00010000 #define SFLAG_SHADOWING 0x00020000 #define SFLAG_OPERATOR 0x00040000 #define SFLAG_DRO_UNLOCKED 0x00080000 #define SFLAG_WAVEPLATE_MOVED 0x00100000 #define SFLAG_MISCELLANEOUS 0x00200000 float N[11]; /* The refractivity used for the atmospheric delay */ /* correction for each antenna. */ float Tamb[11]; /* The ambient temperature (C) used for each */ /* antenna's atmospheric delay correction. */ float pressure[11]; /* The atmospheric pressure, in mbar, used for each */ /* antenna's atmospheric delay correction. */ float humid[11]; /* The relative humidity, in percent, used for each */ /* antenna's atmospheric delay correction. */ float windSpeed[11]; /* Wind speed in m/sec at each antenna. -1.0 if */ /* no hardware exists to measure this */ float windDir[11]; /* Wind direction, in radians measured from north */ /* through east, for each antenna. -1.0 if no */ /* hardware exists. */ float h2o[11]; /* The boresite precipitable water vapor measured */ /* at each antenna. -1.0 if no hardware exists. */ } wehDef; /* The size of wehDef is 356 bytes */