continue improving automated reductions

mcoughlin · Apr 21, 2021 · c9f1922 · c9f1922
1 parent 1231a98
commit c9f1922
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 121 deletions.
diff --git a/bin/kp84_photometric_reduction b/bin/kp84_photometric_reduction
@@ -77,8 +77,6 @@ def parse_commandline():
 
     parser.add_option("--doDynamicAperture",  action="store_true", default=False)
 
-    parser.add_option("--doCheckCatalog",  action="store_true", default=False)
-
     # Transient options
     parser.add_option("--doTransient",  action="store_true", default=False)
     parser.add_option("--doSubtraction",  action="store_true", default=False, help="subtract source extractor background from science image before force photometry")
@@ -120,7 +118,6 @@ if opts.doTransient:
     nimages = np.inf
 doOffTune = opts.doOffTune
 doOffRefit = opts.doOffRefit
-doCheckCatalog = opts.doCheckCatalog
 
 aper_size = opts.aper_size
 sky_in = opts.sky_in
@@ -211,51 +208,29 @@ if not doSkipRegis:
         if not os.path.isdir(path_out_dir):
             os.makedirs(path_out_dir)
         print("%d/%d: %s"%(i+1, nfiles, fitsfileSplit))
-        if fitsfileSplit[14:] in xyfiles_:
-            index = np.where(xyfiles_ == fitsfileSplit[14:])[0][0]
-            xyframe = int(xyexts_[index])
-            x = int(xs_[index])
-            y = int(ys_[index])
-            update_wcsstatus(fitsfile, xyframe)
-            print ("  Using the specified location: x = %d, y = %d, xyext = %d"%(x, y, xyframe))
-            flag = 2
+        regisfile = fitsfile.replace("/processing/", "/registration/")
+        if not os.path.isfile(regisfile):
+            shiftfiles = glob.glob(os.path.join(dataDir,'registration', '%s_proc_shift.dat'%fitsfileSplit))
+            shiftfile = shiftfiles[0]
+            refit = operator.not_(doOffRefit)
+            regis2HDU = register_images(fitsfile, shiftfile, xyframe,
+                                        int(opts.xstar), int(opts.ystar),
+                                        path_out_dir, 
+                                        maxdist = maxdist,
+                                        aper_size = aper_size, 
+                                        refit = refit, offtune = doOffTune)
         else:
-            wcsfiles = glob.glob(os.path.join(dataDir,'wcs', '%s*wcs.fits'%fitsfileSplit))
-            print(wcsfiles)
-            if len(wcsfiles)!=0:
-                print ("  Finding it on the wcs file extension...")
-                # The astrometry is successfully found:
-                wcsfile = wcsfiles[0]
-                x, y, xyframe = get_wcs_xy(ra, dec, wcsfile, fitsfile, get_distance = True)
-                print ("  Found the location with wcs solution: x = %d, y=%d, xyext = %d"%(x, y, xyframe))
-            else: # astrometry.net solution failed:
-                flag = 0
-                print ("  Astrometry failed and no wcs specified -- discard %s!"%fitsfileSplit)
-        if flag in [1,2]:
-            if (not opts.xstar == "") and (not opts.ystar == ""):
-                shiftfiles = glob.glob(os.path.join(dataDir,'registration', '%s_proc_shift.dat'%fitsfileSplit))
-                shiftfile = shiftfiles[0]
-                refit = operator.not_(doOffRefit)
-                regis2HDU = register_images(fitsfile, shiftfile, xyframe, int(opts.xstar), int(opts.ystar), path_out_dir, 
-                                            maxdist = maxdist, aper_size = aper_size, 
-                                            refit = refit, offtune = doOffTune)
-            else:
-                print(fitsfile)
-                regis2HDU = register_transients(fitsfile, ra, dec)
-
-            regisfile = fitsfile.replace("/processing/", "/registration/")
-            regisfolder = "/".join(regisfile.split("/")[:-1])
-            if not os.path.isdir(regisfolder):
-                os.makedirs(regisfolder)
-            regisfile = regisfile[:-5]+'_regis.fits'
-
-            if regis2HDU is not None:
-                regis2HDU[0].header["OBJNAME"] = objName
-                regis2HDU[0].header["RA_OBJ"] = ra
-                regis2HDU[0].header["DEC_OBJ"] = dec
-                print ("  Writing to %s"%regisfile)
-                regis2HDU.writeto(regisfile, overwrite=True)
-                print ("")
+            print(fitsfile)
+            regis2HDU = register_transients(regisfile, ra, dec)
+
+        regisfile = regisfile[:-5]+'_regis.fits'
+        if regis2HDU is not None:
+            regis2HDU[0].header["OBJNAME"] = objName
+            regis2HDU[0].header["RA_OBJ"] = ra
+            regis2HDU[0].header["DEC_OBJ"] = dec
+            print ("  Writing to %s"%regisfile)
+            regis2HDU.writeto(regisfile, overwrite=True)
+            print ("")
 
 fitsfiles = sorted(glob.glob(os.path.join(dataDir,'registration','*_regis.fits'))) 
 if day == "20191222" and objName == "ZTFJ0538+1953":
@@ -370,15 +345,14 @@ if doStack:
         plt.savefig(plotName)
         plt.close(fig)
 
-if opts.doCheckCatalog:
+if doMakeMovie:
     result = ps1_query(ra, dec, 20.0/60.0, maxmag=16.0,
                        maxsources=10000)
     ra_ps1, dec_ps1 = result['RAJ2000'], result['DEJ2000']
     moviedir = os.path.join(path_out_dir, 'catalog')
     if not os.path.isdir(moviedir):
         os.makedirs(moviedir)
     cnt = 0
-    print(moviedir)
 
 print ("========================")
 print ("Source Extraction!")
@@ -396,7 +370,7 @@ for ii in range(nfiles):
 
     ztfsub.utils.sextractor(fitsfile,defaultsDir,doSubtractBackground=opts.doSubtractBackground,catfile=catfile,backfile=backfile)
 
-    if opts.doCheckCatalog:
+    if doMakeMovie:
         framenum = 1
         procHDU = fits.open(fitsfile)
         for framenum in range(len(procHDU)):
@@ -426,12 +400,13 @@ for ii in range(nfiles):
             plt.savefig(figname)
             plt.close()
             cnt = cnt + 1
-if opts.doCheckCatalog:
+if doMakeMovie:
     print ("Generating the mpg file...")
     moviefiles = os.path.join(moviedir,"image_%04d.png")
     filename = os.path.join(moviedir,"movie.mpg")
     ffmpeg_command = 'ffmpeg -an -y -r 20 -i %s -b:v %s %s'%(moviefiles,'5000k',filename)
     os.system(ffmpeg_command)
+    shutil.copy(moviedir+"/movie.mpg", outputProDir+"/movie.mpg")
 
 if not doSkipFindRef:
     print ("")
@@ -670,18 +645,3 @@ if doSubtractBackground:
     finalsubforcefile = os.path.join(outputProDir,"lightcurve.subtracted.forced")
     plotsubName = os.path.join(outputProDir,'subtracted_mag_forced.pdf')
     save_forced_tb(tblsubforced, finalsubforcefile, plotsubName)
-
-if doMakeMovie and not doStack:
-    print ("")
-    print ("==================")
-    print ("Making the Movie!")
-    print ("==================")
-    print ("")
-    movieDir = os.path.join(outputDir,'movie')
-    if not os.path.isdir(movieDir):
-        os.makedirs(movieDir)
-    makemovie(movieDir, fitsfiles, aper_size = aper_size,
-              sky_in = sky_in, sky_out = sky_out,
-              halfwidth = halfwidth)
-    shutil.copy(movieDir+"/movie.mpg", outputProDir+"/movie.mpg")
-
diff --git a/bin/kp84_setup_reduction b/bin/kp84_setup_reduction
@@ -443,17 +443,17 @@ for obj in objs:
     if wcsmode!=2:
         ra, dec, radius = get_ra_dec_radius(obj.split("_")[1], opts.object_lists, wcsmode=wcsmode)
     folderName = "%s/%s"%(outputDir,obj)
-    folderName_raw = "%s/raw"%(folderName)
-    fitsfiles = sorted(glob.glob('%s/*.fit*'%(folderName_raw)))
     folderName_processing = "%s/processing"%(folderName)
+    fitsfiles = sorted(glob.glob('%s/*.fit*'%(folderName_processing)))
     folderName_wcs = "%s/wcs"%(folderName)
-    folderName_upload = "%s/upload"%(folderName)
     folderName_sextract = "%s/sextract"%(folderName)
+    folderName_processing_wcs = "%s/processing_wcs"%(folderName)
+    if not os.path.isdir(folderName_processing_wcs):
+        os.makedirs(folderName_processing_wcs)
+
     fitsfiles = sorted(glob.glob('%s/*.fit*'%(folderName_processing)))
     if not os.path.isdir(folderName_wcs):
         os.makedirs(folderName_wcs)
-    if not os.path.isdir(folderName_upload):
-        os.makedirs(folderName_upload)
     folderName_sextract = "%s/sextract"%(folderName)
     for i in range(len(fitsfiles)):
         fitsfile = fitsfiles[i]
@@ -499,14 +499,16 @@ for obj in objs:
                                        index_xyls=index_xyls,
                                        wcsfile=wcsfile)
 
-        header = fits.open(wcsfile)[0].header
-        w = WCS(header)
+        if os.path.isfile(wcsfile):
+            header = fits.open(wcsfile)[0].header
+            w = WCS(header)
 
-        hdu = fits.open(fitsfile)
-        for key in header.keys():
-            if key in ["COMMENT", "HISTORY", "SIMPLE", "BITPIX", "NAXIS", "EXTEND"]: continue
-            hdu[0].header[key] = header[key]
-        hdu.writeto(fitsfile, overwrite=True)
+            wcsfitsfile = os.path.join(folderName_processing_wcs, fitsfile.split("/")[-1])
+            hdu = fits.open(fitsfile)
+            for key in header.keys():
+                if key in ["COMMENT", "HISTORY", "SIMPLE", "BITPIX", "NAXIS", "EXTEND"]: continue
+                hdu[0].header[key] = header[key]
+            hdu.writeto(wcsfitsfile, overwrite=True)
 
 print ("")
 print ("================================")
@@ -520,18 +522,17 @@ for obj in objs:
         continue
     ra, dec, radius = get_ra_dec_radius(obj.split("_")[1], opts.object_lists)
     folderName = "%s/%s"%(outputDir,obj)
-    folderName_processing = "%s/processing"%(folderName)
+    folderName_processing_wcs = "%s/processing_wcs"%(folderName)
     folderName_registration = "%s/registration"%(folderName)
-    folderName_wcs = "%s/wcs"%(folderName)
-    folderName_upload = "%s/upload"%(folderName)
-    fitsfiles = sorted(glob.glob('%s/*.fit*'%(folderName_processing)))
+    fitsfiles = sorted(glob.glob('%s/*.fit*'%(folderName_processing_wcs)))
     if not os.path.isdir(folderName_registration):
         os.makedirs(folderName_registration)
     if wcsmode!=2:
         ra, dec, radius = get_ra_dec_radius(obj.split("_")[1], opts.object_lists, wcsmode=wcsmode)
     for i in range(len(fitsfiles)):
         fitsfile = fitsfiles[i]
         filename = fitsfile.split("/")[-1].split(".fit")[0]
+        registerfitsfile = os.path.join(folderName_registration, fitsfile.split("/")[-1])
         print ("%s"%filename)
         print ("  Calculating shift w.r.t extension 1...")
         tb_shift, data_stacked = stack_shifted_frames(fitsfile, fix_ref = doFixRef)
@@ -548,4 +549,4 @@ for obj in objs:
                     hdu[ii].header[key] = header[key] - tb_shift[ii]["yshift"]
                 else:
                     hdu[ii].header[key] = header[key]
-        hdu.writeto(fitsfile, overwrite=True)
+        hdu.writeto(registerfitsfile, overwrite=True)
diff --git a/bin/kp84_slack_bot.py b/bin/kp84_slack_bot.py
@@ -181,7 +181,7 @@ def run_reductions(channel_id, setupDir="", outputDir="", bypass=False,
 
             if not os.path.isdir(outputProDir):
                 if objType == "variable":
-                    setup_command = "python kp84_photometric_reduction --day %s --objName %s --doMakeMovie --doDynamicAperture --halfwidth 200 --doCheckCatalog" % (day, objNameTmp)
+                    setup_command = "python kp84_photometric_reduction --day %s --objName %s --doMakeMovie --doDynamicAperture --halfwidth 200" % (day, objNameTmp)
                 elif objType == "transient":
                     setup_command = "python kp84_photometric_reduction --day %s --objName %s --doMakeMovie --doDynamicAperture --doTransient --doSubtraction" % (day, objNameTmp)
                 else:

diff --git a/kp84/models.py b/kp84/models.py
@@ -306,6 +306,7 @@ def ingest_images(config, lookback, repeat=False):
                 imgid = int(filenameSplit[1] + filenameSplit[2])
                 objname = filenameSplit[3]
 
+                if len(hdul) < 2: continue
                 gpstime_start = Time(hdul[1].header['GPS_TIME'],
                                      format='isot', scale='utc')
                 gpstime_end = Time(hdul[-1].header['GPS_TIME'],

diff --git a/kp84/reduction_utils.py b/kp84/reduction_utils.py
@@ -251,12 +251,9 @@ def get_reference_pos(scienceimage, cat, objName, zp=0, passband='sg', xoff=0, y
     if rafield==None:
         ra = sciHDU[0].header["RA_OBJ"]
         dec = sciHDU[0].header["DEC_OBJ"]
-        wcsauto = sciHDU[0].header["WCSAUTO"]
+        wcsauto = "CRPIX1" in sciHDU[0].header
 
-        if "stack" in scienceimage:
-            framenum = 1
-        else:
-            framenum = sciHDU[0].header["WCSFRAME"]
+        framenum = 1
 
         if wcsauto==1:
             w0 = WCS(sciHDU[framenum].header)

diff --git a/kp84/templates/date.html b/kp84/templates/date.html
@@ -10,25 +10,34 @@
 	    {% endblock %}
 
 {% block body %}
-	<h2>Images from specified date</h2>
+<h2>Images from specified date</h2>
 	<div class=list-group>
-		{% for im in ims %}
-		<p>{{ "Filename: " + im.filename }}
+		{% for exposure in exposures %}
+		<p>
+		{{ "Object:" }}
+		<a href={{ "http://localhost:8001/obj/" + exposure.objname + "/" }}>{{ exposure.objname }}</a>
 		<br>
-		{{ "Object Name: " + im.objname }}
-		<br>
-		{{ "Exposure Time:" }}
-		{{ im.exposure_time }}
-		<br>
-		{{ "Date:" }}
-		{{ im.date }}
-		<br>
-		{{ "Right Acension:" }}
-		{{ im.RA }}
-		<br>
-		{{ "Declination:" }}
-		{{ im.Dec }}</p>
-	</a>
+		{% for obj in objs %}
+			{{ "RA:"}}
+			{{ obj.RA }}
+			<br>
+			{{ "Dec:" }}
+			{{ obj.Dec }}
+			<br>
+		{% endfor %}
+		{% for cube in exposure.cubes %}
+			{{ "Exposure Time:" }}
+			{{ cube.exposure_time }}
+			<br>
+			{{ "Filename:" }}
+			{{ cube.filename }}
+			<img src={{ "http://localhost:8001/obj/" + exposure.objname + "/image_id/" + exposure.image_id|string + "/exposure.png"}} alt = {{ exposure.image_id|string }}>
+			<br>
+		{% endfor %}
+		{% for reduction in exposure.reductions %}
+			<img src={{ "http://localhost:8001/obj/" + reduction.objname + "/image_id/" + reduction.image_id|string + "/reduction.png"}} alt = {{ reduction.image_id|string }}>
+		{% endfor %}
+		</p>
 	{% endfor %}
     </ul>
 {% endblock %}

diff --git a/kp84/templates/obj.html b/kp84/templates/obj.html
@@ -29,23 +29,28 @@ <h2>Images of specified object</h2>
             <p>
             {{ "Exposure Date:" }}
             {{ exposure.date }}
-            <br>
+	    <a href={{ "htt://localhost:8001/date/" + exposure.dateshort + "/" }}>{{ "More exposures from this date" }}</a>
+	    <br>
             {{ "Image ID:" }}
             {{ exposure.image_id }}
+
 	    {% for cube in exposure.cubes %}
-   	        <p>
+ 		<br>
 	        {{ "Exposure Time:" }}
 	        {{ cube.exposure_time }}
-	        <br>
+
+		<br>
+
 	        {{ "Filename:" }}
 	        {{ cube.filename }}
             <img src={{ "http://localhost:8001/obj/" + exposure.objname + "/image_id/" + exposure.image_id|string + "/exposure.png" }} alt={{ exposure.image_id|string }}>
 	        <br>
             {% endfor %}
             {% for reduction in exposure.reductions %}
-                <p>
+	    	<br>
 		<img src={{ "http://localhost:8001/obj/" + reduction.objname + "/image_id/" + reduction.image_id|string + "/reduction.png" }} alt={{ reduction.image_id|string }}>
-	    {% endfor %} 
+	    	</p>
+		{% endfor %} 
         {% endfor %}
     </ul>
 {% endblock %}

diff --git a/kp84/views.py b/kp84/views.py
@@ -213,22 +213,20 @@ def reduction(objname, image_id):
 @app.route('/date/<day>/')
 def date(day):
 
-    query = models.db.session.query(models.Image.dateshort == day)
+    query = models.db.session.query(models.Exposure).filter_by(dateshort=day)
 
     try:
-        idxs = query.all()
+        date = query.first()
     except NoResultFound:
         abort(404)
 
-    imsall = models.db.session.query(models.Image).all()
-    ims = []
-    for im, idx in zip(imsall, idxs):
-        if idx[0] == False: continue
-        ims.append(im)
-
-    print(im.dateshort)
+    exposures = models.db.session.query(models.Exposure).filter_by(dateshort=day)
+    objname = exposures.objname
+    obj = models.db.session.query(models.Object).filter_by(objname=objname)
 
     return render_template(
         'date.html',
-        ims=ims)
+        date=date,
+        objs=objs,
+        exposures=exposures)