Merge pull request #9 from FRBs/kko

FRB Status + KKO

YSE_App/admin.py

@@ -84,4 +84,8 @@ class UserQueryAdmin(admin.ModelAdmin):
 # FRBs
 admin.site.register(FRBTransient)
 admin.site.register(FRBFollowUpResource)
-admin.site.register(FRBGalaxy)
+admin.site.register(FRBFollowUpRequest)
+admin.site.register(FRBFollowUpObservation)
+admin.site.register(FRBGalaxy)
+admin.site.register(Path)
+admin.site.register(GalaxyPhotData)

YSE_App/chime/chime_kko.py

@@ -0,0 +1,60 @@
+""" Build up the Test DB (may be more than one option)"""
+
+import datetime
+import os
+from pkg_resources import resource_filename
+
+from django.contrib import auth
+
+import pandas
+
+from YSE_App.models import *
+from YSE_App.chime import chime_test_utils as ctu
+from YSE_App import frb_init
+from YSE_App import frb_status
+from YSE_App import frb_utils
+
+def init_kko(clean:bool=True):
+    """ Build the CHIME KKO DB from scratch """
+
+    user = auth.authenticate(username='root', password='F4isthebest')
+
+    # Clean first
+    ctu.clean_all(skip_resources=True)
+
+    # ##############################
+    # Init the DB
+    # Status
+    frb_init.init_statuses(user)
+
+    # Observing Groups
+    frb_init.init_obsgroups(user)
+
+    # Load up the table
+    csv_file = os.path.join(
+        resource_filename('YSE_App', 'chime'), 'chime_kko_sep2023.csv')
+    df_frbs = pandas.read_csv(csv_file)
+
+    # Add CHIME Observation Group?
+    survey_names = [survey.name for survey in FRBSurvey.objects.all()]
+    for survey in ['CHIME/FRB']:
+        if survey not in survey_names:
+            obs = FRBSurvey(name=survey, created_by_id=user.id, modified_by_id=user.id)
+            obs.save()
+
+    # Add em 
+    transients = ctu.add_df_to_db(df_frbs, user, 
+                         delete_existing=False)
+
+    # Set the KKO tag
+    kko_tag = frb_utils.add_or_grab_obj(
+            FRBTag, dict(name='CHIME-KKO'), {}, user)
+    unkn_tag = frb_utils.add_or_grab_obj(
+            FRBTag, dict(name='CHIME-Unknown'), {}, user)
+    for transient in transients:
+        # Drop unknown
+        transient.frb_tags.remove(unkn_tag)
+        #
+        transient.frb_tags.add(kko_tag)
+        # Update status (and save)
+        frb_status.set_status(transient)

YSE_App/chime/chime_test_db.py

@@ -8,7 +8,8 @@
 
 from YSE_App.models import *
 from YSE_App.chime import chime_path_test
-from YSE_App.data_utils import add_or_grab_obj
+from YSE_App.chime import chime_test_utils
+from YSE_App.frb_utils import add_or_grab_obj
 from YSE_App import frb_init
 from YSE_App import frb_observing
 
@@ -18,12 +19,12 @@ def build_chime_test_db():
     user = auth.authenticate(username='root', password='F4isthebest')
 
     # Clean first
-    clean_all()
+    chime_test_utils.clean_all()
 
     # ##############################
     # Init the DB
     # Status
-    frb_init.init_status(user)
+    frb_init.init_statuses(user)
 
     # Add objects
 
@@ -68,22 +69,3 @@ def build_chime_test_db():
     obsplan = pandas.DataFrame([row])
     code, msg = frb_observing.ingest_obsplan(obsplan, user)
 
-
-def clean_all():
-    """ Wipe clean the DB """
-
-    print("Removing Path objects")
-    for ipath in Path.objects.all():
-        ipath.delete()
-
-    print("Removing FRBGalaxy objects")
-    for galaxy in FRBGalaxy.objects.all():
-        galaxy.delete()
-
-    print("Removing FRBFollowUpResource objects")
-    for frb_fu in FRBFollowUpResource.objects.all():
-        frb_fu.delete()
-
-    print("Removing FRBTransient objects")
-    for itransient in FRBTransient.objects.all():
-        itransient.delete()

YSE_App/chime/chime_test_utils.py

@@ -4,7 +4,7 @@
 
 from django.db.models import ForeignKey
 
-from YSE_App.models import FRBTransient
+from YSE_App.models import *
 
 def add_df_to_db(df_frbs:pandas.DataFrame, user, delete_existing:bool=False):
     """ Add a pandas DataFrame of FRBs to the database
@@ -75,4 +75,25 @@ def add_df_to_db(df_frbs:pandas.DataFrame, user, delete_existing:bool=False):
         dbtransient.save()
 
     # Return
-    return dbtransients
+    return dbtransients
+
+def clean_all(skip_resources:bool=False):
+    """ Wipe clean the DB """
+
+    print("Removing Path objects")
+    for ipath in Path.objects.all():
+        ipath.delete()
+
+    print("Removing FRBGalaxy objects")
+    for galaxy in FRBGalaxy.objects.all():
+        galaxy.delete()
+
+    # FRBFollowUpResource
+    if not skip_resources:
+        print("Removing FRBFollowUpResource objects")
+        for frb_fu in FRBFollowUpResource.objects.all():
+            frb_fu.delete()
+
+    print("Removing FRBTransient objects")
+    for itransient in FRBTransient.objects.all():
+        itransient.delete()

YSE_App/chime/tags.py

@@ -5,11 +5,13 @@
 # "Unbiased" sample
 blind_sample = dict(
     name='CHIME-Blind',
-    version='1.0',
+    version='0.1',
     ra_rngs=[[0, 225]],
     dec_rngs=[[-10, 10]],
     max_EBV=0.3,
-    min_POx=0.9,
+    mr_max=24.5,     # Faintest magnitude to try spectroscopy
+    max_P_Ux=0.5,    # Host is consdiered unseen if P(U|x) is greater than this
+    min_POx=0.0,    # Minimum P(O|x) to consider (of top 2)
     end_date='2030-12-31', # Not implemented yet
     prob=0.3,
     label=['Stripe 82'],
@@ -18,20 +20,34 @@
 # High DM (notional)
 highDM_sample = dict(
     name='CHIME-HighDM',
-    version='0.0',
+    version='0.1',
     max_EBV=0.3,
     min_DM=1000.,
+    max_P_Ux=0.5,    # Host is consdiered unseen if P(U|x) is greater than this
+    mr_max=24.5,   # Faintest magnitude to try spectroscopy
     prob=0.8,
     )
 
 # Repeater (notional)
 repeater_sample = dict(
     name='CHIME-Repeater',
-    version='0.0',
+    version='0.1',
     prob=0.8,
+    mr_max=24.5,   # Faintest magnitude to try spectroscopy
     )
 
-all_samples = [blind_sample, highDM_sample, repeater_sample]
+# High DM (notional)
+kko_sample = dict(
+    name='CHIME-KKO',
+    version='0.1',
+    max_EBV=0.3,
+    mr_max=23.,      # Faintest magnitude to try spectroscopy
+    min_POx=0.9,     # Minimum P(O|x) to consider (of top 2)
+    max_P_Ux=0.5,    # Host is consdiered unseen if P(U|x) is greater than this
+    prob=0.8,
+    )
+
+all_samples = [blind_sample, highDM_sample, repeater_sample, kko_sample]
 
 
 def set_from_instance(instance):
@@ -52,7 +68,7 @@ def set_from_instance(instance):
     # ###################################
     # Blind?
     flag_blind = False
-    for ra_rng, dec_rng, label in zip(
+    for ra_rng, dec_rng, _ in zip(
         blind_sample['ra_rngs'], 
         blind_sample['dec_rngs'], 
         blind_sample['label']):

YSE_App/data_utils.py

@@ -1429,37 +1429,6 @@ def getRADecBox(ra,dec,size=None,dec_size=None):
             ramax+=360.0
     return(ramin,ramax,decmin,decmax)
 
-def add_or_grab_obj(iclass, uni_fields:dict, extra_fields:dict, user=None):
-    """ Convenience utility to add/grab an object in the DB
-
-    Args:
-        iclass (django model): Model to add/grab
-        uni_fields (dict): 
-            dict of fields that uniquely identify the object
-        extra_fields (dict): 
-            dict of additional fields to add to the object if it is created
-        user (django user object, optional): user object. Defaults to None.
-
-    Returns:
-        django instance: Object either grabbed or added to the DB
-    """
-    try:
-        obj = iclass.objects.get(**uni_fields)
-    except ObjectDoesNotExist:
-        # Merge
-        all_fields = uni_fields.copy()
-        all_fields.update(extra_fields)
-        # Add user?
-        if user is not None:
-            all_fields['created_by'] = user
-            all_fields['modified_by'] = user
-        obj = iclass(**all_fields)
-        obj.save()
-        print("Object created")
-    else:
-        print("Object existed, returning it")
-    # Return
-    return obj
 
 
 # FRB items
@@ -1557,7 +1526,6 @@ def ingest_path(request):
       - obs_group (str): name of the instrument; must be present in the
         ObservationGroup table
       - P_Ux (float): Unseen posterior;  added to the transient
-      - deeper (bool): Is a deeper image required?
 
     Args:
         request (requests.request): 
@@ -1589,7 +1557,6 @@ def ingest_path(request):
             data['F'], 
             data['instrument'], data['obs_group'],
             data['P_Ux'], user,
-            data['deeper'],
             remove_previous=True) # May wish to make this optional
     except:
         print("Ingestion failed")
@@ -1658,6 +1625,7 @@ def ingest_obsplan(request):
         -- TNS: TNS name
         -- Resource: Resource name
         -- mode: observing mode ['image', 'longslit', 'mask']
+      - override (bool): if True, will override existing entries
 
     Args:
         request (requests.request): 
@@ -1680,7 +1648,8 @@ def ingest_obsplan(request):
     obs_tbl = pandas.read_json(data['table'])
 
     # Run
-    code, msg = frb_observing.ingest_obsplan(obs_tbl, user)
+    code, msg = frb_observing.ingest_obsplan(obs_tbl, user,
+                                            override=data['override'])
 
     # Return
     return JsonResponse({"message":f"{msg}"}, status=code)
@@ -1703,6 +1672,7 @@ def ingest_obslog(request):
             -- texp (float)
             -- date (timestamp)
             -- success (bool)
+       - override (bool): if True, will override existing entries
 
     Args:
         request (requests.request): 
@@ -1725,7 +1695,43 @@ def ingest_obslog(request):
     obs_tbl = pandas.read_json(data['table'])
 
     # Run
-    code, msg = frb_observing.ingest_obslog(obs_tbl, user)
+    code, msg = frb_observing.ingest_obslog(obs_tbl, user,
+                                            override=data['override'])
 
     # Return
-    return JsonResponse({"message":f"{msg}"}, status=code)
+    return JsonResponse({"message":f"{msg}"}, status=code)
+
+
+@csrf_exempt
+@login_or_basic_auth_required
+def add_frb_followup_resource(request):
+    """ Add an FRBFollowUpResource to the DB from an 
+    outside request
+
+    This is mainly intended for recovering from a "problem"
+
+    Args:
+        request (_type_): _description_
+
+    Returns:
+        JsonResponse: _description_
+    """
+
+    data = JSONParser().parse(request)
+
+    # Authenticate
+    # TODO -- SHOULD RESTRICT TO ADMIN
+    auth_method, credentials = request.META['HTTP_AUTHORIZATION'].split(' ', 1)
+    credentials = base64.b64decode(credentials.strip()).decode('utf-8')
+    username, password = credentials.split(':', 1)
+    user = auth.authenticate(username=username, password=password)
+
+    # Use Serializer
+    serializer = FRBFollowUpResourceSerializer(data=data)
+    if serializer.is_valid():
+        serializer.save()
+        print(f"Generated FRBFollowUpResource: {data['name']}")
+    else:
+        print(f"Not valid!")
+
+    return JsonResponse(serializer.data, status=201)