Release 2.4.0 (#347)

Added W7 Standard Bosses
Optimized profile fetching to fetch one era at a time
Fixed restat calculation to only work over new encounters
Fixed bug where loading a page would make 2 API requests
Other fixes and improvements

analyser/analyser.py

@@ -654,7 +654,7 @@ def determine_success_reward(self, events, encounter):
         success_time = events.time.max()
         success = False
         if self.boss_info.kind == Kind.RAID and encounter.version >= '20170905':
-            success_types = [55821, 60685]
+            success_types = [55821, 60685, 914, 22797]
             if (not events[(events.state_change == parser.StateChange.REWARD)
                              & events.value.isin(success_types)].empty):
                 success = True

analyser/bosses.py

@@ -523,6 +523,36 @@ def find_end_time(self,
         Metric('Fire Wave', 'Shockwave', MetricType.COUNT, True, False),
         Metric('Fiery Dance', 'Fiery Platform Edges', MetricType.COUNT, True, False),
     ]),
+    # W7 Bosses
+    Boss('Cardinal Adina', Kind.RAID, [22006], enrage = 8 * MINUTES, success_min_health_limit = 95, phases = [
+        Phase("Phase 1", True, phase_end_health = 75, phase_end_damage_stop = 10000),
+        Phase("Hands 1", False, phase_end_damage_start = 1000),
+        Phase("Phase 2", True, phase_end_health = 50, phase_end_damage_stop = 10000),
+        Phase("Hands 2", False, phase_end_damage_start = 1000),
+        Phase("Phase 3", True, phase_end_health = 25, phase_end_damage_stop = 10000),
+        Phase("Hands 3", False, phase_end_damage_start = 1000),
+        Phase("Phase 4", True)
+    ]),
+    Boss('Cardinal Sabir', Kind.RAID, [21964], enrage = 9 * MINUTES, success_min_health_limit = 95, phases = [
+        Phase("Phase 1", True, phase_end_health = 80, phase_end_damage_stop = 1000),
+        Phase("Jump Puzzle 1", False, phase_end_damage_start = 1000),
+        Phase("Phase 2", True, phase_end_health = 60, phase_end_damage_stop = 1000),
+        Phase("Jump Puzzle 2", False, phase_end_damage_start = 1000),
+        Phase("Phase 3", True)
+    ]),
+        Boss('Qadim the Peerless', Kind.RAID, [22000], enrage = 12 * MINUTES, success_min_health_limit = 95, phases = [
+        Phase("Phase 1", True, phase_end_health = 80, phase_end_damage_stop = 1000),
+        Phase("Fire Phase 1", False, phase_end_damage_start = 1000),
+        Phase("Phase 2", True, phase_end_health = 60, phase_end_damage_stop = 1000),
+        Phase("Fire Phase 2", False, phase_end_damage_start = 1000),
+        Phase("Phase 3", True, phase_end_health = 40, phase_end_damage_stop = 1000),
+        Phase("Pylon Break 1", False, phase_end_damage_start = 1000),
+        Phase("Phase 4", True, phase_end_health = 30, phase_end_damage_stop = 1000),
+        Phase("Pylon Break 2", False, phase_end_damage_start = 1000),
+        Phase("Phase 5", True, phase_end_health = 20, phase_end_damage_stop = 1000),
+        Phase("Pylon Break 3", False, phase_end_damage_start = 1000),
+        Phase("Phase 6", True)
+    ]),
     Boss('Standard Kitty Golem', Kind.DUMMY, [16199]),
     Boss('Average Kitty Golem', Kind.DUMMY, [16177]),
     Boss('Vital Kitty Golem', Kind.DUMMY, [16198]),
@@ -639,6 +669,14 @@ def find_end_time(self,
                     IDS['Qadim'],
                 ],
             },
+            {
+                "name": "The Key of Ahdashim",
+                "bosses": [
+                    IDS['Cardinal Adina'],
+                    IDS['Cardinal Sabir'],
+                    IDS['Qadim the Peerless'],
+                ],
+            },            
         ],
     },
     {

gw2raidar/settings.py

@@ -14,8 +14,8 @@
 from dateutil import parser
 
 VERSION = {
-        'id': '2.3.6',
-        'timestamp': 1552259255, # date +%s
+        'id': '2.4.0',
+        'timestamp': 1563162107, # date +%s
         }
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)

raidar/management/commands/restat.py

@@ -227,14 +227,24 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         with single_process('restat'), necessary() as last_run:
             start = time()
+            start_date = datetime.now()
             self.delete_old_files(*args, **options)
-            self.calculate_stats(last_run, *args, **options)
+            eraCount, areasCount, usersCount, newEncountersCount = self.calculate_stats(last_run, *args, **options)
             end = time()
+            end_date = datetime.now()
 
             if options['verbosity'] >= 1:
                 print()
                 print("Completed in %ss" % (end - start))
-
+
+            RestatPerfStats.objects.create(
+                started_on=start_date,
+                ended_on=end_date,
+                number_users=usersCount,
+                number_eras=eraCount,
+                number_areas=areasCount,
+                number_new_encounters=newEncountersCount,
+                was_force=options['force'])
 
     def delete_old_files(self, *args, **options):
         GB = 1024 * 1024 * 1024
@@ -278,16 +288,16 @@ def add_leaderboard_stats(container, period, stat, item):
             leaderboards[stat].append(item)
             leaderboards[stat] = sorted(leaderboards[stat], key=lambda x: x[stat])[:10]
 
-        def initialise_era_area_stats(count):
+        def initialise_era_area_stats():
             leaderboards = {
                     'periods': {},
                 }
             return {}, leaderboards
 
-        def initialise_era_user_stats(count):
+        def initialise_era_user_stats():
             return {}
 
-        def initialise_era_stats(count):
+        def initialise_era_stats():
             return {}
 
 
@@ -418,18 +428,18 @@ def add_participation_to_era_user_stats(participation, totals_for_player):
                 raise RestatException("Error in %s" % participation)
 
 
-        def finalise_era_area_stats(era, area, totals_in_area, leaderboards_in_area):
+        def finalise_era_area_stats(era, area_id, totals_in_area, leaderboards_in_area):
             finalise_stats(totals_in_area)
             EraAreaStore.objects.update_or_create(
-                    era=era, area=area, defaults={
+                    era=era, area_id=area_id, defaults={
                         "val": totals_in_area,
                         "leaderboards": leaderboards_in_area,
                     })
 
-        def finalise_era_user_stats(era, user, totals_for_player):
+        def finalise_era_user_stats(era, user_id, totals_for_player):
             finalise_stats(totals_for_player)
             EraUserStore.objects.update_or_create(
-                    era=era, user=user, defaults={ "val": totals_for_player })
+                    era=era, user_id=user_id, defaults={ "val": totals_for_player })
 
         def finalise_era_stats(era, totals_in_era):
             finalise_stats(totals_in_era)
@@ -445,46 +455,71 @@ def verbose(title, content):
                 for key in sorted(flattened.keys()):
                     print_node(key, flattened[key])
 
-        def calculate_area_stats(era):
-            totals_in_era = initialise_era_stats(era.encounters.count())
-
-            area_queryset = Area.objects.all()
-            for area in area_queryset.iterator():
-                encounter_queryset = era.encounters.prefetch_related('participations__account').filter(area=area).order_by('?')
-                totals_in_area, leaderboards_in_area = initialise_era_area_stats(encounter_queryset.count())
-
-                if area.id in BOSSES:
-                    kind = BOSSES[area.id].kind.name.lower()
-                else:
-                    kind = "unknown"
-                totals_for_kind = navigate(totals_in_era, 'kind', 'All %s bosses' % kind)
-                for encounter in encounter_queryset.iterator():
-                    add_encounter_to_era_area_stats(encounter, totals_in_area, totals_for_kind, leaderboards_in_area)
-                finalise_era_area_stats(era, area, totals_in_area, leaderboards_in_area)
-                verbose("Totals for era %s, area %s" % (era, area), totals_in_area)
+        def calculate_area_stats(era, new_encounters, forceRecalulation):
+            totals_in_era = initialise_era_stats()
+            areasCount = 0
+            area_queryset = new_encounters.order_by('area_id').distinct('area').values('area')
+            for area in area_queryset:
+                area_id = area['area']
+                if area_id:
+                    areasCount = areasCount + 1
+                    encounter_queryset = Encounter.objects.filter(area=area_id, era=era).order_by('?')
+                    totals_in_area, leaderboards_in_area = initialise_era_area_stats()
+
+                    if area_id in BOSSES:
+                        kind = BOSSES[area_id].kind.name.lower()
+                    else:
+                        kind = "unknown"
+                    totals_for_kind = navigate(totals_in_era, 'kind', 'All %s bosses' % kind)
+                    for encounter in encounter_queryset.iterator():
+                        add_encounter_to_era_area_stats(encounter, totals_in_area, totals_for_kind, leaderboards_in_area)
+                    finalise_era_area_stats(era, area_id, totals_in_area, leaderboards_in_area)
+                    verbose("Totals for era %s, area %s" % (era, area_id), totals_in_area)
 
             finalise_era_stats(era, totals_in_era)
             verbose("Totals for era %s" % era, totals_in_era)
-
-
-        def calculate_user_stats(era):
-            user_queryset = User.objects.all()
-            for user in user_queryset.iterator():
-                participation_queryset = Participation.objects.prefetch_related('encounter', 'account').filter(account__user=user, encounter__era=era).order_by('?')
-                totals_for_player = initialise_era_user_stats(participation_queryset.count())
-                for participation in participation_queryset.iterator():
-                    add_participation_to_era_user_stats(participation, totals_for_player)
-                finalise_era_user_stats(era, user, totals_for_player)
-                verbose("Totals for era %s, user %s" % (era, user), totals_for_player)
-
-
+            return areasCount
+
+
+        def calculate_user_stats(era, new_encounters, forceRecalulation):
+            participations_queryset = Participation.objects.filter(encounter__in=new_encounters)
+            usersCount = 0
+            unique_user_queryset = participations_queryset.order_by('account__user').distinct('account__user').values('account__user')
+            for user in unique_user_queryset:
+                user_id = user['account__user']
+                if user_id:
+                    usersCount = usersCount + 1
+                    participation_queryset = participations_queryset.filter(account__user=user['account__user'], encounter__era=era).order_by('?')
+                    totals_for_player = {}
+                    if not forceRecalulation:
+                        try:
+                            totals_for_player = EraUserStore.objects.get(era=era, user=user['account__user']).val
+                        except EraUserStore.DoesNotExist:
+                            pass
+                    for participation in participation_queryset.iterator():
+                        add_participation_to_era_user_stats(participation, totals_for_player)
+                    finalise_era_user_stats(era, user['account__user'], totals_for_player)
+                    verbose("Totals for era %s, user %s" % (era, user['account__user']), totals_for_player)
+            return usersCount
+
+        eraCount = 0
+        newEncountersCount = 0
+        areasCount = 0
+        usersCount = 0
         for era in Era.objects.all():
-            new_encounters = Encounter.objects.filter(era=era, uploaded_at__gte=last_run).count()
-            if new_encounters or options['force']:
-                calculate_area_stats(era)
-                calculate_user_stats(era)
+            eraCount = eraCount + 1
+            forceRecalulation = options['force']
+            last_run_timestamp = last_run
+            if forceRecalulation:
+                last_run_timestamp = 0
+            new_encounters = Encounter.objects.filter(era=era, uploaded_at__gte=last_run_timestamp)
+            if new_encounters:
+                newEncountersCount = newEncountersCount + len(new_encounters) # fine because we're iterating over all of them anyway
+                areasCount = areasCount + calculate_area_stats(era, new_encounters, forceRecalulation)
+                usersCount = usersCount + calculate_user_stats(era, new_encounters, forceRecalulation)
             elif options['verbosity'] >= 2:
                 print('Skipped era %s' % era)
+        return eraCount, areasCount, usersCount, newEncountersCount
 
 
 

raidar/migrations/0041_restat_metrics.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.3 on 2019-07-13 17:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('raidar', '0040_eraareastore_leaderboards_value'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RestatPerfStats',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('started_on', models.DateTimeField()),
+                ('ended_on', models.DateTimeField()),
+                ('number_users', models.IntegerField()),
+                ('number_eras', models.IntegerField()),
+                ('number_areas', models.IntegerField()),
+                ('number_new_encounters', models.IntegerField()),
+                ('was_force', models.BooleanField()),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='userprofile',
+            name='privacy',
+            field=models.PositiveSmallIntegerField(choices=[(1, 'Private'), (2, 'Squad'), (3, 'Public')], default=2, editable=False),
+        ),
+    ]

raidar/models.py

@@ -386,3 +386,12 @@ def leaderboards(self, value):
 class EraUserStore(ValueModel):
     era = models.ForeignKey(Era, on_delete=models.CASCADE, related_name="era_user_stores")
     user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="era_user_stores")
+
+class RestatPerfStats(models.Model):
+    started_on = models.DateTimeField()
+    ended_on = models.DateTimeField()
+    number_users = models.IntegerField()
+    number_eras = models.IntegerField()
+    number_areas = models.IntegerField()
+    number_new_encounters = models.IntegerField()
+    was_force = models.BooleanField()

raidar/static/raidar/script.js

@@ -694,14 +694,13 @@ ${body}
         'page.area': 'All raid bosses',
       });
       $.get({
-        url: 'profile.json',
+        url: getPageURLFromObject(page).substring(1) + '.json',
       }).then(setData).then(() => {
-        let eras = r.get('profile.eras');
-        let eraOrder = Object.values(eras)
-          .filter(era => 'encounter' in era.profile)
+        let era = r.get('profile.era');
+        let eraOrder =  Object.values(r.get('profile.eras_for_dropdown'))
           .sort((e1, e2) => e2.started_at - e1.started_at);
         r.set({
-          'page.era': eraOrder[0].id,
+          'page.era': Object.keys(era)[0],
           'profile.era_order': eraOrder,
         });
       });
@@ -928,15 +927,6 @@ ${body}
     }
     return false;
   }
-  let url = getPageURLFromObject(initPage);
-  history.replaceState(initPage, null, url);
-  if (pageInit[initPage.name]) {
-    pageInit[initPage.name](initPage);
-  }
-  if (window.ga) {
-    window.ga('set', 'page', url);
-    window.ga('send', 'pageview');
-  }
 
   function notification(str, style) {
     UIkit.notification(str, style);

raidar/templates/raidar/info_releasenotes.html

@@ -17,6 +17,14 @@ <h4>
   <span class="ignore">released on</span>
   [[formatDate(data.version.timestamp)]]
 </h4>
+<ul>
+  <li>Added W7 Standard Bosses</li>
+  <li>Optimized profile fetching to fetch one era at a time</li>
+  <li>Fixed restat calculation to only work over new encounters</li>
+  <li>Fixed bug where loading a page would make 2 API requests</li>
+  <li>Other fixes and improvements</li>
+</ul>
+<h4>Version 2.3.6</h4>
 <ul>
   <li>Added HP check to all Raid and Fractal bosses</li>
 </ul>

raidar/templates/raidar/profile.html

@@ -1,6 +1,6 @@
 {% load staticfiles %}
 [[#with profile]]
-  [[#with profile.eras[page.era]]]
+  [[#with profile.era[page.era]]]
     [[#with profile]]
       <div class="uk-button-group uk-width-1-1 boundary uk-margin-bottom uk-margin-top uk-visible@s">
         [[#each data.boss_locations]]
@@ -83,11 +83,11 @@ <h6 class="uk-font-family-secondary uk-hidden@s uk-margin-top">Encounter</h6>
 
       <div>
         <h6 class="uk-font-family-secondary uk-text-uppercase">Time Frame</h6>
-        <button class="uk-button uk-width-auto@l uk-width-1-1 uk-margin-right@l" type="button">[[eras[page.era].name]] <i class="material-icons">expand_more</i></button>
+        <button class="uk-button uk-width-auto@l uk-width-1-1 uk-margin-right@l" type="button">[[name]] <i class="material-icons">expand_more</i></button>
         <div uk-dropdown="mode: click">
           <ul class="uk-nav uk-dropdown-nav">
             [[#each era_order]]
-              <li><a href="#" on-click="@this.set({'page.era': id})">[[name]]</a></li>
+              <li><a on-click="@this.fire('global_stats_nav', 'page.no', id)">[[name]]</a></li>
             [[/each]]
           </ul>
         </div>

raidar/urls.py

@@ -30,7 +30,7 @@
     url(r'^change_password.json$', views.change_password, name = "change_password"),
     url(r'^add_api_key.json$', views.add_api_key, name = "add_api_key"),
     url(r'^encounter/(?P<url_id>\w+)(?P<json>\.json)?$', views.encounter, name = "encounter"),
-    url(r'^profile.json$', views.profile, name = "profile"),
+    url(r'^profile(?:/(?P<era_id>[0-9]+))?\.json$', views.profile, name = "profile"),
     url(r'^reset/(?P<uidb64>[0-9A-Za-z_\-]+)/(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$',
         auth_views.password_reset_confirm, name='password_reset_confirm'),
     url(r'^reset/done/$', auth_views.password_reset_complete, name='password_reset_complete'),